From 0084f2e068c66e397f0f1cbad3ca2a548d3857a4 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:30:35 +0000 Subject: [PATCH 01/11] refactor(names)!: runtime strings spell the new names Error, panic and assert messages, sketch_type() labels and test output that named an old type, method or field from #209 now name the new one. Algorithm names in prose and ASAPv1 wire strings are unchanged. Co-Authored-By: Claude Opus 5.5 --- examples/accuracy_probe.rs | 50 ++++++------ examples/compare_univmon_q.rs | 14 ++-- examples/compare_univmon_q_bundle.rs | 2 +- examples/evaluate_univmon_q_entropy.rs | 2 +- examples/evaluate_univmon_q_skew.rs | 8 +- examples/hhheap_probe.rs | 11 ++- src/common/hash.rs | 2 +- src/common/structure_utils.rs | 2 +- src/common/structures/fixed_structure.rs | 4 +- src/common/structures/matrix_storage.rs | 4 +- src/common/structures/vector3d.rs | 16 ++-- src/message_pack_format/envelope.rs | 16 ++-- .../portable/countminsketch.rs | 4 +- .../portable/set_aggregator/wire.rs | 8 +- src/message_pack_format/wire_key.rs | 30 +++++-- src/sketch_framework/eh/wire.rs | 2 +- src/sketch_framework/eh_bucket_sketch.rs | 8 +- src/sketch_framework/eh_bucket_sketch/wire.rs | 24 +++--- src/sketch_framework/eh_univmon.rs | 12 +-- src/sketch_framework/hashlayer.rs | 6 +- src/sketch_framework/hydra/wire.rs | 7 +- src/sketch_framework/octo.rs | 4 +- src/sketch_framework/tumbling.rs | 8 +- src/sketch_framework/univmon.rs | 4 +- src/sketch_framework/univmon/wire.rs | 24 +++--- src/sketch_framework/univmon_q.rs | 12 +-- src/sketch_framework/univmon_q/wire.rs | 30 +++---- src/sketch_framework/univmon_two_tier.rs | 8 +- src/sketch_framework/univmon_two_tier/wire.rs | 22 ++--- src/sketches/bloom.rs | 2 +- src/sketches/coco.rs | 4 +- src/sketches/countminsketch.rs | 10 +-- src/sketches/countminsketch/wire.rs | 4 +- src/sketches/countminsketch_heap.rs | 2 +- src/sketches/countminsketch_heap/delta.rs | 32 +++++--- src/sketches/countminsketch_heap/wire.rs | 35 +++++--- src/sketches/countminsketch_hll.rs | 10 ++- src/sketches/countminsketch_space_saving.rs | 2 +- .../summary_wire.rs | 21 +++-- .../countminsketch_space_saving/wire.rs | 9 ++- src/sketches/countsketch.rs | 6 +- src/sketches/countsketch/wire.rs | 4 +- src/sketches/countsketch_heap.rs | 12 +-- src/sketches/countsketch_heap/delta.rs | 27 ++++--- src/sketches/countsketch_heap/l2hh_wire.rs | 32 ++++---- src/sketches/countsketch_heap/wire.rs | 29 ++++--- src/sketches/countsketch_space_saving.rs | 2 +- src/sketches/countsketch_space_saving/wire.rs | 20 +++-- src/sketches/elastic.rs | 2 +- src/sketches/elastic/wire.rs | 4 +- src/sketches/hll/wire.rs | 4 +- src/sketches/kll.rs | 4 +- src/sketches/kll_dynamic.rs | 10 +-- src/sketches/kll_dynamic/wire.rs | 4 +- src/sketches/kmv.rs | 6 +- src/sketches/microscope.rs | 12 +-- tests/asapv1_golden.rs | 6 +- tests/conformance_kit.rs | 80 ++++++++++++------ tests/e2e/cardinality.rs | 6 +- tests/e2e/composition.rs | 18 ++--- tests/e2e/data_input.rs | 12 +-- tests/e2e/experimental.rs | 14 ++-- tests/e2e/frameworks.rs | 18 ++--- tests/e2e/frequency.rs | 65 ++++++++------- tests/e2e/heavy_hitters.rs | 18 ++--- tests/e2e/matrix_instances.rs | 66 ++++++++------- tests/e2e/nitro.rs | 45 ++++++----- tests/e2e/numeric_types.rs | 18 ++--- tests/e2e/octo.rs | 14 ++-- tests/e2e/quantiles.rs | 18 ++--- tests/e2e/topk.rs | 30 +++++-- tests/e2e/windows.rs | 81 ++++++++++--------- tests/e2e/wire.rs | 44 +++++----- tests/hll_custom_precision.rs | 4 +- tests/pbt/coco.rs | 4 +- tests/pbt/ensemble.rs | 6 +- tests/pbt/microscope.rs | 2 +- tests/pbt/octo_delta.rs | 20 ++--- tests/sketches_go_parity_probe.rs | 8 +- 79 files changed, 697 insertions(+), 553 deletions(-) diff --git a/examples/accuracy_probe.rs b/examples/accuracy_probe.rs index f6941618..78a99150 100644 --- a/examples/accuracy_probe.rs +++ b/examples/accuracy_probe.rs @@ -128,7 +128,7 @@ fn main() { // ---------------------------------------------------------------- CountMinSketch fn probe_countmin() { let mut p = Probe::new(); - println!("--- CountMin (core, Vector2D) ---"); + println!("--- CountMinSketch (core, Vector2D) ---"); // Sparse dims: no collisions => exact counts, incl. turnstile negatives. let mut cm = @@ -184,13 +184,13 @@ fn probe_countmin() { ), worst_excess <= eps * n as f64, ); - p.finish("CountMin"); + p.finish("CountMinSketch"); } // ------------------------------------------------------------- CountSketch fn probe_countsketch() { let mut p = Probe::new(); - println!("--- CountSketch (core Count, Vector2D) ---"); + println!("--- CountSketch (core, Vector2D) ---"); // True turnstile: net-zero stream must estimate ~0. let mut cs = @@ -248,7 +248,7 @@ fn probe_countsketch() { // -------------------------------------------------------------- Top-K heaps fn probe_topk_heaps() { let mut p = Probe::new(); - println!("--- CMSHeap / CSHeap top-k recall ---"); + println!("--- CountMinSketchWithHeap / CountSketchWithHeap top-k recall ---"); let (n, domain, top_k) = (20_000usize, 1024usize, 16usize); let mut z = Zipf::new(domain, 1.1, 44); let mut truth: HashMap = HashMap::new(); @@ -277,8 +277,8 @@ fn probe_topk_heaps() { cs_heap.update(&d); } for (label, heap_items) in [ - ("CMSHeap", cms_heap.heap().heap().to_vec()), - ("CSHeap", cs_heap.heap().heap().to_vec()), + ("CountMinSketchWithHeap", cms_heap.heap().heap().to_vec()), + ("CountSketchWithHeap", cs_heap.heap().heap().to_vec()), ] { let found = heap_items .iter() @@ -307,7 +307,7 @@ fn probe_topk_heaps() { .find(|it| matches!(it.key, asap_sketchlib::OwnedDataInput::U32(999))) { let cnt = it.count; - p.note(&format!("CSHeap stores negative heap count {cnt} for pure-decrement key (design wart, not a crash)")); + p.note(&format!("CountSketchWithHeap stores negative heap count {cnt} for pure-decrement key (design wart, not a crash)")); } p.finish("TopK heaps"); } @@ -315,7 +315,7 @@ fn probe_topk_heaps() { // --------------------------------------------------------------- CountSketchWithL2Norm fn probe_countl2hh() { let mut p = Probe::new(); - println!("--- CountL2HH (F2 / L2 estimation) ---"); + println!("--- CountSketchWithL2Norm (F2 / L2 estimation) ---"); // Moderate stream F2 accuracy (truth = sum over KEYS of count^2). let mut sk = @@ -375,13 +375,13 @@ fn probe_countl2hh() { }, saturated, ); - p.finish("CountL2HH"); + p.finish("CountSketchWithL2Norm"); } // --------------------------------------------------------------------- HLL fn probe_hll() { let mut p = Probe::new(); - println!("--- HLL cardinality (Classic / ErtlMLE / HIP, p14) ---"); + println!("--- HLL cardinality (Classic / Ertl / HIP, p14) ---"); let checkpoints: [u64; 4] = [10_000, 100_000, 1_000_000, 10_000_000]; let mut classic = HyperLogLog::::new(); let mut ertl = HyperLogLog::::new(); @@ -405,7 +405,7 @@ fn probe_hll() { rel_err(ec, t) <= 0.02, ); p.check( - &format!("ErtlMLE @ {target} within 2%"), + &format!("Ertl @ {target} within 2%"), format!("rel {:.4}", rel_err(ee, t)), rel_err(ee, t) <= 0.02, ); @@ -437,7 +437,7 @@ fn probe_hll() { // --------------------------------------------------------------------- KLL fn probe_kll() { let mut p = Probe::new(); - println!("--- KLL / KLLDynamic quantiles (k=200) ---"); + println!("--- KLL / KllDynamic quantiles (k=200) ---"); let qs = [0.1f64, 0.25, 0.5, 0.75, 0.9]; let n = 100_000usize; @@ -445,7 +445,7 @@ fn probe_kll() { let mut data: Vec = (0..n).map(|_| rng.random::() * 1_000_000.0).collect(); data.sort_by(|a, b| a.partial_cmp(b).unwrap()); - for label in ["KLL", "KLLDynamic"] { + for label in ["KLL", "KllDynamic"] { let (e1, w1) = if label == "KLL" { let mut sk = asap_sketchlib::Kll::with_k(200); for v in &data { @@ -636,9 +636,9 @@ fn probe_hydra() { let x = 500.0; let got = hydra.query_cdf(&[Some("eu")], x).expect("cdf"); let truth_cdf = sorted.partition_point(|v| *v <= x) as f64 / n as f64; - p.note("Hydra::query_quantile(key, threshold) returns CDF(threshold), NOT a quantile value (name inversion)"); + p.note("Hydra::query_cdf(key, threshold) returns CDF(threshold), NOT a quantile value (name inversion)"); p.check( - "query_quantile(x)=CDF fraction <= x", + "query_cdf(x)=CDF fraction <= x", format!("x={x}, expected ~{truth_cdf:.3}, got {got:.3}"), (got - truth_cdf).abs() < 0.03, ); @@ -725,7 +725,7 @@ fn probe_univmon() { ); // Entropy unit sanity: bits value should be log2 larger than nats value. - p.note("UnivMon calc_entropy returns BITS (log2); UnivMonQ estimate_entropy returns NATS (ln). Cross-family comparisons need conversion."); + p.note("UnivMon estimate_entropy returns BITS (log2); UnivMonQ estimate_entropy returns NATS (ln). Cross-family comparisons need conversion."); p.finish("UnivMon"); } @@ -733,7 +733,7 @@ fn probe_univmon() { fn probe_univmon_pyramid() { use asap_sketchlib::UnivMonTwoTier; let mut p = Probe::new(); - println!("--- UnivMonPyramid (experimental) ---"); + println!("--- UnivMonTwoTier (experimental) ---"); let mut up = UnivMonTwoTier::with_defaults(); let mut truth_w: HashMap = HashMap::new(); let mut z = Zipf::new(1000, 1.2, 50); @@ -757,7 +757,7 @@ fn probe_univmon_pyramid() { format!("expected {card_truth}, got {card}"), rel_err(card, card_truth) < 0.10, ); - p.finish("UnivMonPyramid"); + p.finish("UnivMonTwoTier"); } // ---------------------------------------------------------------- UnivMonQ @@ -916,7 +916,7 @@ fn probe_nitro() { (est - n as f64).abs() <= 0.05 * n as f64, ); } - p.note("Pre-fix: insert hashed with raw hash128_seeded while estimation used the Packed64 matrix hash (estimates read empty cells), each sample updated ONE row (truth/rows bias), and the skip draw used ceil instead of floor (effective rate 1/(1/p+2))."); + p.note("Pre-fix: update hashed with raw hash128_seeded while estimation used the Packed64 matrix hash (estimates read empty cells), each sample updated ONE row (truth/rows bias), and the skip draw used ceil instead of floor (effective rate 1/(1/p+2))."); p.finish("Nitro"); } @@ -925,7 +925,7 @@ fn probe_nitro() { fn probe_eh_univ() { use asap_sketchlib::ExponentialHistogramUnivMon; let mut p = Probe::new(); - println!("--- EHUnivOptimized exact map tier (experimental) ---"); + println!("--- ExponentialHistogramUnivMon exact map tier (experimental) ---"); let window = 100u64; let mut eh = ExponentialHistogramUnivMon::with_defaults(2, window); // Stream updates over times 0..150; only [50,149] survive at t=149+window semantics. @@ -965,7 +965,7 @@ fn probe_eh_univ() { false, ), } - p.finish("EHUnivOptimized"); + p.finish("ExponentialHistogramUnivMon"); } // ---------------------------------------------------------------- Tumbling @@ -973,7 +973,7 @@ fn probe_tumbling() { use asap_sketchlib::{CountMinSketchFoldConfig, TumblingWindow}; let mut p = Probe::new(); - println!("--- TumblingWindow FoldCMS + KLL ---"); + println!("--- TumblingWindow CountMinSketchFold + KLL ---"); // CountMinSketchFold: weighted counts per key. let cfg = CountMinSketchFoldConfig { @@ -992,12 +992,12 @@ fn probe_tumbling() { let fa = merged.estimate(&DataInput::Str("A")); let fb = merged.estimate(&DataInput::Str("B")); p.check( - "FoldCMS key A == 50", + "CountMinSketchFold key A == 50", format!("expected 50, got {fa}"), fa == 50, ); p.check( - "FoldCMS key B == 25", + "CountMinSketchFold key B == 25", format!("expected 25, got {fb}"), fb == 25, ); @@ -1022,7 +1022,7 @@ fn probe_tumbling() { let merged_k = tkw.query_all(); let med = merged_k.quantile(0.5); let truth_med = all_vals[all_vals.len() / 2]; - p.note("TumblingWindow::insert ignores its numeric `value` param; observations are the keys themselves"); + p.note("TumblingWindow::update_weighted ignores its numeric `weight` param; observations are the keys themselves"); p.check( "tumbling KLL median approx", format!("expected ~{truth_med:.1}, got {med:.1}"), diff --git a/examples/compare_univmon_q.rs b/examples/compare_univmon_q.rs index 393aefdf..61cbf85e 100644 --- a/examples/compare_univmon_q.rs +++ b/examples/compare_univmon_q.rs @@ -395,7 +395,7 @@ fn main() { let q_memory_prepared_cdf_query = average_query(|| q_memory_prepared.cdf(), QUERY_REPEATS); println!( - "workload: n={N}, domain={DOMAIN}, levels={LEVELS}, width={WIDTH}, depth={DEPTH}, candidates={CANDIDATES}" + "workload: n={N}, domain={DOMAIN}, layers={LEVELS}, top_layer_cols={WIDTH}, rows={DEPTH}, heap_size={CANDIDATES}" ); println!( "exact: L1={exact_l1:.0}, freq(0)={}, F0={exact_f0:.0}, F2={exact_f2:.0}, entropy={exact_entropy_bits:.6} bits", @@ -407,12 +407,12 @@ fn main() { println!(); println!("update (median of {REPEATS} release runs):"); println!( - " UnivMon insert {:>9.3} M/s {:?}", + " UnivMon update_weighted {:>9.3} M/s {:?}", rate(regular_time), regular_time ); println!( - " UnivMon fast_insert {:>9.3} M/s {:?}", + " UnivMon update_weighted_terminal {:>9.3} M/s {:?}", rate(fast_time), fast_time ); @@ -434,11 +434,11 @@ fn main() { println!(); println!("state size:"); println!( - " UnivMon insert wire={regular_wire} bytes, counters>={} bytes", + " UnivMon update_weighted wire={regular_wire} bytes, counters>={} bytes", LEVELS * DEPTH * WIDTH * 8 ); println!( - " UnivMon fast_insert wire={fast_wire} bytes, counters>={} bytes", + " UnivMon update_weighted_terminal wire={fast_wire} bytes, counters>={} bytes", LEVELS * DEPTH * WIDTH * 8 ); println!( @@ -574,7 +574,7 @@ fn main() { println!(); println!("accuracy (relative error):"); println!( - " UnivMon insert L1={:.4}, freq={:.4}, F0={:.4}, F2={:.4}, H={:.4}, HH-recall={:.2}", + " UnivMon update_weighted L1={:.4}, freq={:.4}, F0={:.4}, F2={:.4}, H={:.4}, HH-recall={:.2}", relative_error(regular_l1, exact_l1), relative_error(regular_frequency, exact[&0] as f64), relative_error(regular_card, exact_f0), @@ -583,7 +583,7 @@ fn main() { heavy_hitter_recall(®ular_hh, &exact_top), ); println!( - " UnivMon fast_insert L1={:.4}, freq={:.4}, F0={:.4}, F2={:.4}, H={:.4}, HH-recall={:.2}", + " UnivMon update_weighted_terminal L1={:.4}, freq={:.4}, F0={:.4}, F2={:.4}, H={:.4}, HH-recall={:.2}", relative_error(fast_l1, exact_l1), relative_error(fast_frequency, exact[&0] as f64), relative_error(fast_card, exact_f0), diff --git a/examples/compare_univmon_q_bundle.rs b/examples/compare_univmon_q_bundle.rs index 0e45bbf4..ab10b8f0 100644 --- a/examples/compare_univmon_q_bundle.rs +++ b/examples/compare_univmon_q_bundle.rs @@ -55,7 +55,7 @@ fn main() { "bundle: HLL(p=14) + {DEPTH}x{BUNDLE_WIDTH} CountSketch(i32) + SpaceSaving({BUNDLE_CANDIDATES}) + occurrence bottom-k({BUNDLE_RESERVOIR}) + KLL(k={BUNDLE_KLL_K}) + exact count/extrema" ); println!( - "Q-matched: width={}, candidates={}; Q-quality: width={}, candidates={}; both use {} ordered samples", + "Q-matched: top_layer_cols={}, heap_size={}; Q-quality: top_layer_cols={}, heap_size={}; both use {} ordered samples", q_matched.top_layer_cols, q_matched.heap_size, q_quality.top_layer_cols, diff --git a/examples/evaluate_univmon_q_entropy.rs b/examples/evaluate_univmon_q_entropy.rs index 79142d36..0f659174 100644 --- a/examples/evaluate_univmon_q_entropy.rs +++ b/examples/evaluate_univmon_q_entropy.rs @@ -29,7 +29,7 @@ fn main() { for profile in &profiles { let empty = UnivMonQ::new(profile.config).unwrap(); println!( - "profile={}: {:.3} MiB, width={}, halve/{}, candidates={}, samples={}", + "profile={}: {:.3} MiB, top_layer_cols={}, halve/{}, heap_size={}, samples={}", profile.name, empty.estimated_memory_bytes() as f64 / (1024.0 * 1024.0), profile.config.top_layer_cols, diff --git a/examples/evaluate_univmon_q_skew.rs b/examples/evaluate_univmon_q_skew.rs index 0dea33d5..af277dd4 100644 --- a/examples/evaluate_univmon_q_skew.rs +++ b/examples/evaluate_univmon_q_skew.rs @@ -62,7 +62,7 @@ fn main() { seed_index: 5, } .with_window_bound(n as u64, 1e-9) - .expect("the evaluation window must fit within 63 levels"); + .expect("the evaluation window must fit within 63 layers"); let um_candidates = candidates; let um_counter_bytes = base_config.layers * base_config.rows * base_config.top_layer_cols * size_of::(); @@ -78,7 +78,7 @@ fn main() { println!("n={n}, trials={trials}, domain={domain}, shards={SHARDS}"); println!("profile={profile}"); println!( - "config: levels={}, width={}, halve/{} levels, depth={}, candidates={}, ordered_samples={}", + "config: layers={}, top_layer_cols={}, halve/{} layers, rows={}, heap_size={}, ordered_samples={}", base_config.layers, base_config.top_layer_cols, base_config.cols_halving_period, @@ -227,7 +227,9 @@ fn main() { .map(|summary| summary.warm_p50_ns / 1_000.0) .fold(0.0, f64::max), ); - println!(" UnivMon uses terminal-only `fast_insert`; cumulative-layer `insert` is excluded."); + println!( + " UnivMon uses terminal-only `update_weighted_terminal`; cumulative-layer `update_weighted` is excluded." + ); let q_violations: usize = summaries.iter().map(|summary| summary.q_violations).sum(); let um_violations: usize = summaries.iter().map(|summary| summary.um_violations).sum(); diff --git a/examples/hhheap_probe.rs b/examples/hhheap_probe.rs index b74b5f33..ca311b1b 100644 --- a/examples/hhheap_probe.rs +++ b/examples/hhheap_probe.rs @@ -335,9 +335,12 @@ fn sketch_ceiling() { } let plain_rate = stream.len() as f64 / started.elapsed().as_secs_f64() / 1e6; - println!("\nCMSHeap::insert with the current HHHeap (rows=4, cols=4096)"); - println!(" bare CountMin::insert: {plain_rate:.3} Minsert/s"); - println!("{:>6} {:>16} {:>12}", "top_k", "Minsert/s", "heap share"); + println!("\nCountMinSketchWithHeap::update with the current TopKHeap (rows=4, cols=4096)"); + println!(" bare CountMinSketch::update: {plain_rate:.3} Minsert/s"); + println!( + "{:>6} {:>16} {:>12}", + "heap_size", "Minsert/s", "heap share" + ); for cap in CAPACITIES { let mut sketch = CountMinSketchWithHeap::, FastPath>::new(4, 4096, cap); let started = Instant::now(); @@ -364,7 +367,7 @@ fn main() { ); println!( "{:>6} {:>14} {:>14} {:>14} {:>8}", - "cap", "HHHeap Mups/s", "indexed Mups/s", "scan Mups/s", "speedup" + "cap", "TopKHeap Mups/s", "indexed Mups/s", "scan Mups/s", "speedup" ); for cap in CAPACITIES { diff --git a/src/common/hash.rs b/src/common/hash.rs index 7a6dd4a1..6b59723c 100644 --- a/src/common/hash.rs +++ b/src/common/hash.rs @@ -654,7 +654,7 @@ mod tests { #[test] #[should_panic( - expected = "SketchHasher hash type u64 cannot represent fast-path hash for rows=8, cols=4096; use u128 or MatrixHashType" + expected = "SketchHasher hash type u64 cannot represent fast-path hash for rows=8, cols=4096; use u128 or MatrixHash" )] fn packed64_hasher_rejects_oversized_dimensions() { let key = DataInput::U64(19); diff --git a/src/common/structure_utils.rs b/src/common/structure_utils.rs index bfdf8d0a..381a8a7a 100644 --- a/src/common/structure_utils.rs +++ b/src/common/structure_utils.rs @@ -501,7 +501,7 @@ mod nitro_tests { assert_eq!( nitro.cursor(), 1, - "init_nitro must draw the first skip from entry 0 and leave the cursor at 1" + "new must draw the first skip from entry 0 and leave the cursor at 1" ); let mut seen = std::collections::HashSet::new(); diff --git a/src/common/structures/fixed_structure.rs b/src/common/structures/fixed_structure.rs index b9e54e94..2ba3fa3c 100644 --- a/src/common/structures/fixed_structure.rs +++ b/src/common/structures/fixed_structure.rs @@ -83,11 +83,11 @@ macro_rules! impl_hll_registers { const _: () = { ::std::assert!( $num_registers == 1_usize << $precision, - "impl_hll_bucket_list!: num_registers must equal 1 << precision", + "impl_hll_registers!: num_registers must equal 1 << precision", ); ::std::assert!( $precision >= 1, - "impl_hll_bucket_list!: precision must be at least 1", + "impl_hll_registers!: precision must be at least 1", ); }; diff --git a/src/common/structures/matrix_storage.rs b/src/common/structures/matrix_storage.rs index cb363f15..b6c15df8 100644 --- a/src/common/structures/matrix_storage.rs +++ b/src/common/structures/matrix_storage.rs @@ -135,7 +135,7 @@ impl MatrixFastHash for u64 { let bits_required = bits_per_row.saturating_mul(rows); assert!( bits_required <= 64, - "SketchHasher hash type u64 cannot represent fast-path hash for rows={rows}, cols={cols}; use u128 or MatrixHashType" + "SketchHasher hash type u64 cannot represent fast-path hash for rows={rows}, cols={cols}; use u128 or MatrixHash" ); } @@ -166,7 +166,7 @@ impl MatrixFastHash for u128 { let bits_required = bits_per_row.saturating_mul(rows); assert!( bits_required <= 128, - "SketchHasher hash type u128 cannot represent fast-path hash for rows={rows}, cols={cols}; use MatrixHashType" + "SketchHasher hash type u128 cannot represent fast-path hash for rows={rows}, cols={cols}; use MatrixHash" ); } diff --git a/src/common/structures/vector3d.rs b/src/common/structures/vector3d.rs index 302f7ea7..f8dae3d0 100644 --- a/src/common/structures/vector3d.rs +++ b/src/common/structures/vector3d.rs @@ -100,17 +100,19 @@ where let input = Vector3DDeserialize::deserialize(deserializer)?; if input.rows == 0 || input.cols == 0 || input.bucket_len == 0 { return Err(serde::de::Error::custom( - "Vector3D: rows, cols and depth must all be non-zero", + "Vector3D: rows, cols and bucket_len must all be non-zero", )); } let expected = input .rows .checked_mul(input.cols) .and_then(|n| n.checked_mul(input.bucket_len)) - .ok_or_else(|| serde::de::Error::custom("Vector3D: rows * cols * depth overflows"))?; + .ok_or_else(|| { + serde::de::Error::custom("Vector3D: rows * cols * bucket_len overflows") + })?; if input.data.len() != expected { return Err(serde::de::Error::custom(format!( - "Vector3D: data length {} does not match rows * cols * depth = {expected}", + "Vector3D: data length {} does not match rows * cols * bucket_len = {expected}", input.data.len() ))); } @@ -138,15 +140,15 @@ impl Vector3D { pub fn unfilled(rows: usize, cols: usize, bucket_len: usize) -> Self { assert!( rows > 0 && cols > 0 && bucket_len > 0, - "Vector3D dimensions must be non-zero, got rows={rows}, cols={cols}, depth={bucket_len}" + "Vector3D dimensions must be non-zero, got rows={rows}, cols={cols}, bucket_len={bucket_len}" ); let len = rows .checked_mul(cols) .and_then(|n| n.checked_mul(bucket_len)) .unwrap_or_else(|| { panic!( - "Vector3D rows * cols * depth overflows usize, got \ - rows={rows}, cols={cols}, depth={bucket_len}" + "Vector3D rows * cols * bucket_len overflows usize, got \ + rows={rows}, cols={cols}, bucket_len={bucket_len}" ) }); let mask_bits = cols_mask_bits(cols); @@ -467,7 +469,7 @@ mod tests { .expect_err("a short payload must be rejected"); assert!( err.to_string() - .contains("does not match rows * cols * depth"), + .contains("does not match rows * cols * bucket_len"), "unexpected error: {err}" ); } diff --git a/src/message_pack_format/envelope.rs b/src/message_pack_format/envelope.rs index 9b697866..a92970f9 100644 --- a/src/message_pack_format/envelope.rs +++ b/src/message_pack_format/envelope.rs @@ -314,13 +314,13 @@ mod tests { case!("CS delta", cs_delta, sketches::CountSketchDelta), case!("KLL", kll, Kll), case!("SetAggregator", set, SetAggregator), - case!("DeltaResult", delta, SetAggregatorDelta), + case!("SetAggregatorDelta", delta, SetAggregatorDelta), sketch!("KLL dynamic", kll_dynamic, KllDynamic), - sketch!("CMSHeap", cms_heap, CmsHeap), - sketch!("CMSHeap bytes", cms_heap_bytes, CmsHeap), - case!("CMSHeap delta", cms_heap_delta, CountMinSketchWithHeapDelta), - sketch!("CSHeap", cs_heap, CsHeap), - sketch!("CountL2HH", l2hh, CountSketchWithL2Norm), + sketch!("CountMinSketchWithHeap", cms_heap, CmsHeap), + sketch!("CountMinSketchWithHeap bytes", cms_heap_bytes, CmsHeap), + case!("CountMinSketchWithHeap delta", cms_heap_delta, CountMinSketchWithHeapDelta), + sketch!("CountSketchWithHeap", cs_heap, CsHeap), + sketch!("CountSketchWithL2Norm", l2hh, CountSketchWithL2Norm), sketch!("DDSketch", positive, DdSketch), case!("DDSketch delta", dd_delta, DdSketchDelta), sketch!("Bloom", bloom, Bloom), @@ -331,10 +331,10 @@ mod tests { sketch!("Elastic", elastic, ElasticSketch), sketch!("UnivMon", univmon, UnivMon), sketch!("UnivMon bytes", univmon_bytes, UnivMon), - sketch!("UnivMonPyramid", pyramid, UnivMonTwoTier), + sketch!("UnivMonTwoTier", pyramid, UnivMonTwoTier), sketch!("UnivMonQ", univmon_q, UnivMonQ), sketch!("ExponentialHistogram", eh, ExponentialHistogram), - sketch!("EHSketchList", eh_list, EhBucketSketch), + sketch!("EhBucketSketch", eh_list, EhBucketSketch), sketch!("Hydra CM", hydra_cm, Hydra), sketch!("Hydra HLL", hydra_hll, Hydra), sketch!("Hydra KLL", hydra_kll, Hydra), diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs index a86e8e2b..3b99f664 100644 --- a/src/message_pack_format/portable/countminsketch.rs +++ b/src/message_pack_format/portable/countminsketch.rs @@ -889,7 +889,7 @@ mod tests { assert_eq!( got, want, - "CountMin delta hh_keys bytes diverge from the Go reference golden \ + "CountMinSketch delta hh_keys bytes diverge from the Go reference golden \ ({} bytes got vs {} bytes want)", got.len(), want.len(), @@ -991,7 +991,7 @@ mod tests { assert_eq!( got, want, - "CountMin delta bytes diverge from the Go reference golden \ + "CountMinSketch delta bytes diverge from the Go reference golden \ ({} bytes got vs {} bytes want)", got.len(), want.len(), diff --git a/src/message_pack_format/portable/set_aggregator/wire.rs b/src/message_pack_format/portable/set_aggregator/wire.rs index d9db7983..152ac16b 100644 --- a/src/message_pack_format/portable/set_aggregator/wire.rs +++ b/src/message_pack_format/portable/set_aggregator/wire.rs @@ -67,7 +67,7 @@ fn read_ascending( fn check_disjoint(added: &HashSet, removed: &HashSet) -> Result<(), String> { match added.intersection(removed).next() { Some(member) => Err(format!( - "ASAPv1 DeltaResult: member {member:?} is both added and removed" + "ASAPv1 SetAggregatorDelta: member {member:?} is both added and removed" )), None => Ok(()), } @@ -133,10 +133,10 @@ impl SetAggregatorDelta { /// Deserializes a delta from an ASAPv1 MessagePack envelope. pub fn from_asapv1(bytes: &[u8]) -> Result { - let payload = open(bytes, kind::SET_AGGREGATOR_DELTA, "DeltaResult")?; + let payload = open(bytes, kind::SET_AGGREGATOR_DELTA, "SetAggregatorDelta")?; let p: DeltaPayload = envelope::decode_block(payload)?; - let added = read_ascending("DeltaResult added", p.added)?; - let removed = read_ascending("DeltaResult removed", p.removed)?; + let added = read_ascending("SetAggregatorDelta added", p.added)?; + let removed = read_ascending("SetAggregatorDelta removed", p.removed)?; check_disjoint(&added, &removed).map_err(RmpDecodeError::Uncategorized)?; Ok(SetAggregatorDelta { added, removed }) } diff --git a/src/message_pack_format/wire_key.rs b/src/message_pack_format/wire_key.rs index bbd1e170..ddd49374 100644 --- a/src/message_pack_format/wire_key.rs +++ b/src/message_pack_format/wire_key.rs @@ -400,13 +400,31 @@ mod tests { row("CS f64 count", cs, -2.25, F32, cs_ok), row("CS f64 delta", cs_delta, 2.25, Int, cs_delta_ok), row("KLL f64 item", kll, 7.0, Int, kll_ok), - row("CMSHeap f64 count", heap, 1.0, Int, heap_ok), - row("CMSHeap f32 key", heap, 1.5, F64, heap_ok), - row("CMSHeap f64 heap count", heap, 11.0, Int, heap_ok), - row("CMSHeap f64 delta", heap_delta, 2.5, Int, heap_delta_ok), - row("CMSHeap delta f32 key", heap_delta, 1.5, F64, heap_delta_ok), + row("CountMinSketchWithHeap f64 count", heap, 1.0, Int, heap_ok), + row("CountMinSketchWithHeap f32 key", heap, 1.5, F64, heap_ok), row( - "CMSHeap delta f64 heap count", + "CountMinSketchWithHeap f64 heap count", + heap, + 11.0, + Int, + heap_ok, + ), + row( + "CountMinSketchWithHeap f64 delta", + heap_delta, + 2.5, + Int, + heap_delta_ok, + ), + row( + "CountMinSketchWithHeap delta f32 key", + heap_delta, + 1.5, + F64, + heap_delta_ok, + ), + row( + "CountMinSketchWithHeap delta f64 heap count", heap_delta, 13.0, Int, diff --git a/src/sketch_framework/eh/wire.rs b/src/sketch_framework/eh/wire.rs index dd9f5283..bf76e49c 100644 --- a/src/sketch_framework/eh/wire.rs +++ b/src/sketch_framework/eh/wire.rs @@ -398,7 +398,7 @@ mod tests { let list = EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(3, 8), ); - let list_bytes = list.to_asapv1().expect("serialize EHSketchList"); + let list_bytes = list.to_asapv1().expect("serialize EhBucketSketch"); assert!(ExponentialHistogram::from_asapv1(&list_bytes).is_err()); let cms = CountMinSketch::, FastPath>::with_dimensions(3, 8); diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index 42603e87..31755766 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -272,9 +272,9 @@ impl EhBucketSketch { /// Get the type of sketch as a string pub fn sketch_type(&self) -> &'static str { match self { - EhBucketSketch::CountMinSketch(_) => "CountMin", + EhBucketSketch::CountMinSketch(_) => "CountMinSketch", EhBucketSketch::CocoSketch(_) => "Coco", - EhBucketSketch::CountSketchWithL2Norm(_) => "CountL2HH", + EhBucketSketch::CountSketchWithL2Norm(_) => "CountSketchWithL2Norm", EhBucketSketch::CountSketch(_) => "CountSketch", EhBucketSketch::DdSketch(_) => "DDSketch", EhBucketSketch::ElasticSketch(_) => "Elastic", @@ -300,10 +300,10 @@ mod tests { for _ in 0..9 { count_l2hh.update(&key); } - let l2hh_est = count_l2hh.query(&key).expect("query COUNTL2HH"); + let l2hh_est = count_l2hh.query(&key).expect("query CountSketchWithL2Norm"); assert!( l2hh_est >= 9.0, - "expected COUNTL2HH estimate >= 9, got {l2hh_est}" + "expected CountSketchWithL2Norm estimate >= 9, got {l2hh_est}" ); let mut um = EhBucketSketch::UnivMon(UnivMon::default()); diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index 3944ec22..22a2ae5b 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -41,9 +41,9 @@ use super::EhBucketSketch; /// decoding dispatch on the bytes. pub(crate) fn variant_name(kind_id: &[u8]) -> Option<&'static str> { match kind_id { - kind::COUNT_MIN => Some("CountMin"), + kind::COUNT_MIN => Some("CountMinSketch"), kind::COCO => Some("Coco"), - kind::COUNT_SKETCH_WITH_L2_NORM => Some("CountL2HH"), + kind::COUNT_SKETCH_WITH_L2_NORM => Some("CountSketchWithL2Norm"), kind::COUNT_SKETCH => Some("CountSketch"), kind::DDSKETCH => Some("DDSketch"), kind::ELASTIC => Some("Elastic"), @@ -87,7 +87,7 @@ pub(crate) struct SketchState { /// Rejects a kind_id outside the ten. pub(crate) fn unknown_variant(kind_id: &[u8]) -> RmpDecodeError { RmpDecodeError::Uncategorized(format!( - "ASAPv1 EHSketchList: kind_id {kind_id:02x?} is not a wire variant" + "ASAPv1 EhBucketSketch: kind_id {kind_id:02x?} is not a wire variant" )) } @@ -96,7 +96,7 @@ pub(crate) fn unknown_variant(kind_id: &[u8]) -> RmpDecodeError { fn check_variant_available(kind_id: &[u8]) -> Result<(), RmpDecodeError> { match kind_id { kind::UNIFORM_SAMPLING => Err(RmpDecodeError::Uncategorized(format!( - "ASAPv1 EHSketchList: variant {} (kind_id {kind_id:02x?}) needs the experimental feature", + "ASAPv1 EhBucketSketch: variant {} (kind_id {kind_id:02x?}) needs the experimental feature", variant_name(kind_id).unwrap_or("unnamed") ))), _ => Ok(()), @@ -114,7 +114,7 @@ fn triple(kind: &'static [u8], bytes: &[u8]) -> Result Result<(), String> { EhBucketSketch::Hll(s) => s.admission_rate(), _ => return Ok(()), }; - admission_rate::check_admits_all(rate).map_err(|e| format!("ASAPv1 EHSketchList: {e}")) + admission_rate::check_admits_all(rate).map_err(|e| format!("ASAPv1 EhBucketSketch: {e}")) } /// The triple one EhBucketSketch contributes, in the order the wire emits it. @@ -235,13 +235,13 @@ impl EhBucketSketch { envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::EH_BUCKET_SKETCH { return Err(RmpDecodeError::Uncategorized(format!( - "EHSketchList kind_id mismatch: stored {kind_id:?}, expected {EH_BUCKET_SKETCH:?}" + "EhBucketSketch kind_id mismatch: stored {kind_id:?}, expected {EH_BUCKET_SKETCH:?}" ))); } let meta: EhSketchListMetadata = envelope::decode_block(metadata)?; if meta != eh_sketch_list_metadata() { return Err(RmpDecodeError::Uncategorized( - "ASAPv1 EHSketchList envelope: metadata mismatch".to_string(), + "ASAPv1 EhBucketSketch envelope: metadata mismatch".to_string(), )); } let triple: SketchState = envelope::decode_block(payload)?; @@ -379,12 +379,12 @@ pub(crate) mod tests { ); sketch.update(&DataInput::U64(42)); - let encoded = sketch.to_asapv1().expect("serialize EHSketchList"); + let encoded = sketch.to_asapv1().expect("serialize EhBucketSketch"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x14, 0x00]); // kind_id_len=2, kind_id=[0x14,0x00] let decoded = EhBucketSketch::from_asapv1(&encoded).expect("deserialize"); - assert_eq!(decoded.sketch_type(), "CountMin"); + assert_eq!(decoded.sketch_type(), "CountMinSketch"); assert_eq!( decoded.query(&DataInput::U64(42)), sketch.query(&DataInput::U64(42)) @@ -417,9 +417,9 @@ pub(crate) mod tests { #[test] fn eh_sketch_list_kind_ids_are_build_independent() { let table: [(&[u8], &str); 10] = [ - (kind::COUNT_MIN, "CountMin"), + (kind::COUNT_MIN, "CountMinSketch"), (kind::COCO, "Coco"), - (kind::COUNT_SKETCH_WITH_L2_NORM, "CountL2HH"), + (kind::COUNT_SKETCH_WITH_L2_NORM, "CountSketchWithL2Norm"), (kind::COUNT_SKETCH, "CountSketch"), (kind::DDSKETCH, "DDSketch"), (kind::ELASTIC, "Elastic"), diff --git a/src/sketch_framework/eh_univmon.rs b/src/sketch_framework/eh_univmon.rs index fd1c590b..d690d551 100644 --- a/src/sketch_framework/eh_univmon.rs +++ b/src/sketch_framework/eh_univmon.rs @@ -472,9 +472,9 @@ impl ExponentialHistogramUnivMon { } pub fn print_buckets(&self) { - println!("=== EHUnivOptimized Buckets ==="); + println!("=== ExponentialHistogramUnivMon Buckets ==="); println!( - "k: {}, window: {}, max_map_size: {}", + "k: {}, window: {}, exact_bucket_budget: {}", self.k, self.window, self.exact_bucket_budget ); println!( @@ -485,14 +485,14 @@ impl ExponentialHistogramUnivMon { println!("Sketch buckets ({}):", self.sketch_buckets.len()); for (i, b) in self.sketch_buckets.iter().enumerate() { println!( - " [S{}] min_time={}, max_time={}, bucket_size={}, l22={:.2}", + " [S{}] min_time={}, max_time={}, total_weight={}, l2_squared={:.2}", i, b.min_time, b.max_time, b.total_weight, b.l2_squared ); } - println!("Map buckets ({}):", self.exact_buckets.len()); + println!("Exact buckets ({}):", self.exact_buckets.len()); for (i, b) in self.exact_buckets.iter().enumerate() { println!( - " [M{}] min_time={}, max_time={}, bucket_size={}, l22={:.2}, keys={}", + " [E{}] min_time={}, max_time={}, total_weight={}, l2_squared={:.2}, keys={}", i, b.min_time, b.max_time, @@ -941,7 +941,7 @@ mod tests { let rel_diff = (b.l2_squared - actual_l22).abs() / actual_l22.max(1e-12); assert!( rel_diff < 0.01, - "Sketch bucket {i}: stored l22={:.4} vs actual={actual_l22:.4}", + "Sketch bucket {i}: stored l2_squared={:.4} vs actual={actual_l22:.4}", b.l2_squared ); } diff --git a/src/sketch_framework/hashlayer.rs b/src/sketch_framework/hashlayer.rs index 4ee98858..fc39a079 100644 --- a/src/sketch_framework/hashlayer.rs +++ b/src/sketch_framework/hashlayer.rs @@ -80,8 +80,8 @@ impl EnsembleSketch { /// Returns a short sketch family label. pub fn sketch_type(&self) -> &'static str { match self { - EnsembleSketch::CountMinFast(_) => "CountMin", - EnsembleSketch::CountSketchFast(_) => "Count", + EnsembleSketch::CountMinFast(_) => "CountMinSketch", + EnsembleSketch::CountSketchFast(_) => "CountSketch", EnsembleSketch::HllErtl(_) | EnsembleSketch::HllClassic(_) | EnsembleSketch::HllHip(_) => "HLL", @@ -557,7 +557,7 @@ mod tests { assert!(layer.get(2).is_none()); let sketch = layer.get_mut(0).expect("mutable ref"); - assert_eq!(sketch.sketch_type(), "CountMin"); + assert_eq!(sketch.sketch_type(), "CountMinSketch"); } #[test] diff --git a/src/sketch_framework/hydra/wire.rs b/src/sketch_framework/hydra/wire.rs index 7683537d..24621ba9 100644 --- a/src/sketch_framework/hydra/wire.rs +++ b/src/sketch_framework/hydra/wire.rs @@ -611,7 +611,7 @@ impl Hydra { return Ok(()); } Err(encode_error(format!( - "ASAPv1 Hydra envelope: type_to_clone holds data; the {} prototype carries only its geometry", + "ASAPv1 Hydra envelope: prototype holds data; the {} prototype carries only its geometry", self.prototype ))) } @@ -1539,10 +1539,7 @@ mod tests { .to_asapv1() .expect_err("a prototype holding data must not serialize") .to_string(); - assert!( - problem.contains("type_to_clone holds data"), - "got {problem}" - ); + assert!(problem.contains("prototype holds data"), "got {problem}"); } /// A UnivMon grid whose cells hold different `OwnedDataInput` variants has no diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index 42f931ac..016d6130 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -1284,7 +1284,7 @@ where pub fn update(&mut self, input: DataInput<'_>) { let core = self.core.as_ref().expect("runtime core missing"); if core.closed.load(Ordering::Acquire) { - panic!("cannot insert after runtime has been closed"); + panic!("cannot update after runtime has been closed"); } let worker_id = core.worker_for(&input); @@ -4326,7 +4326,7 @@ mod runtime_tests { } #[test] - #[should_panic(expected = "cannot insert after runtime has been closed")] + #[should_panic(expected = "cannot update after runtime has been closed")] fn octo_runtime_insert_after_close_panics() { let mut runtime = OctoRuntime::new(&config(2), HllOctoPlan::new(), HllOctoAggregator::new); runtime.close(); diff --git a/src/sketch_framework/tumbling.rs b/src/sketch_framework/tumbling.rs index 3914514e..92b75989 100644 --- a/src/sketch_framework/tumbling.rs +++ b/src/sketch_framework/tumbling.rs @@ -1293,7 +1293,7 @@ mod tests { let bytes_b = tw_b.query_all().to_asapv1().unwrap(); assert_eq!( bytes_a, bytes_b, - "same KLLConfig seed must give byte-identical tumbling state" + "same KllConfig seed must give byte-identical tumbling state" ); // A different seed must not silently alias onto the same coin stream. @@ -1313,7 +1313,7 @@ mod tests { let bytes_c = tw_c.query_all().to_asapv1().unwrap(); assert_ne!( bytes_a, bytes_c, - "different KLLConfig seeds must produce different state" + "different KllConfig seeds must produce different state" ); } @@ -2133,7 +2133,7 @@ mod tests { let pct = within as f64 / truth.len() as f64 * 100.0; assert!( pct > 90.0, - "FoldCMS single-window: only {pct:.1}% within bound" + "CountMinSketchFold single-window: only {pct:.1}% within bound" ); } @@ -2176,7 +2176,7 @@ mod tests { let pct = within as f64 / truth.len() as f64 * 100.0; assert!( pct > 90.0, - "FoldCS single-window: only {pct:.1}% within bound" + "CountSketchFold single-window: only {pct:.1}% within bound" ); } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 8dfc93b3..5f0871ac 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -136,7 +136,7 @@ impl UnivMon { assert!(layers > 0, "layer count must be positive"); assert!( layers <= UNIVMON_MAX_LAYERS, - "layer count must be at most MAX_LAYER_SIZE {UNIVMON_MAX_LAYERS}" + "layer count must be at most UNIVMON_MAX_LAYERS {UNIVMON_MAX_LAYERS}" ); let sk_vec: Vec = (0..layers) .map(|i| { @@ -590,7 +590,7 @@ mod tests { /// Layer `UNIVMON_BOTTOM_LAYER_SEED_INDEX` would hash at the finder's own seed. #[test] - #[should_panic(expected = "MAX_LAYER_SIZE")] + #[should_panic(expected = "UNIVMON_MAX_LAYERS")] fn init_univmon_rejects_a_layer_on_the_finder_seed() { UnivMon::new(4, 2, 16, UNIVMON_BOTTOM_LAYER_SEED_INDEX + 1); } diff --git a/src/sketch_framework/univmon/wire.rs b/src/sketch_framework/univmon/wire.rs index 51cd936c..a3fc4d64 100644 --- a/src/sketch_framework/univmon/wire.rs +++ b/src/sketch_framework/univmon/wire.rs @@ -145,7 +145,7 @@ pub(crate) fn update_mode_of(tag: u8) -> Result Result<(), String> { if layers > UNIVMON_MAX_LAYERS { return Err(format!( - "{sketch} layer_size {layers} exceeds MAX_LAYER_SIZE {UNIVMON_MAX_LAYERS}" + "{sketch} layers {layers} exceeds UNIVMON_MAX_LAYERS {UNIVMON_MAX_LAYERS}" )); } Ok(()) @@ -162,7 +162,7 @@ pub(crate) fn check_univmon_dimensions( ) -> Result<(), String> { if layers == 0 || rows == 0 || cols == 0 || heap_size == 0 { return Err(format!( - "UnivMon layer_size, sketch_row, sketch_col and heap_size must be non-zero: layer_size={layers}, sketch_row={rows}, sketch_col={cols}, heap_size={heap_size}" + "UnivMon layers, rows, cols and heap_size must be non-zero: layers={layers}, rows={rows}, cols={cols}, heap_size={heap_size}" )); } Ok(()) @@ -478,13 +478,13 @@ pub(crate) fn rebuild_layers( let complain = |problem: String| RmpDecodeError::Uncategorized(problem); if decoded.heap_lens.len() != layers { return Err(complain(format!( - "UnivMon heap_lens length {} != layer_size {layers}", + "UnivMon heap_lens length {} != layers {layers}", decoded.heap_lens.len() ))); } if decoded.candidate_complete.len() != layers { return Err(complain(format!( - "UnivMon candidate_complete length {} != layer_size {layers}", + "UnivMon candidate_complete length {} != layers {layers}", decoded.candidate_complete.len() ))); } @@ -976,10 +976,10 @@ mod tests { let count_l2hh = crate::sketches::countsketch_heap::CountSketchWithL2Norm::::with_dimensions(2, 8) .to_asapv1() - .expect("serialize CountL2HH"); + .expect("serialize CountSketchWithL2Norm"); let pyramid = crate::UnivMonTwoTier::new(4, 1, 2, 8, 2, 4, 2) .to_asapv1() - .expect("serialize UnivMonPyramid"); + .expect("serialize UnivMonTwoTier"); let univmon_q = crate::UnivMonQ::new(crate::UnivMonQConfig { layers: 2, top_layer_cols: 8, @@ -1144,12 +1144,12 @@ mod tests { /// built by writing the public field directly. #[test] fn univmon_refuses_to_serialize_a_zero_dimension() { - for zero in ["layer_size", "sketch_row", "sketch_col", "heap_size"] { + for zero in ["layers", "rows", "cols", "heap_size"] { let mut um = UnivMon::new(4, 2, 16, 3); match zero { - "layer_size" => um.layers = 0, - "sketch_row" => um.rows = 0, - "sketch_col" => um.cols = 0, + "layers" => um.layers = 0, + "rows" => um.rows = 0, + "cols" => um.cols = 0, _ => um.heap_size = 0, } let problem = match um.to_asapv1() { @@ -1181,11 +1181,11 @@ mod tests { let problem = UnivMon::from_asapv1(&crafted(&meta, &payload)) .expect_err("a pyramid past the shift bound must be rejected") .to_string(); - assert!(problem.contains("MAX_LAYER_SIZE"), "got {problem}"); + assert!(problem.contains("UNIVMON_MAX_LAYERS"), "got {problem}"); } #[test] - #[should_panic(expected = "at most MAX_LAYER_SIZE")] + #[should_panic(expected = "at most UNIVMON_MAX_LAYERS")] fn univmon_refuses_constructing_past_the_shift_bound() { UnivMon::new(4, 2, 16, UNIVMON_MAX_LAYERS + 1); } diff --git a/src/sketch_framework/univmon_q.rs b/src/sketch_framework/univmon_q.rs index 396186d8..04006b72 100644 --- a/src/sketch_framework/univmon_q.rs +++ b/src/sketch_framework/univmon_q.rs @@ -79,7 +79,7 @@ impl UnivMonQConfig { failure_probability: f64, ) -> Result { if self.heap_size == 0 { - return Err(UnivMonQError::new("candidates must be positive")); + return Err(UnivMonQError::new("heap_size must be positive")); } if !failure_probability.is_finite() || !(0.0..1.0).contains(&failure_probability) { return Err(UnivMonQError::new("failure probability must be in (0, 1)")); @@ -95,7 +95,7 @@ impl UnivMonQConfig { } } Err(UnivMonQError::new( - "window bound does not fit the candidate table within 63 levels", + "window bound does not fit the candidate table within 63 layers", )) } } @@ -1553,19 +1553,19 @@ fn allocate_source_id() -> u64 { fn validate_config(config: UnivMonQConfig) -> Result<(), UnivMonQError> { if !(2..=63).contains(&config.layers) { - return Err(UnivMonQError::new("levels must be in 2..=63")); + return Err(UnivMonQError::new("layers must be in 2..=63")); } if config.top_layer_cols == 0 { - return Err(UnivMonQError::new("width must be positive")); + return Err(UnivMonQError::new("top_layer_cols must be positive")); } if config.rows == 0 || config.rows % 2 == 0 { - return Err(UnivMonQError::new("depth must be a positive odd number")); + return Err(UnivMonQError::new("rows must be a positive odd number")); } if !matches!(config.counter_bits, 32 | 64) { return Err(UnivMonQError::new("counter_bits must be 32 or 64")); } if config.heap_size == 0 { - return Err(UnivMonQError::new("candidates must be positive")); + return Err(UnivMonQError::new("heap_size must be positive")); } Ok(()) } diff --git a/src/sketch_framework/univmon_q/wire.rs b/src/sketch_framework/univmon_q/wire.rs index 2a8c81c5..0567f028 100644 --- a/src/sketch_framework/univmon_q/wire.rs +++ b/src/sketch_framework/univmon_q/wire.rs @@ -86,7 +86,7 @@ pub(crate) fn univmon_q_metadata( seed_derivation: H::SEED_DERIVATION.to_string(), input_encoding: H::INPUT_ENCODING.to_string(), seed_list: H::seed_list(), - seed_index: field("hash_seed", config.seed_index)?, + seed_index: field("seed_index", config.seed_index)?, layers: field("layers", config.layers)?, top_layer_cols: field("top_layer_cols", config.top_layer_cols)?, cols_halving_period: config.cols_halving_period, @@ -261,7 +261,7 @@ fn read_payload( fn validate_shared(config: UnivMonQConfig, shared: &SharedPayload) -> Result<(), RmpDecodeError> { if shared.candidate_lens.len() != config.layers || shared.ever_evicted.len() != config.layers { return Err(decode_error(format!( - "UnivMon-Q carries {} candidate runs and {} eviction flags over {} levels", + "UnivMon-Q carries {} candidate runs and {} eviction flags over {} layers", shared.candidate_lens.len(), shared.ever_evicted.len(), config.layers @@ -271,7 +271,7 @@ fn validate_shared(config: UnivMonQConfig, shared: &SharedPayload) -> Result<(), for (index, &len) in shared.candidate_lens.iter().enumerate() { if len as usize > config.heap_size { return Err(decode_error(format!( - "UnivMon-Q candidate capacity exceeded at level {index}" + "UnivMon-Q candidate capacity exceeded at layer {index}" ))); } seated = seated @@ -341,7 +341,7 @@ impl UnivMonQ { )?; if self.layers.len() != self.config.layers { return Err(syntax(format!( - "ASAPv1 UnivMon-Q envelope: {} levels against a config of {}", + "ASAPv1 UnivMon-Q envelope: {} layers against a config of {}", self.layers.len(), self.config.layers ))); @@ -380,12 +380,12 @@ impl UnivMonQ { self.config.counter_bits, ) { return Err(syntax(format!( - "ASAPv1 UnivMon-Q envelope: CountSketch layout mismatch at level {index}" + "ASAPv1 UnivMon-Q envelope: CountSketch layout mismatch at layer {index}" ))); } if level.candidate_scores.len() > self.config.heap_size { return Err(syntax(format!( - "ASAPv1 UnivMon-Q envelope: candidate capacity exceeded at level {index}" + "ASAPv1 UnivMon-Q envelope: candidate capacity exceeded at layer {index}" ))); } if level @@ -394,7 +394,7 @@ impl UnivMonQ { .any(|key| self.sample_level(*key) != index) { return Err(syntax(format!( - "ASAPv1 UnivMon-Q envelope: a candidate sits in the wrong terminal level {index}" + "ASAPv1 UnivMon-Q envelope: a candidate sits in the wrong terminal layer {index}" ))); } let mut candidates: Vec<(u64, u64)> = level @@ -405,7 +405,7 @@ impl UnivMonQ { candidates.sort_unstable(); candidate_lens.push(u32::try_from(candidates.len()).map_err(|_| { syntax(format!( - "ASAPv1 UnivMon-Q envelope: level {index} holds more candidates than u32 can name" + "ASAPv1 UnivMon-Q envelope: layer {index} holds more candidates than u32 can name" )) })?); candidate_keys.extend(candidates.iter().map(|entry| entry.0)); @@ -421,7 +421,7 @@ impl UnivMonQ { Counters::$variant(values) => counters.extend_from_slice(values), _ => { return Err(syntax( - "ASAPv1 UnivMon-Q envelope: levels mix counter widths".to_string(), + "ASAPv1 UnivMon-Q envelope: layers mix counter widths".to_string(), )); } } @@ -494,7 +494,7 @@ impl UnivMonQ { .checked_mul(config.rows) .and_then(|cells| total.checked_add(cells)) }) - .ok_or_else(|| decode_error("UnivMon-Q level layout overflows".to_string()))?; + .ok_or_else(|| decode_error("UnivMon-Q layer layout overflows".to_string()))?; if counters_len(&counters) != cells { return Err(decode_error(format!( "UnivMon-Q carries {} counters against the config's {cells}", @@ -560,7 +560,7 @@ impl UnivMonQ { .any(|key| sketch.sample_level(*key) != index) { return Err(decode_error(format!( - "UnivMon-Q candidate stored in the wrong terminal level {index}" + "UnivMon-Q candidate stored in the wrong terminal layer {index}" ))); } } @@ -815,13 +815,13 @@ mod tests { let count_l2hh = crate::sketches::countsketch_heap::CountSketchWithL2Norm::::with_dimensions(2, 8) .to_asapv1() - .expect("serialize CountL2HH"); + .expect("serialize CountSketchWithL2Norm"); let univmon = crate::UnivMon::new(4, 2, 8, 2) .to_asapv1() .expect("serialize UnivMon"); let pyramid = crate::UnivMonTwoTier::new(4, 1, 2, 8, 2, 4, 2) .to_asapv1() - .expect("serialize UnivMonPyramid"); + .expect("serialize UnivMonTwoTier"); for foreign in [count_sketch, count_l2hh, univmon, pyramid] { assert!( @@ -885,7 +885,7 @@ mod tests { misplaced.candidate_keys.reverse(); assert!( UnivMonQ::::from_asapv1(&crafted(&base, &misplaced)).is_err(), - "a candidate must live in the level its own key hashes into" + "a candidate must live in the layer its own key hashes into" ); } @@ -950,7 +950,7 @@ mod tests { wrong_layout.layers[1].sketch = PackedCountSketch::new(8, 3, 64); assert!( wrong_layout.to_asapv1().is_err(), - "a level whose CountSketch is not the config's size must not serialize" + "a layer whose CountSketch is not the config's size must not serialize" ); let mut wrong_extrema = populated(); diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index fb025e8c..fb46ba90 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -153,7 +153,7 @@ impl UnivMonTwoTier { assert!(layers > 0, "layer count must be positive"); assert!( layers <= UNIVMON_MAX_LAYERS, - "layer count must be at most MAX_LAYER_SIZE {UNIVMON_MAX_LAYERS}" + "layer count must be at most UNIVMON_MAX_LAYERS {UNIVMON_MAX_LAYERS}" ); let sk_vec: Vec = if layers <= elephant_layers { (0..layers) @@ -732,7 +732,7 @@ mod tests { let rel = (est - truth).abs() / truth.max(1e-12); assert!( rel < 0.15, - "Pyramid {name}: error {:.2}%, est={est:.2}, truth={truth:.2}", + "UnivMonTwoTier {name}: error {:.2}%, est={est:.2}, truth={truth:.2}", rel * 100.0 ); }; @@ -768,7 +768,7 @@ mod tests { let rel = (est - truth).abs() / truth.max(1e-12); assert!( rel < 0.15, - "Pyramid fast {name}: error {:.2}%, est={est:.2}, truth={truth:.2}", + "UnivMonTwoTier terminal {name}: error {:.2}%, est={est:.2}, truth={truth:.2}", rel * 100.0 ); }; @@ -795,7 +795,7 @@ mod tests { assert!( pyramid_cols < uniform_cols, - "Pyramid ({pyramid_cols}) should use fewer columns than uniform ({uniform_cols})" + "UnivMonTwoTier ({pyramid_cols}) should use fewer columns than uniform ({uniform_cols})" ); // With defaults: pyramid = 8*2048 + 8*512 = 20480, uniform = 16*2048 = 32768 // Savings = ~37.5% diff --git a/src/sketch_framework/univmon_two_tier/wire.rs b/src/sketch_framework/univmon_two_tier/wire.rs index 34de86eb..227305e2 100644 --- a/src/sketch_framework/univmon_two_tier/wire.rs +++ b/src/sketch_framework/univmon_two_tier/wire.rs @@ -125,7 +125,7 @@ fn check_layout_dimensions( || mouse_cols == 0 { return Err(format!( - "UnivMonPyramid layers, heap_size and both tiers' dimensions must be non-zero: layers={layers}, elephant_rows={elephant_rows}, elephant_cols={elephant_cols}, mouse_rows={mouse_rows}, mouse_cols={mouse_cols}, heap_size={heap_size}" + "UnivMonTwoTier layers, heap_size and both tiers' dimensions must be non-zero: layers={layers}, elephant_rows={elephant_rows}, elephant_cols={elephant_cols}, mouse_rows={mouse_rows}, mouse_cols={mouse_cols}, heap_size={heap_size}" )); } Ok(()) @@ -163,11 +163,11 @@ impl UnivMonTwoTier { self.mouse_cols, ) .map_err(RmpEncodeError::Syntax)?; - check_layers("UnivMonPyramid", self.layers).map_err(RmpEncodeError::Syntax)?; + check_layers("UnivMonTwoTier", self.layers).map_err(RmpEncodeError::Syntax)?; let field = |name: &str, value: usize| { u32::try_from(value).map_err(|_| { RmpEncodeError::Syntax(format!( - "ASAPv1 UnivMonPyramid envelope: {name} {value} exceeds the u32 metadata field" + "ASAPv1 UnivMonTwoTier envelope: {name} {value} exceeds the u32 metadata field" )) }) }; @@ -215,7 +215,7 @@ impl UnivMonTwoTier { envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::UNIVMON_TWO_TIER { return Err(RmpDecodeError::Uncategorized(format!( - "UnivMonPyramid kind_id mismatch: stored {kind_id:?}, expected {UNIVMON_TWO_TIER:?}" + "UnivMonTwoTier kind_id mismatch: stored {kind_id:?}, expected {UNIVMON_TWO_TIER:?}" ))); } let meta: UnivMonTwoTierMetadata = envelope::decode_block(metadata)?; @@ -235,7 +235,7 @@ impl UnivMonTwoTier { ) { return Err(RmpDecodeError::Uncategorized( - "ASAPv1 UnivMonPyramid envelope: metadata mismatch".to_string(), + "ASAPv1 UnivMonTwoTier envelope: metadata mismatch".to_string(), )); } let (layers, heap_size) = (meta.layers as usize, meta.heap_size as usize); @@ -254,7 +254,7 @@ impl UnivMonTwoTier { mouse_cols, ) .map_err(RmpDecodeError::Uncategorized)?; - check_layers("UnivMonPyramid", layers).map_err(RmpDecodeError::Uncategorized)?; + check_layers("UnivMonTwoTier", layers).map_err(RmpDecodeError::Uncategorized)?; let decoded = decode_pyramid(&meta.key_type, payload)?; check_empty_key_type(&meta.key_type, decoded.entries.is_empty()) .map_err(RmpDecodeError::Uncategorized)?; @@ -262,7 +262,7 @@ impl UnivMonTwoTier { // actually carries before the geometry is built from it. if accumulator_count(layers, &layout) != Some(decoded.l2.len()) { return Err(RmpDecodeError::Uncategorized(format!( - "UnivMonPyramid declares a layout the payload's {} accumulators do not match", + "UnivMonTwoTier declares a layout the payload's {} accumulators do not match", decoded.l2.len() ))); } @@ -571,7 +571,7 @@ mod tests { .expect("serialize Count Sketch"); let count_l2hh = CountSketchWithL2Norm::::with_dimensions(2, 8) .to_asapv1() - .expect("serialize CountL2HH"); + .expect("serialize CountSketchWithL2Norm"); let univmon = crate::UnivMon::new(4, 2, 8, 2) .to_asapv1() .expect("serialize UnivMon"); @@ -590,7 +590,7 @@ mod tests { for foreign in [count_sketch, count_l2hh, univmon, univmon_q] { assert!( UnivMonTwoTier::from_asapv1(&foreign).is_err(), - "a foreign envelope must not decode as a UnivMonPyramid" + "a foreign envelope must not decode as a UnivMonTwoTier" ); } } @@ -792,11 +792,11 @@ mod tests { let problem = UnivMonTwoTier::from_asapv1(&crafted(&meta, &payload)) .expect_err("a pyramid past the shift bound must be rejected") .to_string(); - assert!(problem.contains("MAX_LAYER_SIZE"), "got {problem}"); + assert!(problem.contains("UNIVMON_MAX_LAYERS"), "got {problem}"); } #[test] - #[should_panic(expected = "at most MAX_LAYER_SIZE")] + #[should_panic(expected = "at most UNIVMON_MAX_LAYERS")] fn pyramid_refuses_constructing_past_the_shift_bound() { UnivMonTwoTier::new(4, 2, 2, 16, 2, 8, UNIVMON_MAX_LAYERS + 1); } diff --git a/src/sketches/bloom.rs b/src/sketches/bloom.rs index f0c0301e..0345cb97 100644 --- a/src/sketches/bloom.rs +++ b/src/sketches/bloom.rs @@ -682,7 +682,7 @@ mod tests { assert_eq!( filter.inserted(), 8, - "bulk_insert of three left the count at {}", + "update_batch of three left the count at {}", filter.inserted() ); diff --git a/src/sketches/coco.rs b/src/sketches/coco.rs index 852201c1..3e378495 100644 --- a/src/sketches/coco.rs +++ b/src/sketches/coco.rs @@ -296,8 +296,8 @@ impl CocoSketch { } pub fn merge(&mut self, other: &CocoSketch) { - assert_eq!(self.rows, other.rows, "Different depth, do nothing"); - assert_eq!(self.cols, other.cols, "Different width, do nothing"); + assert_eq!(self.rows, other.rows, "Different rows, do nothing"); + assert_eq!(self.cols, other.cols, "Different cols, do nothing"); for i in 0..self.rows { for j in 0..self.cols { if let Some(k) = &other.table[i][j].full_key { diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 3e46c263..9d00a625 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -274,11 +274,11 @@ impl CountMinSketch { assert_eq!( (self_rows, self_cols), (other.counts.rows(), other.counts.cols()), - "dimension mismatch while merging CountMin sketches" + "dimension mismatch while merging Count-Min sketches" ); assert_eq!( self.admission_rate, other.admission_rate, - "admission rate mismatch while merging CountMin sketches" + "admission rate mismatch while merging Count-Min sketches" ); for i in 0..self_rows { @@ -301,11 +301,11 @@ impl CountMinSketch { assert_eq!( (self_rows, self_cols), (other.counts.rows(), other.counts.cols()), - "dimension mismatch while merging CountMin sketches" + "dimension mismatch while merging Count-Min sketches" ); assert_eq!( self.admission_rate, other.admission_rate, - "admission rate mismatch while merging CountMin sketches" + "admission rate mismatch while merging Count-Min sketches" ); for i in 0..self_rows { @@ -847,7 +847,7 @@ mod tests { } #[test] - #[should_panic(expected = "dimension mismatch while merging CountMin sketches")] + #[should_panic(expected = "dimension mismatch while merging Count-Min sketches")] fn merge_requires_matching_dimensions() { let mut left = CountMinSketch::, RegularPath>::with_dimensions(2, 32); let right = CountMinSketch::, RegularPath>::with_dimensions(3, 32); diff --git a/src/sketches/countminsketch/wire.rs b/src/sketches/countminsketch/wire.rs index 01b81c7c..bd7bbded 100644 --- a/src/sketches/countminsketch/wire.rs +++ b/src/sketches/countminsketch/wire.rs @@ -258,12 +258,12 @@ mod tests { sketch.update(&DataInput::U64(42)); sketch.update(&DataInput::U64(7)); - let encoded = sketch.to_asapv1().expect("serialize CountMin"); + let encoded = sketch.to_asapv1().expect("serialize CountMinSketch"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x02, 0x00]); // kind_id_len=2, kind_id=[0x02,0x00] let decoded = CountMinSketch::, RegularPath>::from_asapv1(&encoded) - .expect("deserialize CountMin"); + .expect("deserialize CountMinSketch"); assert_eq!(sketch.rows(), decoded.rows()); assert_eq!(sketch.cols(), decoded.cols()); diff --git a/src/sketches/countminsketch_heap.rs b/src/sketches/countminsketch_heap.rs index dec41396..5ef61a86 100644 --- a/src/sketches/countminsketch_heap.rs +++ b/src/sketches/countminsketch_heap.rs @@ -700,7 +700,7 @@ mod tests { } #[test] - #[should_panic(expected = "dimension mismatch while merging CountMin sketches")] + #[should_panic(expected = "dimension mismatch while merging Count-Min sketches")] fn merge_requires_matching_dimensions_panics() { // Verifies merge rejects sketches with incompatible dimensions. let mut left = CountMinSketchWithHeap::, RegularPath>::new(3, 256, 4); diff --git a/src/sketches/countminsketch_heap/delta.rs b/src/sketches/countminsketch_heap/delta.rs index 7226a752..5b7728cd 100644 --- a/src/sketches/countminsketch_heap/delta.rs +++ b/src/sketches/countminsketch_heap/delta.rs @@ -57,9 +57,9 @@ where deltas: Vec, heap: TopKHeap, ) -> Result { - check_geometry("CMSHeap", rows, cols)?; + check_geometry("CountMinSketchWithHeap", rows, cols)?; check_cells(&indices, &deltas, ..rows as u64 * cols as u64) - .map_err(|e| format!("CMSHeap delta: {e}"))?; + .map_err(|e| format!("CountMinSketchWithHeap delta: {e}"))?; Ok(Self { rows, cols, @@ -77,7 +77,7 @@ where let entries = heap_entries(&self.heap); let key_type = wire_key_type(&entries)?; let metadata = encode_metadata::( - "CMSHeap", + "CountMinSketchWithHeap", self.rows, self.cols, T::COUNTER_TYPE, @@ -103,10 +103,15 @@ where envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::COUNT_MIN_WITH_HEAP_DELTA { return Err(RmpDecodeError::Uncategorized(format!( - "CMSHeap delta kind_id mismatch: stored {kind_id:?}, expected {COUNT_MIN_WITH_HEAP_DELTA:?}" + "CountMinSketchWithHeap delta kind_id mismatch: stored {kind_id:?}, expected {COUNT_MIN_WITH_HEAP_DELTA:?}" ))); } - let shape = decode_metadata::("CMSHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; + let shape = decode_metadata::( + "CountMinSketchWithHeap", + metadata, + T::COUNTER_TYPE, + Mode::MODE, + )?; let (cells, entries) = decode_payload::, _>(&shape.key_type, payload)?; let heap = rebuild_heap(shape.heap_size, entries)?; Self::new(shape.rows, shape.cols, cells.indices, cells.deltas, heap) @@ -130,7 +135,7 @@ where let (rows, cols, heap_size) = (self.rows(), self.cols(), self.heap.capacity()); if (rows, cols, heap_size) != (delta.rows, delta.cols, delta.heap.capacity()) { return Err(format!( - "CMSHeap delta: metadata mismatch: a {}x{} delta with heap_size {} for a \ + "CountMinSketchWithHeap delta: metadata mismatch: a {}x{} delta with heap_size {} for a \ {rows}x{cols} sketch with heap_size {heap_size}", delta.rows, delta.cols, @@ -142,7 +147,7 @@ where &delta.indices, &delta.deltas, ) - .map_err(|e| format!("CMSHeap delta: {e}"))?; + .map_err(|e| format!("CountMinSketchWithHeap delta: {e}"))?; self.heap = delta.heap.clone(); Ok(()) } @@ -161,9 +166,16 @@ mod tests { /// A 2x3 `i64` delta envelope, heap_size 2, around the given payload arrays. fn delta_bytes(indices: &[u64], keys: &[u64], heap_counts: &[i64]) -> Vec { - let metadata = - encode_metadata::("CMSHeap", 2, 3, "i64", "regular", 2, "u64") - .unwrap(); + let metadata = encode_metadata::( + "CountMinSketchWithHeap", + 2, + 3, + "i64", + "regular", + 2, + "u64", + ) + .unwrap(); let deltas = vec![1i64; indices.len()]; let payload = rmp_serde::to_vec(&(indices, deltas, keys, heap_counts)).unwrap(); envelope::encode(kind::COUNT_MIN_WITH_HEAP_DELTA, &metadata, &payload) diff --git a/src/sketches/countminsketch_heap/wire.rs b/src/sketches/countminsketch_heap/wire.rs index b6d61f98..768d0eec 100644 --- a/src/sketches/countminsketch_heap/wire.rs +++ b/src/sketches/countminsketch_heap/wire.rs @@ -52,15 +52,16 @@ where /// heap's keys mix `OwnedDataInput` variants or hold a 128-bit key, or when /// `heap_size` or `cols` overflows the metadata's `u32` field. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { - admission_rate::check_admits_all(self.count_min_sketch.admission_rate()) - .map_err(|e| RmpEncodeError::Syntax(format!("ASAPv1 CMSHeap envelope: {e}")))?; + admission_rate::check_admits_all(self.count_min_sketch.admission_rate()).map_err(|e| { + RmpEncodeError::Syntax(format!("ASAPv1 CountMinSketchWithHeap envelope: {e}")) + })?; let rows = self.count_min_sketch.rows(); let cols = self.count_min_sketch.cols(); let counts = self.count_min_sketch.as_storage().as_slice(); let entries = heap_entries(&self.heap); let key_type = wire_key_type(&entries)?; let metadata = encode_metadata::( - "CMSHeap", + "CountMinSketchWithHeap", rows, cols, T::COUNTER_TYPE, @@ -70,7 +71,7 @@ where )?; if counts.len() != rows.saturating_mul(cols) { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 CMSHeap envelope: counts length {} != rows*cols {}", + "ASAPv1 CountMinSketchWithHeap envelope: counts length {} != rows*cols {}", counts.len(), rows.saturating_mul(cols) ))); @@ -97,17 +98,22 @@ where envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::COUNT_MIN_WITH_HEAP { return Err(RmpDecodeError::Uncategorized(format!( - "CMSHeap kind_id mismatch: stored {kind_id:?}, expected {COUNT_MIN_WITH_HEAP:?}" + "CountMinSketchWithHeap kind_id mismatch: stored {kind_id:?}, expected {COUNT_MIN_WITH_HEAP:?}" ))); } - let shape = decode_metadata::("CMSHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; + let shape = decode_metadata::( + "CountMinSketchWithHeap", + metadata, + T::COUNTER_TYPE, + Mode::MODE, + )?; let (rows, cols) = (shape.rows, shape.cols); let (counts, entries) = decode_payload::, _>(&shape.key_type, payload)?; // Length check precedes the allocation, so crafted dimensions cannot // drive `from_fn` into a huge reserve. if counts.len() != rows.saturating_mul(cols) { return Err(RmpDecodeError::Uncategorized(format!( - "CMSHeap counts length {} != rows*cols {}", + "CountMinSketchWithHeap counts length {} != rows*cols {}", counts.len(), rows.saturating_mul(cols) ))); @@ -158,7 +164,9 @@ mod tests { #[test] fn cms_heap_round_trip_serialization() { let sketch = populated(); - let encoded = sketch.to_asapv1().expect("serialize CMSHeap"); + let encoded = sketch + .to_asapv1() + .expect("serialize CountMinSketchWithHeap"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x03, 0x00]); // kind_id_len=2, kind_id=[0x03,0x00] @@ -171,7 +179,7 @@ mod tests { assert_eq!(meta.key_type, "u64"); let decoded = CountMinSketchWithHeap::, RegularPath>::from_asapv1(&encoded) - .expect("deserialize CMSHeap"); + .expect("deserialize CountMinSketchWithHeap"); assert_eq!(sketch.rows(), decoded.rows()); assert_eq!(sketch.cols(), decoded.cols()); assert_eq!( @@ -298,13 +306,16 @@ mod tests { let count = crate::CountSketch::, RegularPath>::with_dimensions(3, 8); for (bytes, what) in [ - (cs_heap.to_asapv1().expect("CSHeap"), "CSHeap"), + ( + cs_heap.to_asapv1().expect("CountSketchWithHeap"), + "CountSketchWithHeap", + ), (cms.to_asapv1().expect("CMS"), "Count-Min"), - (count.to_asapv1().expect("Count"), "Count Sketch"), + (count.to_asapv1().expect("CountSketch"), "Count Sketch"), ] { assert!( CountMinSketchWithHeap::, RegularPath>::from_asapv1(&bytes).is_err(), - "{what} bytes must not decode as a CMSHeap" + "{what} bytes must not decode as a CountMinSketchWithHeap" ); } } diff --git a/src/sketches/countminsketch_hll.rs b/src/sketches/countminsketch_hll.rs index d85760ff..b7bd2a10 100644 --- a/src/sketches/countminsketch_hll.rs +++ b/src/sketches/countminsketch_hll.rs @@ -113,7 +113,7 @@ impl<'de, H: SketchHasher> Deserialize<'de> for CountMinHll { let expected_depth = 1usize << precision; if buckets.bucket_len() != expected_depth { return Err(serde::de::Error::custom(format!( - "buckets depth {} does not match 2^precision {} = {expected_depth}", + "buckets bucket_len {} does not match 2^precision {} = {expected_depth}", buckets.bucket_len(), precision ))); @@ -542,7 +542,7 @@ mod tests { assert_eq!( sk_seq.as_storage().as_slice(), sk_batch.as_storage().as_slice(), - "insert_many must produce identical state to sequential inserts" + "update_batch must produce identical state to sequential updates" ); } @@ -595,7 +595,11 @@ mod tests { for (label, err, want) in [ ("precision 0", attempt(4, 32, 1, 0), "out of range"), ("precision 19", attempt(4, 32, 1, 19), "out of range"), - ("depth mismatch", attempt(4, 32, 128, 8), "does not match"), + ( + "bucket_len mismatch", + attempt(4, 32, 128, 8), + "does not match", + ), ( "cols not a power of two", attempt(4, 17, 256, 8), diff --git a/src/sketches/countminsketch_space_saving.rs b/src/sketches/countminsketch_space_saving.rs index 576eaa0f..38f9bc93 100644 --- a/src/sketches/countminsketch_space_saving.rs +++ b/src/sketches/countminsketch_space_saving.rs @@ -157,7 +157,7 @@ impl CountMinSketchWithSpaceSaving, RegularPath>::with_dimensions(2, 8); let summary = SpaceSaving::::with_capacity(3); for (bytes, what) in [ - (cms_heap.to_asapv1().expect("CMSHeap"), "CMSHeap"), - (cs_ss.to_asapv1().expect("CSSpaceSaving"), "CSSpaceSaving"), + ( + cms_heap.to_asapv1().expect("CountMinSketchWithHeap"), + "CountMinSketchWithHeap", + ), + ( + cs_ss.to_asapv1().expect("CountSketchWithSpaceSaving"), + "CountSketchWithSpaceSaving", + ), (cms.to_asapv1().expect("Count-Min"), "Count-Min"), (summary.to_asapv1().expect("Space-Saving"), "Space-Saving"), ] { assert!( decode_error(&bytes).contains("kind_id mismatch"), - "{what} bytes must not decode as a CMSSpaceSaving" + "{what} bytes must not decode as a CountMinSketchWithSpaceSaving" ); } } @@ -826,7 +834,10 @@ mod tests { SpaceSaving::with_capacity(4), ); let problem = sampled.to_asapv1().expect_err("sampled").to_string(); - assert!(problem.contains("CMSSpaceSaving"), "got {problem}"); + assert!( + problem.contains("CountMinSketchWithSpaceSaving"), + "got {problem}" + ); let mut mixed = Target::new(2, 3, 4); mixed.update(&DataInput::U64(1)); diff --git a/src/sketches/countminsketch_space_saving/wire.rs b/src/sketches/countminsketch_space_saving/wire.rs index ce221a5f..deec30d0 100644 --- a/src/sketches/countminsketch_space_saving/wire.rs +++ b/src/sketches/countminsketch_space_saving/wire.rs @@ -20,7 +20,7 @@ where H: SketchHasher + HashProfile, { const KIND: SummaryKind = SummaryKind { - name: "CMSSpaceSaving", + name: "CountMinSketchWithSpaceSaving", kind_id: kind::COUNT_MIN_WITH_SPACE_SAVING, counter_type: T::COUNTER_TYPE, mode: Mode::MODE, @@ -30,8 +30,11 @@ where /// when the matrix samples its updates or the state breaks a §3.24 decode /// rule, and on mixed-variant or 128-bit keys. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { - admission_rate::check_admits_all(self.count_min_sketch.admission_rate()) - .map_err(|e| RmpEncodeError::Syntax(format!("ASAPv1 CMSSpaceSaving envelope: {e}")))?; + admission_rate::check_admits_all(self.count_min_sketch.admission_rate()).map_err(|e| { + RmpEncodeError::Syntax(format!( + "ASAPv1 CountMinSketchWithSpaceSaving envelope: {e}" + )) + })?; summary_wire::encode::( &Self::KIND, self.count_min_sketch.rows(), diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 6ddea312..9c5cf48d 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -276,11 +276,11 @@ where assert_eq!( (self_rows, self_cols), (other.counts.rows(), other.counts.cols()), - "dimension mismatch while merging CountMin sketches" + "dimension mismatch while merging Count Sketches" ); assert_eq!( self.admission_rate, other.admission_rate, - "admission rate mismatch while merging Count sketches" + "admission rate mismatch while merging Count Sketches" ); for i in 0..self_rows { @@ -923,7 +923,7 @@ mod tests { } #[test] - #[should_panic(expected = "dimension mismatch while merging CountMin sketches")] + #[should_panic(expected = "dimension mismatch while merging Count Sketches")] fn merge_requires_matching_dimensions() { let mut left = CountSketch::, RegularPath>::with_dimensions(2, 32); let right = CountSketch::, RegularPath>::with_dimensions(3, 32); diff --git a/src/sketches/countsketch/wire.rs b/src/sketches/countsketch/wire.rs index 92e8e2cd..0451ee81 100644 --- a/src/sketches/countsketch/wire.rs +++ b/src/sketches/countsketch/wire.rs @@ -275,12 +275,12 @@ mod tests { sketch.update(&DataInput::U64(42)); sketch.update(&DataInput::U64(7)); - let encoded = sketch.to_asapv1().expect("serialize Count"); + let encoded = sketch.to_asapv1().expect("serialize CountSketch"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x04, 0x00]); // kind_id_len=2, kind_id=[0x04,0x00] let decoded = CountSketch::, RegularPath>::from_asapv1(&encoded) - .expect("deserialize Count"); + .expect("deserialize CountSketch"); assert_eq!(sketch.rows(), decoded.rows()); assert_eq!(sketch.cols(), decoded.cols()); diff --git a/src/sketches/countsketch_heap.rs b/src/sketches/countsketch_heap.rs index 58ec7b40..e94a82fe 100644 --- a/src/sketches/countsketch_heap.rs +++ b/src/sketches/countsketch_heap.rs @@ -791,7 +791,7 @@ mod tests { } #[test] - #[should_panic(expected = "dimension mismatch while merging CountMin sketches")] + #[should_panic(expected = "dimension mismatch while merging Count Sketches")] fn merge_requires_matching_dimensions_panics() { // Verifies merge panics when dimensions differ. let mut left = CountSketchWithHeap::, RegularPath>::new(5, 256, 4); @@ -1050,7 +1050,7 @@ impl CountSketchWithL2Norm { let bits_required = mask_bits.saturating_add(1).saturating_mul(rows); if bits_required > 128 { return Err(format!( - "CountL2HH rows={rows}, cols={cols} needs {bits_required} hash bits \ + "CountSketchWithL2Norm rows={rows}, cols={cols} needs {bits_required} hash bits \ ({mask_bits} column bits + 1 sign bit per row), past the 128 the key hash carries" )); } @@ -1132,11 +1132,11 @@ impl CountSketchWithL2Norm { assert_eq!( (self.rows, self.cols), (other.rows, other.cols), - "dimension mismatch while merging CountL2HH sketches" + "dimension mismatch while merging CountSketchWithL2Norm sketches" ); assert_eq!( self.seed_index, other.seed_index, - "seed mismatch while merging CountL2HH sketches" + "seed mismatch while merging CountSketchWithL2Norm sketches" ); for i in 0..self.rows { @@ -1357,12 +1357,12 @@ mod tests_count_l2_hh { let encoded = sketch .to_asapv1() - .expect("serialize CountL2HH into MessagePack"); + .expect("serialize CountSketchWithL2Norm into MessagePack"); assert!(!encoded.is_empty(), "serialized bytes should not be empty"); let data = encoded.clone(); let decoded: CountSketchWithL2Norm = CountSketchWithL2Norm::from_asapv1(&data) - .expect("deserialize CountL2HH from MessagePack"); + .expect("deserialize CountSketchWithL2Norm from MessagePack"); assert_eq!(sketch.rows(), decoded.rows()); assert_eq!(sketch.cols(), decoded.cols()); diff --git a/src/sketches/countsketch_heap/delta.rs b/src/sketches/countsketch_heap/delta.rs index 16220fb9..95f8db9e 100644 --- a/src/sketches/countsketch_heap/delta.rs +++ b/src/sketches/countsketch_heap/delta.rs @@ -53,9 +53,9 @@ where deltas: Vec, heap: TopKHeap, ) -> Result { - check_geometry("CSHeap", rows, cols)?; + check_geometry("CountSketchWithHeap", rows, cols)?; check_cells(&indices, &deltas, ..rows as u64 * cols as u64) - .map_err(|e| format!("CSHeap delta: {e}"))?; + .map_err(|e| format!("CountSketchWithHeap delta: {e}"))?; Ok(Self { rows, cols, @@ -73,7 +73,7 @@ where let entries = heap_entries(&self.heap); let key_type = wire_key_type(&entries)?; let metadata = encode_metadata::( - "CSHeap", + "CountSketchWithHeap", self.rows, self.cols, T::COUNTER_TYPE, @@ -99,10 +99,11 @@ where envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::COUNT_SKETCH_WITH_HEAP_DELTA { return Err(RmpDecodeError::Uncategorized(format!( - "CSHeap delta kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_HEAP_DELTA:?}" + "CountSketchWithHeap delta kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_HEAP_DELTA:?}" ))); } - let shape = decode_metadata::("CSHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; + let shape = + decode_metadata::("CountSketchWithHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; let (cells, entries) = decode_payload::, _>(&shape.key_type, payload)?; let heap = rebuild_heap(shape.heap_size, entries)?; Self::new(shape.rows, shape.cols, cells.indices, cells.deltas, heap) @@ -126,7 +127,7 @@ where let (rows, cols, heap_size) = (self.rows(), self.cols(), self.heap.capacity()); if (rows, cols, heap_size) != (delta.rows, delta.cols, delta.heap.capacity()) { return Err(format!( - "CSHeap delta: metadata mismatch: a {}x{} delta with heap_size {} for a \ + "CountSketchWithHeap delta: metadata mismatch: a {}x{} delta with heap_size {} for a \ {rows}x{cols} sketch with heap_size {heap_size}", delta.rows, delta.cols, @@ -138,7 +139,7 @@ where &delta.indices, &delta.deltas, ) - .map_err(|e| format!("CSHeap delta: {e}"))?; + .map_err(|e| format!("CountSketchWithHeap delta: {e}"))?; self.heap = delta.heap.clone(); Ok(()) } @@ -157,8 +158,16 @@ mod tests { /// A 2x4 `i64` delta envelope, heap_size 2, around the given indices. fn delta_bytes(indices: &[u64]) -> Vec { - let metadata = - encode_metadata::("CSHeap", 2, 4, "i64", "regular", 2, "u64").unwrap(); + let metadata = encode_metadata::( + "CountSketchWithHeap", + 2, + 4, + "i64", + "regular", + 2, + "u64", + ) + .unwrap(); let deltas = vec![-1i64; indices.len()]; let payload = rmp_serde::to_vec(&(indices, deltas, [1u64], [-3i64])).unwrap(); envelope::encode(kind::COUNT_SKETCH_WITH_HEAP_DELTA, &metadata, &payload) diff --git a/src/sketches/countsketch_heap/l2hh_wire.rs b/src/sketches/countsketch_heap/l2hh_wire.rs index c290abd0..31850b51 100644 --- a/src/sketches/countsketch_heap/l2hh_wire.rs +++ b/src/sketches/countsketch_heap/l2hh_wire.rs @@ -104,10 +104,10 @@ struct CountSketchWithL2NormPayload { pub(crate) fn check_dimensions(rows: usize, cols: usize) -> Result<(), String> { if rows == 0 || cols == 0 { return Err(format!( - "CountL2HH dimensions must be non-zero: rows={rows}, cols={cols}" + "CountSketchWithL2Norm dimensions must be non-zero: rows={rows}, cols={cols}" )); } - check_matrix_rows("CountL2HH", rows)?; + check_matrix_rows("CountSketchWithL2Norm", rows)?; CountSketchWithL2Norm::check_hash_bits(rows, cols) } @@ -117,7 +117,7 @@ pub(crate) fn check_dimensions(rows: usize, cols: usize) -> Result<(), String> { pub(crate) fn check_seed_index(seed_index: usize, seed_list_len: usize) -> Result<(), String> { if seed_index >= seed_list_len { return Err(format!( - "CountL2HH seed_index {seed_index} is past the seed list's {seed_list_len} entries" + "CountSketchWithL2Norm seed_index {seed_index} is past the seed list's {seed_list_len} entries" )); } Ok(()) @@ -136,20 +136,20 @@ pub(crate) fn layer_state( let l2 = sketch.row_l2_squared.as_slice(); if counts.len() != cells { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 CountL2HH state: counts length {} != rows*cols {cells}", + "ASAPv1 CountSketchWithL2Norm state: counts length {} != rows*cols {cells}", counts.len() ))); } if l2.len() != sketch.rows { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 CountL2HH state: l2 length {} != rows {}", + "ASAPv1 CountSketchWithL2Norm state: l2 length {} != rows {}", l2.len(), sketch.rows ))); } if let Some(negative) = l2.iter().find(|&&value| value < 0) { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 CountL2HH state: l2 accumulator {negative} is negative" + "ASAPv1 CountSketchWithL2Norm state: l2 accumulator {negative} is negative" ))); } Ok((counts, l2)) @@ -169,19 +169,19 @@ pub(crate) fn rebuild_layer( let cells = rows.saturating_mul(cols); if counts.len() != cells { return Err(RmpDecodeError::Uncategorized(format!( - "CountL2HH counts length {} != rows*cols {cells}", + "CountSketchWithL2Norm counts length {} != rows*cols {cells}", counts.len() ))); } if l2.len() != rows { return Err(RmpDecodeError::Uncategorized(format!( - "CountL2HH l2 length {} != rows {rows}", + "CountSketchWithL2Norm l2 length {} != rows {rows}", l2.len() ))); } if let Some(negative) = l2.iter().find(|&&value| value < 0) { return Err(RmpDecodeError::Uncategorized(format!( - "CountL2HH l2 accumulator {negative} is negative" + "CountSketchWithL2Norm l2 accumulator {negative} is negative" ))); } Ok(CountSketchWithL2Norm { @@ -213,7 +213,7 @@ impl CountSketchWithL2Norm { let field = |name: &str, value: usize| { u32::try_from(value).map_err(|_| { RmpEncodeError::Syntax(format!( - "ASAPv1 CountL2HH envelope: {name} {value} exceeds the u32 metadata field" + "ASAPv1 CountSketchWithL2Norm envelope: {name} {value} exceeds the u32 metadata field" )) }) }; @@ -242,13 +242,13 @@ impl CountSketchWithL2Norm { envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::COUNT_SKETCH_WITH_L2_NORM { return Err(RmpDecodeError::Uncategorized(format!( - "CountL2HH kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_L2_NORM:?}" + "CountSketchWithL2Norm kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_L2_NORM:?}" ))); } let meta: CountSketchWithL2NormMetadata = envelope::decode_block(metadata)?; if meta != count_sketch_with_l2_norm_metadata::(meta.seed_index, meta.rows, meta.cols) { return Err(RmpDecodeError::Uncategorized( - "ASAPv1 CountL2HH envelope: metadata mismatch".to_string(), + "ASAPv1 CountSketchWithL2Norm envelope: metadata mismatch".to_string(), )); } check_seed_index(meta.seed_index as usize, meta.seed_list.len()) @@ -327,7 +327,7 @@ mod tests { #[test] fn count_l2hh_round_trip_serialization() { let sketch = populated(); - let encoded = sketch.to_asapv1().expect("serialize CountL2HH"); + let encoded = sketch.to_asapv1().expect("serialize CountSketchWithL2Norm"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x19, 0x00]); // kind_id_len=2, kind_id=[0x19,0x00] @@ -335,7 +335,7 @@ mod tests { assert_eq!(meta.metadata_version, 1); assert_eq!((meta.seed_index, meta.rows, meta.cols), (7, 3, 32)); - let decoded = Std::from_asapv1(&encoded).expect("deserialize CountL2HH"); + let decoded = Std::from_asapv1(&encoded).expect("deserialize CountSketchWithL2Norm"); assert_eq!(decoded.rows(), 3); assert_eq!(decoded.cols(), 32); assert_eq!(decoded.seed_index(), 7); @@ -473,7 +473,7 @@ mod tests { .expect("serialize UnivMon"); let pyramid = crate::UnivMonTwoTier::new(4, 1, 2, 8, 2, 4, 2) .to_asapv1() - .expect("serialize UnivMonPyramid"); + .expect("serialize UnivMonTwoTier"); let univmon_q = crate::UnivMonQ::new(crate::UnivMonQConfig { layers: 2, top_layer_cols: 8, @@ -489,7 +489,7 @@ mod tests { for foreign in [count_sketch, univmon, pyramid, univmon_q] { assert!( Std::from_asapv1(&foreign).is_err(), - "a foreign envelope must not decode as a CountL2HH" + "a foreign envelope must not decode as a CountSketchWithL2Norm" ); } } diff --git a/src/sketches/countsketch_heap/wire.rs b/src/sketches/countsketch_heap/wire.rs index 6890fcb2..7f62a7d5 100644 --- a/src/sketches/countsketch_heap/wire.rs +++ b/src/sketches/countsketch_heap/wire.rs @@ -60,15 +60,16 @@ where /// heap's keys mix `OwnedDataInput` variants or hold a 128-bit key, or when /// `heap_size` or `cols` overflows the metadata's `u32` field. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { - admission_rate::check_admits_all(self.count_sketch.admission_rate()) - .map_err(|e| RmpEncodeError::Syntax(format!("ASAPv1 CSHeap envelope: {e}")))?; + admission_rate::check_admits_all(self.count_sketch.admission_rate()).map_err(|e| { + RmpEncodeError::Syntax(format!("ASAPv1 CountSketchWithHeap envelope: {e}")) + })?; let rows = self.count_sketch.rows(); let cols = self.count_sketch.cols(); let counts = self.count_sketch.as_storage().as_slice(); let entries = heap_entries(&self.heap); let key_type = wire_key_type(&entries)?; let metadata = encode_metadata::( - "CSHeap", + "CountSketchWithHeap", rows, cols, T::COUNTER_TYPE, @@ -78,7 +79,7 @@ where )?; if counts.len() != rows.saturating_mul(cols) { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 CSHeap envelope: counts length {} != rows*cols {}", + "ASAPv1 CountSketchWithHeap envelope: counts length {} != rows*cols {}", counts.len(), rows.saturating_mul(cols) ))); @@ -105,17 +106,18 @@ where envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; if kind_id != kind::COUNT_SKETCH_WITH_HEAP { return Err(RmpDecodeError::Uncategorized(format!( - "CSHeap kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_HEAP:?}" + "CountSketchWithHeap kind_id mismatch: stored {kind_id:?}, expected {COUNT_SKETCH_WITH_HEAP:?}" ))); } - let shape = decode_metadata::("CSHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; + let shape = + decode_metadata::("CountSketchWithHeap", metadata, T::COUNTER_TYPE, Mode::MODE)?; let (rows, cols) = (shape.rows, shape.cols); let (counts, entries) = decode_payload::, _>(&shape.key_type, payload)?; // Length check precedes the allocation, so crafted dimensions cannot // drive `from_fn` into a huge reserve. if counts.len() != rows.saturating_mul(cols) { return Err(RmpDecodeError::Uncategorized(format!( - "CSHeap counts length {} != rows*cols {}", + "CountSketchWithHeap counts length {} != rows*cols {}", counts.len(), rows.saturating_mul(cols) ))); @@ -166,7 +168,7 @@ mod tests { #[test] fn cs_heap_round_trip_serialization() { let sketch = populated(); - let encoded = sketch.to_asapv1().expect("serialize CSHeap"); + let encoded = sketch.to_asapv1().expect("serialize CountSketchWithHeap"); assert!(encoded.starts_with(b"ASAPv1")); assert_eq!(&encoded[7..10], &[2u8, 0x0a, 0x00]); // kind_id_len=2, kind_id=[0x0a,0x00] @@ -179,7 +181,7 @@ mod tests { assert_eq!(meta.key_type, "u64"); let decoded = CountSketchWithHeap::, RegularPath>::from_asapv1(&encoded) - .expect("deserialize CSHeap"); + .expect("deserialize CountSketchWithHeap"); assert_eq!(sketch.rows(), decoded.rows()); assert_eq!(sketch.cols(), decoded.cols()); assert_eq!( @@ -302,13 +304,16 @@ mod tests { let count = crate::CountSketch::, RegularPath>::with_dimensions(3, 8); for (bytes, what) in [ - (cms_heap.to_asapv1().expect("CMSHeap"), "CMSHeap"), + ( + cms_heap.to_asapv1().expect("CountMinSketchWithHeap"), + "CountMinSketchWithHeap", + ), (cms.to_asapv1().expect("CMS"), "Count-Min"), - (count.to_asapv1().expect("Count"), "Count Sketch"), + (count.to_asapv1().expect("CountSketch"), "Count Sketch"), ] { assert!( CountSketchWithHeap::, RegularPath>::from_asapv1(&bytes).is_err(), - "{what} bytes must not decode as a CSHeap" + "{what} bytes must not decode as a CountSketchWithHeap" ); } } diff --git a/src/sketches/countsketch_space_saving.rs b/src/sketches/countsketch_space_saving.rs index 81f4143f..fcf8db89 100644 --- a/src/sketches/countsketch_space_saving.rs +++ b/src/sketches/countsketch_space_saving.rs @@ -80,7 +80,7 @@ where assert_eq!( self.summary.capacity(), other.summary.capacity(), - "Space-Saving capacity mismatch while merging CSSpaceSaving sketches" + "Space-Saving capacity mismatch while merging CountSketchWithSpaceSaving sketches" ); self.count_sketch.merge(&other.count_sketch); self.summary.merge(&other.summary); diff --git a/src/sketches/countsketch_space_saving/wire.rs b/src/sketches/countsketch_space_saving/wire.rs index 9cc68972..57441278 100644 --- a/src/sketches/countsketch_space_saving/wire.rs +++ b/src/sketches/countsketch_space_saving/wire.rs @@ -22,7 +22,7 @@ where H: SketchHasher + HashProfile, { const KIND: SummaryKind = SummaryKind { - name: "CSSpaceSaving", + name: "CountSketchWithSpaceSaving", kind_id: kind::COUNT_SKETCH_WITH_SPACE_SAVING, counter_type: T::COUNTER_TYPE, mode: Mode::MODE, @@ -32,8 +32,9 @@ where /// when the matrix samples its updates or the state breaks a §3.24 decode /// rule, and on mixed-variant or 128-bit keys. pub fn to_asapv1(&self) -> Result, RmpEncodeError> { - admission_rate::check_admits_all(self.count_sketch.admission_rate()) - .map_err(|e| RmpEncodeError::Syntax(format!("ASAPv1 CSSpaceSaving envelope: {e}")))?; + admission_rate::check_admits_all(self.count_sketch.admission_rate()).map_err(|e| { + RmpEncodeError::Syntax(format!("ASAPv1 CountSketchWithSpaceSaving envelope: {e}")) + })?; summary_wire::encode::( &Self::KIND, self.count_sketch.rows(), @@ -122,15 +123,18 @@ mod tests { let count = CountSketch::, RegularPath>::with_dimensions(3, 8); for (bytes, what) in [ ( - cms_ss.to_asapv1().expect("CMSSpaceSaving"), - "CMSSpaceSaving", + cms_ss.to_asapv1().expect("CountMinSketchWithSpaceSaving"), + "CountMinSketchWithSpaceSaving", ), - (cs_heap.to_asapv1().expect("CSHeap"), "CSHeap"), - (count.to_asapv1().expect("Count"), "Count Sketch"), + ( + cs_heap.to_asapv1().expect("CountSketchWithHeap"), + "CountSketchWithHeap", + ), + (count.to_asapv1().expect("CountSketch"), "Count Sketch"), ] { assert!( Target::from_asapv1(&bytes).is_err(), - "{what} bytes must not decode as a CSSpaceSaving" + "{what} bytes must not decode as a CountSketchWithSpaceSaving" ); } } diff --git a/src/sketches/elastic.rs b/src/sketches/elastic.rs index 6c4b1123..5e40aefe 100644 --- a/src/sketches/elastic.rs +++ b/src/sketches/elastic.rs @@ -721,7 +721,7 @@ mod tests { ); assert!( light_est >= 6, - "colliding flow should accumulate in CountMin, expected >= 6, got {light_est}" + "colliding flow should accumulate in CountMinSketch, expected >= 6, got {light_est}" ); } diff --git a/src/sketches/elastic/wire.rs b/src/sketches/elastic/wire.rs index e7974d40..bc3ee11d 100644 --- a/src/sketches/elastic/wire.rs +++ b/src/sketches/elastic/wire.rs @@ -160,7 +160,7 @@ impl ElasticSketch { .map_err(RmpEncodeError::Syntax)?; if self.heavy_buckets <= 0 || self.heavy_buckets as usize != heavy_buckets { return Err(RmpEncodeError::Syntax(format!( - "ASAPv1 Elastic envelope: bktlen {} != heavy table length {heavy_buckets}", + "ASAPv1 Elastic envelope: heavy_buckets {} != heavy table length {heavy_buckets}", self.heavy_buckets ))); } @@ -569,7 +569,7 @@ mod tests { desynced.heavy_buckets = 16; assert!( desynced.to_asapv1().is_err(), - "a bktlen disagreeing with the heavy table must not serialize" + "a heavy_buckets disagreeing with the heavy table must not serialize" ); let mut named_free: ElasticSketch = ElasticSketch::with_heavy_and_light(8, 2, 256); diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index afd6a784..112a99d7 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -384,14 +384,14 @@ mod tests { let encoded = sketch .to_asapv1() - .unwrap_or_else(|err| panic!("{name} serialize_to_bytes failed: {err}")); + .unwrap_or_else(|err| panic!("{name} to_asapv1 failed: {err}")); assert!( !encoded.is_empty(), "{name} serialization output should not be empty" ); let decoded = S::from_asapv1(&encoded) - .unwrap_or_else(|err| panic!("{name} deserialize_from_bytes failed: {err}")); + .unwrap_or_else(|err| panic!("{name} from_asapv1 failed: {err}")); let reencoded = decoded .to_asapv1() diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 0d6d819e..0eef240d 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -1708,8 +1708,8 @@ mod tests { ); // Serialization round-trip for the generic specialization. - let bytes = a.to_asapv1().expect("serialize KLL"); - let restored = Kll::::from_asapv1(&bytes).expect("deserialize KLL"); + let bytes = a.to_asapv1().expect("serialize Kll"); + let restored = Kll::::from_asapv1(&bytes).expect("deserialize Kll"); assert_eq!(a.count(), restored.count()); } diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 6d288202..bbfdc65b 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -293,7 +293,7 @@ impl KllDynamic { T: std::fmt::Debug, { println!( - "KLLDynamic Packed (k={}, levels={}, items={})", + "KllDynamic Packed (k={}, levels={}, items={})", self.k, self.num_levels, self.items.len() @@ -906,10 +906,10 @@ mod tests { sketch.update(value); } - let bytes = sketch.to_asapv1().expect("serialize KLLDynamic with rmp"); + let bytes = sketch.to_asapv1().expect("serialize KllDynamic with rmp"); assert!(!bytes.is_empty(), "serialized bytes should not be empty"); - let restored = KllDynamic::from_asapv1(&bytes).expect("deserialize KLLDynamic with rmp"); + let restored = KllDynamic::from_asapv1(&bytes).expect("deserialize KllDynamic with rmp"); assert_eq!(sketch.k, restored.k); assert_eq!(sketch.m, restored.m); assert_eq!(sketch.num_levels, restored.num_levels); @@ -966,8 +966,8 @@ mod tests { "p90={p90} out of range for n={n}" ); - let bytes = sketch.to_asapv1().expect("serialize KLLDynamic"); - let restored = KllDynamic::::from_asapv1(&bytes).expect("deserialize KLLDynamic"); + let bytes = sketch.to_asapv1().expect("serialize KllDynamic"); + let restored = KllDynamic::::from_asapv1(&bytes).expect("deserialize KllDynamic"); assert_eq!(sketch.count(), restored.count()); } diff --git a/src/sketches/kll_dynamic/wire.rs b/src/sketches/kll_dynamic/wire.rs index 4feb16b8..179e0969 100644 --- a/src/sketches/kll_dynamic/wire.rs +++ b/src/sketches/kll_dynamic/wire.rs @@ -122,7 +122,7 @@ mod tests { assert_eq!( decoded.to_asapv1().expect("re-serialize"), bytes, - "KLLDynamic serialized bytes differed after round trip" + "KllDynamic serialized bytes differed after round trip" ); for &q in &[0.0, 0.01, 0.25, 0.5, 0.75, 0.99, 1.0] { assert_eq!( @@ -159,7 +159,7 @@ mod tests { let bytes = sketch.to_asapv1().expect("serialize"); assert!( KllDynamic::::from_asapv1(&bytes).is_err(), - "f64 KLLDynamic bytes must be rejected by an i64 decoder" + "f64 KllDynamic bytes must be rejected by an i64 decoder" ); } } diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index 72bcdec1..bb1228f8 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -66,7 +66,7 @@ impl Kmv { let largest = *self .retained_hashes .peek() - .expect("k_vals should be non-empty when len >= k"); + .expect("retained_hashes should be non-empty when len >= k"); const HASH_RANGE: f64 = 18_446_744_073_709_551_616.0; let mapped: f64 = (largest as f64 + 1.0) / HASH_RANGE; (self.k - 1) as f64 / mapped @@ -102,14 +102,14 @@ mod tests { let encoded = sketch .to_asapv1() - .unwrap_or_else(|err| panic!("KMV serialize_to_bytes failed: {err}")); + .unwrap_or_else(|err| panic!("KMV to_asapv1 failed: {err}")); assert!( !encoded.is_empty(), "KMV serialization output should not be empty" ); let mut decoded: Kmv = Kmv::from_asapv1(&encoded) - .unwrap_or_else(|err| panic!("KMV deserialize_from_bytes failed: {err}")); + .unwrap_or_else(|err| panic!("KMV from_asapv1 failed: {err}")); let reencoded = decoded .to_asapv1() diff --git a/src/sketches/microscope.rs b/src/sketches/microscope.rs index 8c459c13..833703bd 100644 --- a/src/sketches/microscope.rs +++ b/src/sketches/microscope.rs @@ -232,7 +232,7 @@ impl SubWindowClock { *seen / *items_per_sub_window != before } Self::TimeBased { .. } => { - panic!("insert() drives a count-based clock; use insert_at() for a time-based one") + panic!("update() drives a count-based clock; use update_at() for a time-based one") } } } @@ -253,7 +253,7 @@ impl SubWindowClock { now.saturating_sub(*epoch) / *sub_window_len - before } Self::CountBased { .. } => { - panic!("insert_at() drives a time-based clock; use insert() for a count-based one") + panic!("update_at() drives a time-based clock; use update() for a count-based one") } } } @@ -306,7 +306,7 @@ impl<'de, H: SketchHasher> Deserialize<'de> for MicroCm { let layout = MicroLayout::new(params); if cells.bucket_len() != layout.depth() { return Err(serde::de::Error::custom(format!( - "cell depth {} does not match the {} bytes the layout for t={}, c={} needs", + "cell bucket_len {} does not match the {} bytes the layout for t={}, c={} needs", cells.bucket_len(), layout.depth(), params.t, @@ -911,7 +911,7 @@ mod tests { } #[test] - #[should_panic(expected = "use insert_at()")] + #[should_panic(expected = "use update_at()")] fn insert_refuses_a_time_based_clock() { let mut sk: MicroCm = MicroCm::with_dimensions( 1, @@ -924,7 +924,7 @@ mod tests { } #[test] - #[should_panic(expected = "use insert()")] + #[should_panic(expected = "use update()")] fn insert_at_refuses_a_count_based_clock() { let mut sk = sketch(1, 16, 4, 10); sk.update_at(&key(1), 5); @@ -1111,7 +1111,7 @@ mod tests { "zoom base c must be in", ), ( - "depth the layout disagrees with", + "bucket_len the layout disagrees with", attempt(cells_for(2, 16, depth + 4), good, clock), "does not match", ), diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 53969315..5e2d07cc 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -1052,7 +1052,7 @@ fn kll_dynamic_f64_k200_matches_golden() { let sketch = kll_dynamic_of(&KLL_DYN_F64_VALS); let got = sketch.to_asapv1().expect("serialize"); - assert_eq!(got, want, "KLLDynamic f64 bytes diverge from golden"); + assert_eq!(got, want, "KllDynamic f64 bytes diverge from golden"); let decoded = KllDynamic::::from_asapv1(&want).expect("decode"); assert_eq!(decoded.count(), KLL_DYN_F64_VALS.len()); @@ -1073,7 +1073,7 @@ fn kll_dynamic_i64_k200_matches_golden() { let sketch = kll_dynamic_of(&KLL_DYN_I64_VALS); let got = sketch.to_asapv1().expect("serialize"); - assert_eq!(got, want, "KLLDynamic i64 bytes diverge from golden"); + assert_eq!(got, want, "KllDynamic i64 bytes diverge from golden"); let decoded = KllDynamic::::from_asapv1(&want).expect("decode"); assert_eq!(decoded.count(), KLL_DYN_I64_VALS.len()); @@ -1549,7 +1549,7 @@ fn count_l2hh_2x4_seed7_matches_golden() { let sketch: CountSketchWithL2Norm = rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state"); let got = sketch.to_asapv1().expect("serialize"); - assert_eq!(got, want, "CountL2HH bytes diverge from golden"); + assert_eq!(got, want, "CountSketchWithL2Norm bytes diverge from golden"); let decoded: CountSketchWithL2Norm = CountSketchWithL2Norm::from_asapv1(&want).expect("decode"); assert_eq!(decoded.as_storage().as_slice(), flat.as_slice()); diff --git a/tests/conformance_kit.rs b/tests/conformance_kit.rs index 397abe88..17816dad 100644 --- a/tests/conformance_kit.rs +++ b/tests/conformance_kit.rs @@ -234,7 +234,7 @@ fn countmin_passes_frequency_and_merge_conformance() { }; conformance::frequency_battery( - "CountMin", + "CountMinSketch", || { CountMinAdapter(CountMinSketch::, FastPath>::with_dimensions( 4, 4096, @@ -247,7 +247,7 @@ fn countmin_passes_frequency_and_merge_conformance() { .assert_ok(); conformance::merge_equivalence_battery( - "CountMin", + "CountMinSketch", || { CountMinAdapter(CountMinSketch::, FastPath>::with_dimensions( 4, 4096, @@ -274,9 +274,9 @@ fn countsketch_passes_signed_frequency_conformance() { ) }; - conformance::frequency_battery("Count", make, &stream, &truth, spec).assert_ok(); - conformance::turnstile_battery("Count", make, 42i64).assert_ok(); - conformance::merge_equivalence_battery("Count", make, &stream, spec).assert_ok(); + conformance::frequency_battery("CountSketch", make, &stream, &truth, spec).assert_ok(); + conformance::turnstile_battery("CountSketch", make, 42i64).assert_ok(); + conformance::merge_equivalence_battery("CountSketch", make, &stream, spec).assert_ok(); } #[test] @@ -297,7 +297,7 @@ fn hll_variants_pass_cardinality_conformance() { .assert_ok(); conformance::cardinality_battery( - "HyperLogLog", + "HyperLogLog", || HllErtlAdapter(HyperLogLog::::new()), &unique, 100_000, @@ -306,7 +306,7 @@ fn hll_variants_pass_cardinality_conformance() { .assert_ok(); conformance::cardinality_battery( - "HyperLogLogHIP", + "HyperLogLogHip", || HllHipAdapter(HyperLogLogHip::new()), &unique, 100_000, @@ -349,7 +349,7 @@ fn kll_family_passes_quantile_conformance() { .assert_ok(); conformance::quantile_battery( - "KLLDynamic", + "KllDynamic", || KllDynamicAdapter(KllDynamic::::with_k_and_seed(200, KIT_KLL_SEED)), &values, QuantileSpec::default(), @@ -663,10 +663,21 @@ fn heap_backed_frequency_sketches_pass_frequency_and_merge_conformance() { 4, 4_096, 64, )) }; - conformance::frequency_battery("CMSHeap", make_cms, &stream, &truth, one_sided) - .assert_ok(); - conformance::merge_equivalence_battery("CMSHeap", make_cms, &stream, one_sided) - .assert_ok(); + conformance::frequency_battery( + "CountMinSketchWithHeap", + make_cms, + &stream, + &truth, + one_sided, + ) + .assert_ok(); + conformance::merge_equivalence_battery( + "CountMinSketchWithHeap", + make_cms, + &stream, + one_sided, + ) + .assert_ok(); let two_sided = conformance::FrequencySpec { one_sided: false, @@ -678,10 +689,21 @@ fn heap_backed_frequency_sketches_pass_frequency_and_merge_conformance() { 5, 4_096, 64, )) }; - conformance::frequency_battery("CSHeap", make_cs, &stream, &truth, two_sided) - .assert_ok(); - conformance::merge_equivalence_battery("CSHeap", make_cs, &stream, two_sided) - .assert_ok(); + conformance::frequency_battery( + "CountSketchWithHeap", + make_cs, + &stream, + &truth, + two_sided, + ) + .assert_ok(); + conformance::merge_equivalence_battery( + "CountSketchWithHeap", + make_cs, + &stream, + two_sided, + ) + .assert_ok(); } #[test] @@ -695,9 +717,11 @@ fn count_l2hh_passes_signed_frequency_and_merge_conformance() { }; let make = || CountL2hhAdapter(CountSketchWithL2Norm::with_dimensions(5, 4_096)); - conformance::frequency_battery("CountL2HH", make, &stream, &truth, spec).assert_ok(); - conformance::turnstile_battery("CountL2HH", make, 42i64).assert_ok(); - conformance::merge_equivalence_battery("CountL2HH", make, &stream, spec).assert_ok(); + conformance::frequency_battery("CountSketchWithL2Norm", make, &stream, &truth, spec) + .assert_ok(); + conformance::turnstile_battery("CountSketchWithL2Norm", make, 42i64).assert_ok(); + conformance::merge_equivalence_battery("CountSketchWithL2Norm", make, &stream, spec) + .assert_ok(); } #[test] @@ -711,8 +735,10 @@ fn folded_frequency_sketches_pass_frequency_and_merge_conformance() { abs_tol: 12.0, }; let make_cms = || FoldCmsAdapter(CountMinSketchFold::new(4, 8_192, 1, 64)); - conformance::frequency_battery("FoldCMS", make_cms, &stream, &truth, one_sided).assert_ok(); - conformance::merge_equivalence_battery("FoldCMS", make_cms, &stream, one_sided).assert_ok(); + conformance::frequency_battery("CountMinSketchFold", make_cms, &stream, &truth, one_sided) + .assert_ok(); + conformance::merge_equivalence_battery("CountMinSketchFold", make_cms, &stream, one_sided) + .assert_ok(); let two_sided = conformance::FrequencySpec { one_sided: false, @@ -720,9 +746,11 @@ fn folded_frequency_sketches_pass_frequency_and_merge_conformance() { abs_tol: 30.0, }; let make_cs = || FoldCsAdapter(CountSketchFold::new(5, 8_192, 1, 64)); - conformance::frequency_battery("FoldCS", make_cs, &stream, &truth, two_sided).assert_ok(); - conformance::turnstile_battery("FoldCS", make_cs, 42i64).assert_ok(); - conformance::merge_equivalence_battery("FoldCS", make_cs, &stream, two_sided).assert_ok(); + conformance::frequency_battery("CountSketchFold", make_cs, &stream, &truth, two_sided) + .assert_ok(); + conformance::turnstile_battery("CountSketchFold", make_cs, 42i64).assert_ok(); + conformance::merge_equivalence_battery("CountSketchFold", make_cs, &stream, two_sided) + .assert_ok(); } #[cfg(feature = "experimental")] @@ -852,9 +880,9 @@ fn micro_cm_passes_frequency_and_merge_conformance() { not the full per-key count" ); - conformance::frequency_battery("MicroCM", MicroCmAdapter::new, &stream, &truth, spec) + conformance::frequency_battery("MicroCm", MicroCmAdapter::new, &stream, &truth, spec) .assert_ok(); - conformance::merge_equivalence_battery("MicroCM", MicroCmAdapter::new, &stream, spec) + conformance::merge_equivalence_battery("MicroCm", MicroCmAdapter::new, &stream, spec) .assert_ok(); } diff --git a/tests/e2e/cardinality.rs b/tests/e2e/cardinality.rs index ed9e17e5..8ba1610e 100644 --- a/tests/e2e/cardinality.rs +++ b/tests/e2e/cardinality.rs @@ -756,12 +756,12 @@ fn a_custom_precision_merge_reproduces_the_single_pass_registers_for_every_estim assert_eq!( ertl_even.registers_as_slice(), ertl_single.registers_as_slice(), - "ErtlMLE p13 shard merge must reproduce the single pass register for register" + "Ertl p13 shard merge must reproduce the single pass register for register" ); assert_eq!( ertl_even.estimate(), ertl_single.estimate(), - "ErtlMLE p13 identical registers must give an identical estimate" + "Ertl p13 identical registers must give an identical estimate" ); } @@ -1193,7 +1193,7 @@ mod documented_matrix { for (label, spec, estimate) in [ ("Classic", register_spec, classic.estimate() as f64), - ("ErtlMLE", register_spec, ertl.estimate() as f64), + ("Ertl", register_spec, ertl.estimate() as f64), ("HIP", hip_spec, hip.estimate() as f64), ] { if let Err(detail) = spec.check(estimate, distinct) { diff --git a/tests/e2e/composition.rs b/tests/e2e/composition.rs index 84e13543..84fd6990 100644 --- a/tests/e2e/composition.rs +++ b/tests/e2e/composition.rs @@ -85,7 +85,7 @@ fn ensemble_members_match_standalone_sketches_fed_the_same_stream() { let context = format!( "rows={ROWS} cols={COLS} zipf(1.1) domain={DOMAIN} n={N} seed={STREAM_SEED:#x}, \ - members: CountMinFast, CountFast, HllErtl, HllClassic, HllHip" + members: CountMinFast, CountSketchFast, HllErtl, HllClassic, HllHip" ); // Matrix members: identical estimates, key for key. @@ -97,12 +97,12 @@ fn ensemble_members_match_standalone_sketches_fed_the_same_stream() { let b = ref_cm.estimate(&d) as f64; cm_tally.record(a == b, || format!("key {k}: ensemble {a} standalone {b}")); - let c = ens.estimate(1, &d).expect("CountFast cell"); + let c = ens.estimate(1, &d).expect("CountSketchFast cell"); let e = ref_cs.estimate(&d); cs_tally.record(c == e, || format!("key {k}: ensemble {c} standalone {e}")); } cm_tally.assert_none("ensemble CountMinFast vs standalone", &context); - cs_tally.assert_none("ensemble CountFast vs standalone", &context); + cs_tally.assert_none("ensemble CountSketchFast vs standalone", &context); // HLL members are deliberately *not* bit-identical to a standalone HLL. // The ensemble feeds them the low 64 bits of the shared **matrix** hash @@ -201,7 +201,7 @@ fn ensemble_members_stay_inside_their_own_error_models() { &context, ); CountSketchSpec::new(ROWS, COLS).assert_contract( - "ensemble CountFast", + "ensemble CountSketchFast", &truth, |k| ens.estimate(1, &DataInput::U64(k as u64)).expect("cs"), &context, @@ -288,7 +288,7 @@ fn ensemble_composes_by_hash_layout_and_rejects_incompatible_members() { &context, ); CountSketchSpec::new(ROWS, COLS).assert_contract( - "ensemble Count member", + "ensemble CountSketch member", &truth, |k| mixed.estimate(2, &DataInput::U64(k as u64)).expect("cs"), &context, @@ -387,14 +387,14 @@ fn univmonq_configuration_variants_all_build_and_keep_exact_aggregates() { }, ), ( - "width_halving_period=2", + "cols_halving_period=2", UnivMonQConfig { cols_halving_period: 2, ..base }, ), ( - "explicit hash_seed", + "explicit seed_index", UnivMonQConfig { seed_index: 3, ..base @@ -484,7 +484,7 @@ fn univmonq_with_window_bound_chooses_a_hierarchy_that_satisfies_its_own_inequal let upper = mean + (2.0 * mean * log_inv_delta).sqrt() + (2.0 / 3.0) * log_inv_delta; assert!( upper < cfg.heap_size as f64, - "levels={} leaves the deepest stratum with an upper bound of {upper:.1}, \ + "layers={} leaves the deepest stratum with an upper bound of {upper:.1}, \ above the {} candidate slots", cfg.layers, cfg.heap_size @@ -497,7 +497,7 @@ fn univmonq_with_window_bound_chooses_a_hierarchy_that_satisfies_its_own_inequal + (2.0 / 3.0) * log_inv_delta; assert!( upper_lower >= cfg.heap_size as f64, - "levels={} is not minimal: {} levels would already fit", + "layers={} is not minimal: {} layers would already fit", cfg.layers, cfg.layers - 1 ); diff --git a/tests/e2e/data_input.rs b/tests/e2e/data_input.rs index 37a13bda..ab032d3f 100644 --- a/tests/e2e/data_input.rs +++ b/tests/e2e/data_input.rs @@ -12,13 +12,13 @@ fn every_variant() -> Vec<(&'static str, DataInput<'static>)> { ("I32", DataInput::I32(-32)), ("I64", DataInput::I64(-64)), ("I128", DataInput::I128(-128)), - ("ISIZE", DataInput::Isize(-1_000)), + ("Isize", DataInput::Isize(-1_000)), ("U8", DataInput::U8(8)), ("U16", DataInput::U16(16)), ("U32", DataInput::U32(32)), ("U64", DataInput::U64(64)), ("U128", DataInput::U128(128)), - ("USIZE", DataInput::Usize(1_000)), + ("Usize", DataInput::Usize(1_000)), ("F32", DataInput::F32(1.5)), ("F64", DataInput::F64(2.5)), ("Str", DataInput::Str("borrowed")), @@ -95,7 +95,7 @@ fn signed_integer_variants_sign_extend_to_one_canonical_digest() { assert_same_digest( &DataInput::Isize(probe as isize), &widened, - &format!("ISIZE({probe}) must sign-extend onto I64"), + &format!("Isize({probe}) must sign-extend onto I64"), ); } } @@ -122,7 +122,7 @@ fn unsigned_integer_variants_zero_extend_to_one_canonical_digest() { assert_same_digest( &DataInput::Usize(probe as usize), &widened, - &format!("USIZE({probe}) must zero-extend onto U64"), + &format!("Usize({probe}) must zero-extend onto U64"), ); } } @@ -350,13 +350,13 @@ fn the_numeric_variants_reach_the_quantile_sketches_and_the_others_are_refused() ("I32", DataInput::I32(3)), ("I64", DataInput::I64(4)), ("I128", DataInput::I128(5)), - ("ISIZE", DataInput::Isize(6)), + ("Isize", DataInput::Isize(6)), ("U8", DataInput::U8(7)), ("U16", DataInput::U16(8)), ("U32", DataInput::U32(9)), ("U64", DataInput::U64(10)), ("U128", DataInput::U128(11)), - ("USIZE", DataInput::Usize(12)), + ("Usize", DataInput::Usize(12)), ("F32", DataInput::F32(13.0)), ("F64", DataInput::F64(14.0)), ]; diff --git a/tests/e2e/experimental.rs b/tests/e2e/experimental.rs index 4b6208bc..9fdae8eb 100644 --- a/tests/e2e/experimental.rs +++ b/tests/e2e/experimental.rs @@ -127,7 +127,7 @@ fn kmv_estimates_stay_inside_their_relative_standard_error_band_over_independent let outcome = spec.check(estimate, n); tally.record(outcome.is_ok(), || { format!( - "k={k} seed_idx={seed_index} (trial {s}) {regime:?} n={n} \ + "k={k} seed_index={seed_index} (trial {s}) {regime:?} n={n} \ namespace={namespace:#x}: {}", outcome.unwrap_err() ) @@ -605,7 +605,7 @@ fn eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it() { assert!( eh.sketch_buckets.len() >= 2, "test premise: the map tier must have promoted at least two sketch \ - buckets (max_map_size={}, got {} sketch and {} map buckets)", + buckets (exact_bucket_budget={}, got {} sketch and {} map buckets)", eh.exact_bucket_budget, eh.sketch_buckets.len(), eh.exact_buckets.len() @@ -649,7 +649,7 @@ fn eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it() { result.estimate_l2(), l2_truth * 0.90, l2_truth * 1.10, - &format!("EHUnivOptimized sketch-tier L2 (empirical band). {context}"), + &format!("ExponentialHistogramUnivMon sketch-tier L2 (empirical band). {context}"), ); let entropy_bits = truth.entropy(true); @@ -657,7 +657,9 @@ fn eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it() { result.estimate_entropy(), entropy_bits * 0.90, entropy_bits * 1.40, - &format!("EHUnivOptimized sketch-tier entropy in bits (empirical band). {context}"), + &format!( + "ExponentialHistogramUnivMon sketch-tier entropy in bits (empirical band). {context}" + ), ); } @@ -713,8 +715,8 @@ fn eh_univ_optimized_sketch_tier_cardinality_is_documented_as_unrecoverable() { assert!( reported < distinct * 0.10, "sketch-tier cardinality reported {reported} for {distinct} distinct keys. \ - If UnivMon's F0 recovery was fixed or EHUnivOptimized stopped coupling the \ - promotion threshold to layer_size, delete this test and assert a real band \ + If UnivMon's F0 recovery was fixed or ExponentialHistogramUnivMon stopped coupling the \ + promotion threshold to layers, delete this test and assert a real band \ instead of leaving this stale documentation in place" ); } diff --git a/tests/e2e/frameworks.rs b/tests/e2e/frameworks.rs index 96aff71f..61be34c3 100644 --- a/tests/e2e/frameworks.rs +++ b/tests/e2e/frameworks.rs @@ -244,7 +244,7 @@ impl FrequencyOps for HydraCsAdapter { &[Some(key.to_string().as_str())], &HydraQuery::Frequency(DataInput::Str(MEASURE)), ) - .expect("Count counters answer frequency queries") + .expect("CountSketch counters answer frequency queries") } } @@ -1501,7 +1501,7 @@ fn univmon_weighted_metrics_and_fast_insert_parity() { assert_eq!( fast.estimate_l1(), total as f64, - "fast-insert L1 must be exact" + "terminal-update L1 must be exact" ); let l2_truth = truth.l2_norm(); @@ -1728,13 +1728,13 @@ fn hydra_query_quantile_is_the_cumulative_query_it_wraps() { .expect("cdf query"); assert_eq!( wrapper, explicit, - "x={x}: query_quantile disagreed with the query it wraps" + "x={x}: query_cdf disagreed with the query it wraps" ); assert_between( wrapper, truth.cdf(x) - kll_eps, truth.cdf(x) + kll_eps, - &format!("Hydra query_quantile at x={x}"), + &format!("Hydra query_cdf at x={x}"), ); } @@ -1915,12 +1915,12 @@ mod documented_matrix { assert_eq!( fast.estimate_l1(), l1, - "the fast-insert path's L1 must be exact. {context}" + "the terminal-update path's L1 must be exact. {context}" ); assert_eq!( pyramid.estimate_l1(), l1, - "UnivMonPyramid L1 must be exact. {context}" + "UnivMonTwoTier L1 must be exact. {context}" ); let entropy = truth.entropy(true); @@ -1942,13 +1942,13 @@ mod documented_matrix { fast.estimate_l2(), l2 * (1.0 - UM_FAST_L2), l2 * (1.0 + UM_FAST_L2), - &format!("UnivMon fast-insert L2 on ({id}); {context}"), + &format!("UnivMon terminal-update L2 on ({id}); {context}"), ); assert_between( pyramid.estimate_l2(), l2 * (1.0 - PYRAMID_L2), l2 * (1.0 + PYRAMID_L2), - &format!("UnivMonPyramid L2 on ({id}); {context}"), + &format!("UnivMonTwoTier L2 on ({id}); {context}"), ); let distinct = truth.distinct() as f64; @@ -1962,7 +1962,7 @@ mod documented_matrix { pyramid.estimate_cardinality(), distinct * (1.0 - PYRAMID_CARDINALITY), distinct * (1.0 + PYRAMID_CARDINALITY), - &format!("UnivMonPyramid cardinality on ({id}); {context}"), + &format!("UnivMonTwoTier cardinality on ({id}); {context}"), ); } diff --git a/tests/e2e/frequency.rs b/tests/e2e/frequency.rs index 75fe6a3a..7b6fc1fe 100644 --- a/tests/e2e/frequency.rs +++ b/tests/e2e/frequency.rs @@ -66,13 +66,13 @@ fn countmin_fast_path_zipf_conforms_to_the_count_min_model_and_merges_shards_exa let spec = CountMinSpec::new(ROWS, COLS); let context = format!("zipf(1.1) domain=8192 n=100000 stream_seed={STREAM_SEED}"); spec.assert_contract( - "CountMin, FastPath> single-pass", + "CountMinSketch, FastPath> single-pass", &truth, |k| single.estimate(&DataInput::I64(k)) as f64, &context, ); spec.assert_contract( - "CountMin, FastPath> 3-way shard merge", + "CountMinSketch, FastPath> 3-way shard merge", &truth, |k| shard_a.estimate(&DataInput::I64(k)) as f64, &context, @@ -88,7 +88,7 @@ fn countmin_fast_path_zipf_conforms_to_the_count_min_model_and_merges_shards_exa format!("key {k}: single-pass {a} != shard-merged {b}") }); } - merge_equality.assert_none("CountMin merge equality", &context); + merge_equality.assert_none("CountMinSketch merge equality", &context); } #[test] @@ -111,13 +111,13 @@ fn countmin_vecbased_error_bound() { let spec = CountMinSpec::new(r, c); let context = format!("zipf(1.2) domain=512 n=20000 stream_seed={STREAM_SEED}"); spec.assert_contract( - "CountMin, RegularPath>", + "CountMinSketch, RegularPath>", &truth, |k| reg.estimate(&DataInput::I64(k)) as f64, &context, ); spec.assert_contract( - "CountMin, FastPath>", + "CountMinSketch, FastPath>", &truth, |k| fast.estimate(&DataInput::I64(k)) as f64, &context, @@ -146,13 +146,13 @@ fn count_vecbased_error_bound() { let spec = CountSketchSpec::new(r, c); let context = format!("zipf(1.2) domain=512 n=20000 stream_seed={STREAM_SEED}"); spec.assert_contract( - "Count, RegularPath>", + "CountSketch, RegularPath>", &truth, |k| reg.estimate(&DataInput::I64(k)), &context, ); spec.assert_contract( - "Count, FastPath>", + "CountSketch, FastPath>", &truth, |k| fast.estimate(&DataInput::I64(k)), &context, @@ -187,13 +187,13 @@ fn countmin_matbased_error_bound() { } let spec = CountMinSpec::new($rows, $cols); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", RegularPath>"), + concat!("CountMinSketch<", stringify!($mat), ", RegularPath>"), &truth, |k| reg.estimate(&DataInput::I64(k)) as f64, &context, ); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", FastPath>"), + concat!("CountMinSketch<", stringify!($mat), ", FastPath>"), &truth, |k| fast.estimate(&DataInput::I64(k)) as f64, &context, @@ -247,13 +247,13 @@ fn count_matbased_error_bound() { } let spec = CountSketchSpec::new($rows, $cols); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", RegularPath>"), + concat!("CountMinSketch<", stringify!($mat), ", RegularPath>"), &truth, |k| reg.estimate(&DataInput::I64(k)) as f64, &context, ); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", FastPath>"), + concat!("CountMinSketch<", stringify!($mat), ", FastPath>"), &truth, |k| fast.estimate(&DataInput::I64(k)) as f64, &context, @@ -373,7 +373,7 @@ fn countmin_regular_path_satisfies_the_count_min_theorem() { RegularPath hashes each row with its own seed from the table" ); spec.assert_contract( - &format!("CountMin RegularPath {name}"), + &format!("CountMinSketch RegularPath {name}"), &truth, |k| regular.estimate(&to_input(k)) as f64, &context, @@ -410,7 +410,7 @@ fn countmin_fast_path_conforms_to_the_count_min_model() { row independence is a model, not a hypothesis the theorem supplies" ); spec.assert_contract( - &format!("CountMin FastPath {name}"), + &format!("CountMinSketch FastPath {name}"), &truth, |k| fast.estimate(&to_input(k)) as f64, &context, @@ -488,9 +488,12 @@ fn countsketch_both_paths_meet_the_l2_median_bound() { — the keys of one sketch share it and are not independent trials)", spec.kappa ); - simul.assert_none(&format!("Count {label} / simultaneous L2 bound"), &context); + simul.assert_none( + &format!("CountSketch {label} / simultaneous L2 bound"), + &context, + ); marg.assert_rate_at_most( - &format!("Count {label} / marginal L2 median bound"), + &format!("CountSketch {label} / marginal L2 median bound"), spec.marginal_failure(), &context, ); @@ -583,7 +586,7 @@ fn countl2hh_weighted_turnstile_satisfies_the_l2_median_bound() { stream_seed={STREAM_SEED} hash_seed_idx={HASH_SEED_IDX}" ); CountSketchSpec::new(ROWS, COLS).assert_contract( - "CountL2HH point estimate", + "CountSketchWithL2Norm point estimate", &truth, |k| sk.estimate(&DataInput::I64(k)), &context, @@ -594,7 +597,7 @@ fn countl2hh_weighted_turnstile_satisfies_the_l2_median_bound() { // real bound, not an exactly maintained total. let f2_spec = SecondMomentSpec::new(ROWS, COLS); if let Err(detail) = f2_spec.check(sk.l2_squared(), truth.f2()) { - panic!("CountL2HH F2: {detail}\n context: {context}"); + panic!("CountSketchWithL2Norm F2: {detail}\n context: {context}"); } } @@ -688,13 +691,13 @@ fn folded_sketches_keep_their_own_bounds_through_a_sixteen_way_merge() { ); CountMinSpec::new(ROWS, folded_cols).assert_contract( - "FoldCMS after a 16-way hierarchical merge", + "CountMinSketchFold after a 16-way hierarchical merge", &truth, |k| cms_merged.estimate(&DataInput::U64(k as u64)) as f64, &context, ); CountSketchSpec::new(ROWS, folded_cols).assert_contract( - "FoldCS after a 16-way hierarchical merge", + "CountSketchFold after a 16-way hierarchical merge", &truth, |k| cs_merged.estimate(&DataInput::U64(k as u64)) as f64, &context, @@ -780,13 +783,13 @@ fn countmin_documented_vector_grid( } let spec = CountMinSpec::new(rows, cols); spec.assert_contract( - &format!("CountMin, RegularPath> {rows}x{cols}"), + &format!("CountMinSketch, RegularPath> {rows}x{cols}"), truth, |k| regular.estimate(&input.data(k)) as f64, context, ); spec.assert_contract( - &format!("CountMin, FastPath> {rows}x{cols}"), + &format!("CountMinSketch, FastPath> {rows}x{cols}"), truth, |k| fast.estimate(&input.data(k)) as f64, context, @@ -814,13 +817,13 @@ fn countmin_documented_fixed_grid( } let spec = CountMinSpec::new($rows, $cols); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", RegularPath>"), + concat!("CountMinSketch<", stringify!($mat), ", RegularPath>"), truth, |k| regular.estimate(&input.data(k)) as f64, context, ); spec.assert_contract( - concat!("CountMin<", stringify!($mat), ", FastPath>"), + concat!("CountMinSketch<", stringify!($mat), ", FastPath>"), truth, |k| fast.estimate(&input.data(k)) as f64, context, @@ -873,13 +876,13 @@ fn countsketch_documented_vector_grid( } let spec = CountSketchSpec::new(rows, cols); spec.assert_contract( - &format!("Count, RegularPath> {rows}x{cols}"), + &format!("CountSketch, RegularPath> {rows}x{cols}"), truth, |k| regular.estimate(&input.data(k)), context, ); spec.assert_contract( - &format!("Count, FastPath> {rows}x{cols}"), + &format!("CountSketch, FastPath> {rows}x{cols}"), truth, |k| fast.estimate(&input.data(k)), context, @@ -905,13 +908,13 @@ fn countsketch_documented_fixed_grid( } let spec = CountSketchSpec::new($rows, $cols); spec.assert_contract( - concat!("Count<", stringify!($mat), ", RegularPath>"), + concat!("CountSketch<", stringify!($mat), ", RegularPath>"), truth, |k| regular.estimate(&input.data(k)), context, ); spec.assert_contract( - concat!("Count<", stringify!($mat), ", FastPath>"), + concat!("CountSketch<", stringify!($mat), ", FastPath>"), truth, |k| fast.estimate(&input.data(k)), context, @@ -1030,7 +1033,7 @@ fn countl2hh_documented_input(id: u8) { ); CountSketchSpec::new(ROWS, COLS).assert_contract( - "CountL2HH point estimate", + "CountSketchWithL2Norm point estimate", &truth, |k| sk.estimate(&input.data(k)), &context, @@ -1041,7 +1044,7 @@ fn countl2hh_documented_input(id: u8) { let hot_rel = ((hot_est - hot_truth) / hot_truth).abs(); assert!( hot_rel <= 0.02, - "CountL2HH hottest key after the decrement: est {hot_est:.1} vs true {hot_truth} \ + "CountSketchWithL2Norm hottest key after the decrement: est {hot_est:.1} vs true {hot_truth} \ is {:.3}% off, past the documented 2%. {context}", hot_rel * 100.0 ); @@ -1051,14 +1054,14 @@ fn countl2hh_documented_input(id: u8) { let f2_rel = ((f2_est - f2_truth) / f2_truth).abs(); assert!( f2_rel <= 0.10, - "CountL2HH F2: est {f2_est:.3e} vs true {f2_truth:.3e} is {:.2}% off, past the \ + "CountSketchWithL2Norm F2: est {f2_est:.3e} vs true {f2_truth:.3e} is {:.2}% off, past the \ documented 10%. {context}", f2_rel * 100.0 ); let f2_spec = SecondMomentSpec::new(ROWS, COLS); if let Err(detail) = f2_spec.check(f2_est, f2_truth) { - panic!("CountL2HH F2 model: {detail}\n context: {context}"); + panic!("CountSketchWithL2Norm F2 model: {detail}\n context: {context}"); } } diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index 7ffce85a..71a82dc7 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -1329,7 +1329,7 @@ mod partial_key_and_heavy_maintenance { == partial), "a projection predicate must reject a non-projection" ); - assert_eq!(bucket.val, 12, "add_v must accumulate"); + assert_eq!(bucket.val, 12, "add must accumulate"); } fn elephant_stream(prefix: &str, elephants: usize, weight: usize) -> Vec { @@ -1728,7 +1728,7 @@ mod documented_matrix { &spec, &truth, &heavy, - &format!("CountMin, RegularPath> {rows}x{cols}"), + &format!("CountMinSketch, RegularPath> {rows}x{cols}"), |k| regular.estimate(&input.data(k)) as f64, &context, ); @@ -1736,7 +1736,7 @@ mod documented_matrix { &spec, &truth, &heavy, - &format!("CountMin, FastPath> {rows}x{cols}"), + &format!("CountMinSketch, FastPath> {rows}x{cols}"), |k| fast.estimate(&input.data(k)) as f64, &context, ); @@ -1758,7 +1758,7 @@ mod documented_matrix { &spec, &truth, &heavy, - concat!("CountMin<", stringify!($mat), ", RegularPath>"), + concat!("CountMinSketch<", stringify!($mat), ", RegularPath>"), |k| regular.estimate(&input.data(k)) as f64, &context, ); @@ -1766,7 +1766,7 @@ mod documented_matrix { &spec, &truth, &heavy, - concat!("CountMin<", stringify!($mat), ", FastPath>"), + concat!("CountMinSketch<", stringify!($mat), ", FastPath>"), |k| fast.estimate(&input.data(k)) as f64, &context, ); @@ -1814,7 +1814,7 @@ mod documented_matrix { &spec, &truth, &heavy, - &format!("Count, RegularPath> {rows}x{cols}"), + &format!("CountSketch, RegularPath> {rows}x{cols}"), |k| regular.estimate(&input.data(k)), &context, ); @@ -1822,7 +1822,7 @@ mod documented_matrix { &spec, &truth, &heavy, - &format!("Count, FastPath> {rows}x{cols}"), + &format!("CountSketch, FastPath> {rows}x{cols}"), |k| fast.estimate(&input.data(k)), &context, ); @@ -1844,7 +1844,7 @@ mod documented_matrix { &spec, &truth, &heavy, - concat!("Count<", stringify!($mat), ", RegularPath>"), + concat!("CountSketch<", stringify!($mat), ", RegularPath>"), |k| regular.estimate(&input.data(k)), &context, ); @@ -1852,7 +1852,7 @@ mod documented_matrix { &spec, &truth, &heavy, - concat!("Count<", stringify!($mat), ", FastPath>"), + concat!("CountSketch<", stringify!($mat), ", FastPath>"), |k| fast.estimate(&input.data(k)), &context, ); diff --git a/tests/e2e/matrix_instances.rs b/tests/e2e/matrix_instances.rs index c129b644..85343943 100644 --- a/tests/e2e/matrix_instances.rs +++ b/tests/e2e/matrix_instances.rs @@ -111,7 +111,7 @@ macro_rules! countmin_instance { let (rows, cols) = (single.rows(), single.cols()); let label = concat!( - "CountMin<", + "CountMinSketch<", stringify!($storage), ", ", stringify!($path), @@ -300,7 +300,13 @@ macro_rules! count_instance { left.merge(&right); let (rows, cols) = (single.rows(), single.cols()); - let label = concat!("Count<", stringify!($storage), ", ", stringify!($path), ">"); + let label = concat!( + "CountSketch<", + stringify!($storage), + ", ", + stringify!($path), + ">" + ); let ctx = context(label, rows, cols); let spec = CountSketchSpec::new(rows, cols); spec.assert_contract( @@ -384,7 +390,7 @@ fn countsketch_both_paths_are_exact_on_a_collision_free_workload() { } } let label = concat!( - "Count<", + "CountSketch<", stringify!($storage), "> collision-free exactness on both paths" ); @@ -434,7 +440,7 @@ macro_rules! cmsheap_instance { let (rows, cols) = (single.rows(), single.cols()); let label = concat!( - "CMSHeap<", + "CountMinSketchWithHeap<", stringify!($storage), ", ", stringify!($path), @@ -501,7 +507,7 @@ macro_rules! csheap_instance { let (rows, cols) = (single.rows(), single.cols()); let label = concat!( - "CSHeap<", + "CountSketchWithHeap<", stringify!($storage), ", ", stringify!($path), @@ -685,7 +691,7 @@ fn csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { macro_rules! probe { ($storage:ty, $path:ty, $ctor:expr) => {{ let label = concat!( - "CSHeap<", + "CountSketchWithHeap<", stringify!($storage), ", ", stringify!($path), @@ -722,7 +728,7 @@ fn csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { assert_eq!( heap_count_from_estimate(est), i64::MAX, - "{label}: cs_heap_count must saturate at i64::MAX" + "{label}: heap_count_from_estimate must saturate at i64::MAX" ); assert_eq!( heap_count(huge.heap().heap(), &key), @@ -828,7 +834,7 @@ fn cmsheap_inert_instances_construct_and_report_their_geometry() { macro_rules! inert { ($storage:ty, $path:ty, $ctor:expr, $rows:expr, $cols:expr) => {{ let label = concat!( - "CMSHeap<", + "CountMinSketchWithHeap<", stringify!($storage), ", ", stringify!($path), @@ -842,7 +848,7 @@ fn cmsheap_inert_instances_construct_and_report_their_geometry() { assert_eq!( sketch.count_min_sketch().rows(), $rows, - "{label}: the wrapped CountMin reports the same geometry" + "{label}: the wrapped CountMinSketch reports the same geometry" ); }}; } @@ -1035,14 +1041,14 @@ fn countmin_holds_its_contract_across_the_depth_and_width_axis_on_both_paths() { fast.update(&DataInput::U64(*k)); } countmin_axis_contract( - "CountMin, RegularPath> axis", + "CountMinSketch, RegularPath> axis", rows, cols, &truth, |key| regular.estimate(&DataInput::U64(key as u64)) as f64, ); countmin_axis_contract( - "CountMin, FastPath> axis", + "CountMinSketch, FastPath> axis", rows, cols, &truth, @@ -1101,8 +1107,8 @@ fn countsketch_holds_its_l2_contract_across_the_depth_and_width_axis() { |key| regular.estimate(&DataInput::U64(key as u64)), ); regular_simultaneous.assert_none( - "Count, RegularPath> axis / simultaneous L2", - &context("Count, RegularPath> axis", rows, cols), + "CountSketch, RegularPath> axis / simultaneous L2", + &context("CountSketch, RegularPath> axis", rows, cols), ); let mut fast_simultaneous = Tally::default(); let mut fast_marginal = Tally::default(); @@ -1110,8 +1116,8 @@ fn countsketch_holds_its_l2_contract_across_the_depth_and_width_axis() { fast.estimate(&DataInput::U64(key as u64)) }); fast_simultaneous.assert_none( - "Count, FastPath> axis / simultaneous L2", - &context("Count, FastPath> axis", rows, cols), + "CountSketch, FastPath> axis / simultaneous L2", + &context("CountSketch, FastPath> axis", rows, cols), ); } } @@ -1134,14 +1140,14 @@ fn countmin_answers_a_non_power_of_two_width_on_both_paths() { assert_eq!(regular.cols(), cols, "regular path reported width"); assert_eq!(fast.cols(), cols, "fast path reported width"); countmin_axis_contract( - "CountMin, RegularPath> non-power-of-two", + "CountMinSketch, RegularPath> non-power-of-two", 3, cols, &truth, |key| regular.estimate(&DataInput::U64(key as u64)) as f64, ); countmin_axis_contract( - "CountMin, FastPath> non-power-of-two", + "CountMinSketch, FastPath> non-power-of-two", 3, cols, &truth, @@ -1172,8 +1178,8 @@ fn countsketch_answers_a_non_power_of_two_width_on_both_paths() { |key| regular.estimate(&DataInput::U64(key as u64)), ); regular_simultaneous.assert_none( - "Count, RegularPath> non-power-of-two / simultaneous L2", - &context("Count non-power-of-two regular", 5, cols), + "CountSketch, RegularPath> non-power-of-two / simultaneous L2", + &context("CountSketch non-power-of-two regular", 5, cols), ); let mut fast_simultaneous = Tally::default(); let mut fast_marginal = Tally::default(); @@ -1181,8 +1187,8 @@ fn countsketch_answers_a_non_power_of_two_width_on_both_paths() { fast.estimate(&DataInput::U64(key as u64)) }); fast_simultaneous.assert_none( - "Count, FastPath> non-power-of-two / simultaneous L2", - &context("Count non-power-of-two fast", 5, cols), + "CountSketch, FastPath> non-power-of-two / simultaneous L2", + &context("CountSketch non-power-of-two fast", 5, cols), ); } } @@ -1350,17 +1356,17 @@ fn countmin_precomputed_hash_entry_points_match_the_value_entry_points() { assert_eq!( by_hash.estimate(&probe), expected, - "key {key}: fast_insert_with_hash_value diverged from insert" + "key {key}: update_with_hash diverged from update" ); assert_eq!( by_bulk_hash.estimate(&probe), expected, - "key {key}: bulk_insert_with_hashes diverged from insert" + "key {key}: update_batch_with_hashes diverged from update" ); assert_eq!( by_value.estimate_with_hash(&hashed), expected, - "key {key}: fast_estimate_with_hash diverged from estimate" + "key {key}: estimate_with_hash diverged from estimate" ); } } @@ -1403,12 +1409,12 @@ fn countmin_weighted_batch_entry_points_match_a_loop_of_single_inserts() { assert_eq!( by_bulk_many.estimate(&probe), expected, - "key {key}: bulk_insert_many diverged from a loop of insert_many" + "key {key}: update_weighted_batch diverged from a loop of update_weighted" ); assert_eq!( by_hashed_many.estimate(&probe), expected, - "key {key}: bulk_insert_many_with_hashes diverged from a loop of insert_many" + "key {key}: update_weighted_batch_with_hashes diverged from a loop of update_weighted" ); } @@ -1418,7 +1424,7 @@ fn countmin_weighted_batch_entry_points_match_a_loop_of_single_inserts() { assert_eq!( single.estimate(&DataInput::U64(7)), 9, - "fast_insert_many_with_hash_value must apply the whole weight" + "update_weighted_with_hash must apply the whole weight" ); } @@ -1442,12 +1448,12 @@ fn countsketch_precomputed_hash_entry_points_match_the_value_entry_points() { assert_eq!( by_hash.estimate(&probe), expected, - "key {key}: fast_insert_with_hash_value diverged from insert" + "key {key}: update_with_hash diverged from update" ); assert_eq!( by_value.estimate_with_hash(&hashed), expected, - "key {key}: fast_estimate_with_hash diverged from estimate" + "key {key}: estimate_with_hash diverged from estimate" ); } @@ -1457,6 +1463,6 @@ fn countsketch_precomputed_hash_entry_points_match_the_value_entry_points() { assert_eq!( weighted.estimate(&DataInput::U64(11)), 6.0, - "fast_insert_many_with_hash_value must apply the whole weight" + "update_weighted_with_hash must apply the whole weight" ); } diff --git a/tests/e2e/nitro.rs b/tests/e2e/nitro.rs index a704cde5..71d39724 100644 --- a/tests/e2e/nitro.rs +++ b/tests/e2e/nitro.rs @@ -125,27 +125,27 @@ struct Path { const PATHS: [Path; 5] = [ Path { - name: "CountMin::fast_insert_nitro", + name: "CountMinSketch::nitro_update", run: row_countmin, rows: ROWS, }, Path { - name: "Count::fast_insert_nitro", + name: "CountSketch::nitro_update", run: row_countsketch, rows: ROWS, }, Path { - name: "NitroBatch::insert", + name: "NitroBatch::update_batch", run: batch_live, rows: 1, }, Path { - name: "NitroBatch::insert_cached_step", + name: "NitroBatch::update_batch_cached_step", run: batch_cached, rows: 1, }, Path { - name: "NitroBatch>::insert", + name: "NitroBatch>::update_batch", run: batch_bare, rows: 5, }, @@ -371,7 +371,7 @@ fn full_sampling_is_exact_and_the_query_reads_the_cells_the_insert_wrote() { assert_eq!( *mass, F as f64, - "{} rate=1.0: row {r} holds {mass} for {F} updates. 0 means insert and \ + "{} rate=1.0: row {r} holds {mass} for {F} updates. 0 means update and \ query disagree on the hash domain; {:.1} means the path divided the \ work across its {} rows", path.name, @@ -390,12 +390,12 @@ fn full_sampling_is_exact_and_the_query_reads_the_cells_the_insert_wrote() { assert_eq!( cm.nitro_estimate(&key), F as f64, - "CountMin::nitro_estimate must return the exact count at rate 1.0" + "CountMinSketch::nitro_estimate must return the exact count at rate 1.0" ); for (name, run) in [ - ("NitroBatch::insert", false), - ("NitroBatch::insert_cached_step", true), + ("NitroBatch::update_batch", false), + ("NitroBatch::update_batch_cached_step", true), ] .map(|(n, cached)| (n, cached)) { @@ -479,8 +479,11 @@ fn nitro_merge_lands_in_the_combined_band() { }; for (name, merged) in [ - ("CountMin::fast_insert_nitro + CountMin::merge", row_level), - ("NitroBatch::insert + NitroBatch::merge", batch), + ( + "CountMinSketch::nitro_update + CountMinSketch::merge", + row_level, + ), + ("NitroBatch::update_batch + NitroBatch::merge", batch), ] { for &rate in &RATES { let spec = SamplingConfidenceSpec::new(rate, Z); @@ -552,7 +555,7 @@ fn row_level_countmin_nitro_tracks_a_zipf_stream_within_the_combined_band() { let collision_budget = |f: f64| cm_spec.simultaneous_bound(total, f, distinct, SIMULTANEOUS_LEVEL); let context = format!( - "row-level CountMin Nitro over zipf(1.1) domain={DOMAIN} n={N} \ + "row-level CountMinSketch Nitro over zipf(1.1) domain={DOMAIN} n={N} \ seed={STREAM_SEED:#x}, rate={rate}, rows={ROWS} cols={COLS}" ); @@ -576,11 +579,11 @@ fn row_level_countmin_nitro_tracks_a_zipf_stream_within_the_combined_band() { // Both halves are union-bounded over the probed keys, so neither // tolerates a violation. low.assert_none( - &format!("row-level CountMin Nitro rate={rate} / lower band"), + &format!("row-level CountMinSketch Nitro rate={rate} / lower band"), &context, ); high.assert_none( - &format!("row-level CountMin Nitro rate={rate} / upper band"), + &format!("row-level CountMinSketch Nitro rate={rate} / upper band"), &context, ); } @@ -619,7 +622,7 @@ fn row_level_countmin_nitro_separates_keys_of_similar_frequency() { }); } tally.assert_none( - &format!("row-level CountMin Nitro rate={rate} / {KEYS} equal-frequency keys"), + &format!("row-level CountMinSketch Nitro rate={rate} / {KEYS} equal-frequency keys"), &format!("each key inserted {PER_KEY} times, rows={ROWS} cols={COLS}"), ); } @@ -677,8 +680,8 @@ const CONTINUATION_RATES: [f64; 2] = [0.3, 0.07]; fn round_trip( cm: &CountMinSketch, FastPath>, ) -> CountMinSketch, FastPath> { - let bytes = rmp_serde::to_vec_named(cm).expect("serialize a Nitro-enabled CountMin"); - rmp_serde::from_slice(&bytes).expect("deserialize a Nitro-enabled CountMin") + let bytes = rmp_serde::to_vec_named(cm).expect("serialize a Nitro-enabled CountMinSketch"); + rmp_serde::from_slice(&bytes).expect("deserialize a Nitro-enabled CountMinSketch") } fn nitro_enabled(rate: f64, seed: u64) -> CountMinSketch, FastPath> { @@ -935,7 +938,7 @@ fn count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes() assert_eq!( counters(sampled.as_storage()), counters(plain.as_storage()), - "rate=1 Nitro insertion must write the same counters as insert" + "rate=1 Nitro insertion must write the same counters as update" ); for (key, count) in truth.pairs() { let probe = DataInput::U64(key as u64); @@ -984,7 +987,7 @@ fn count_min_after_disable_nitro_leaves_the_sampled_insert_path_inert() { assert_ne!( counters(sketch.as_storage()), before, - "disabling sampling must not disable the ordinary insert path" + "disabling sampling must not disable the ordinary update path" ); } @@ -1050,7 +1053,7 @@ fn count_sketch_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes assert_eq!( counters(sampled.as_storage()), counters(plain.as_storage()), - "rate=1 Nitro insertion must write the same signed counters as insert" + "rate=1 Nitro insertion must write the same signed counters as update" ); for (key, _) in truth.pairs() { let probe = DataInput::U64(key as u64); @@ -1080,6 +1083,6 @@ fn count_sketch_seeded_nitro_sampling_is_reproducible() { assert_eq!( counters(build(0x5A11_0002).as_storage()), counters(build(0x5A11_0002).as_storage()), - "two Count sketches at the same rate and seed must admit the same subset" + "two Count Sketches at the same rate and seed must admit the same subset" ); } diff --git a/tests/e2e/numeric_types.rs b/tests/e2e/numeric_types.rs index d51fab3e..a55a1dd5 100644 --- a/tests/e2e/numeric_types.rs +++ b/tests/e2e/numeric_types.rs @@ -102,7 +102,7 @@ macro_rules! kll_type_case { } s }); - trial!(("KLLDynamic", "single pass"), |seed| { + trial!(("KllDynamic", "single pass"), |seed| { let mut s = KllDynamic::<$ty>::with_k_and_seed(K, seed); for v in &values { s.update(v); @@ -124,7 +124,7 @@ macro_rules! kll_type_case { left.merge(&right); left }); - trial!(("KLLDynamic", "two-shard merge"), |seed: u64| { + trial!(("KllDynamic", "two-shard merge"), |seed: u64| { let mut left = KllDynamic::<$ty>::with_k_and_seed(K, seed); let mut right = KllDynamic::<$ty>::with_k_and_seed(K, seed ^ 0x5555); for (i, v) in values.iter().enumerate() { @@ -144,7 +144,7 @@ macro_rules! kll_type_case { s.update_batch(&values); s }); - trial!(("KLLDynamic", "bulk update"), |seed| { + trial!(("KllDynamic", "bulk update"), |seed| { let mut s = KllDynamic::<$ty>::with_k_and_seed(K, seed); s.update_batch(&values); s @@ -179,7 +179,7 @@ fn every_numeric_type_satisfies_the_kll_rank_error_characterization() { kll_type_case!(tally, trial, f64, 1_000_000_000_000_000); tally.assert_independent_binomial( - "KLL / KLLDynamic across every built-in NumericalValue type", + "Kll / KllDynamic across every built-in NumericalValue type", KllRankSpec::datasketches(K as usize).trial_failure_probability, &format!( "one trial = one sketch with its own compaction seed, scored on its worst \ @@ -215,7 +215,7 @@ fn signed_numeric_types_order_negative_values_correctly_in_kll() { } spec.record_trial( &mut $tally, - &format!("KLL<{type_name}> mixed-sign seed={seed:#x}"), + &format!("Kll<{type_name}> mixed-sign seed={seed:#x}"), truth.sorted(), &QS, |q| fixed.quantile(q), @@ -229,7 +229,7 @@ fn signed_numeric_types_order_negative_values_correctly_in_kll() { } spec.record_trial( &mut $tally, - &format!("KLLDynamic<{type_name}> mixed-sign seed={seed:#x}"), + &format!("KllDynamic<{type_name}> mixed-sign seed={seed:#x}"), truth.sorted(), &QS, |q| dynamic.quantile(q), @@ -350,7 +350,7 @@ fn every_numeric_type_satisfies_the_ddsketch_relative_value_error_contract() { // Deterministic guarantee: no hashing, no sampling, so no violations // are tolerated at any alpha. tally.assert_none( - "DDSketch::add across every built-in NumericalValue type", + "DdSketch::add across every built-in NumericalValue type", &format!("alpha={alpha} stream_seed={STREAM_SEED:#x} n={N} q grid {QS:?}"), ); } @@ -425,14 +425,14 @@ fn the_f64_projection_is_exact_below_two_to_the_53() { } spec.record_trial( &mut rank_tally, - &format!("KLL above 2^70 seed={seed:#x}"), + &format!("Kll above 2^70 seed={seed:#x}"), truth.sorted(), &QS, |q| kll.quantile(q), ); } rank_tally.assert_independent_binomial( - "KLL over values above 2^70", + "Kll over values above 2^70", spec.trial_failure_probability, &format!("k={K}, 16 seeds from kll_trial_seed(0x2E70_0000..)"), ); diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index 28f6eabc..b39753e8 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -383,7 +383,7 @@ fn count_deltas_carry_exactly_the_signed_threshold() { assert_eq!( d.count.unsigned_abs(), OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, - "Count resets on promotion, so a delta is always ±the mask" + "CountSketch resets on promotion, so a delta is always ±the mask" ); assert!((d.row as usize) < ROWS, "row {} out of range", d.row); assert!((d.col as usize) < COLS, "col {} out of range", d.col); @@ -608,9 +608,9 @@ fn hll_promotion_is_exact_for_every_variant_and_precision() { check!(HyperLogLogP12, "P12/Classic"); check!(HyperLogLog, "P14/Classic"); check!(HyperLogLogP16, "P16/Classic"); - check!(HyperLogLogP12, "P12/ErtlMLE"); - check!(HyperLogLog, "P14/ErtlMLE"); - check!(HyperLogLogP16, "P16/ErtlMLE"); + check!(HyperLogLogP12, "P12/Ertl"); + check!(HyperLogLog, "P14/Ertl"); + check!(HyperLogLogP16, "P16/Ertl"); } #[test] @@ -1230,7 +1230,7 @@ mod runtime { } #[test] - #[should_panic(expected = "cannot insert after runtime has been closed")] + #[should_panic(expected = "cannot update after runtime has been closed")] fn inserting_after_close_panics() { let workers = 2; let mut runtime = OctoRuntime::new(&config(workers), SumPlan, || SumAggregator { @@ -2183,7 +2183,7 @@ fn count_sketch_online_accuracy_beats_sketch_merge() { parity_counters: parity.sent_counters, ten_x_counters: ten_x.sent_counters, }; - comparison.report("Count"); + comparison.report("CountSketch"); comparison.assert_baselines_were_funded(); assert!(comparison.octo < comparison.merge_at_parity / 2.0); assert!(comparison.octo < comparison.merge_at_ten_x); @@ -5056,7 +5056,7 @@ mod partition_accuracy { assert_eq!( parent.sketch.estimate_l1(), delivered as f64, - "the parent's L1 is restored from the last weight_total each worker \ + "the parent's L1 is restored from the last total_weight each worker \ stamped on a delta, so it must equal their sum exactly. {ctx}" ); diff --git a/tests/e2e/quantiles.rs b/tests/e2e/quantiles.rs index ab5050f9..88c08e0c 100644 --- a/tests/e2e/quantiles.rs +++ b/tests/e2e/quantiles.rs @@ -262,7 +262,7 @@ fn kll_family_stays_within_the_datasketches_maximum_rank_error_characterization( trial += 1; if let Some(dynamic) = feed_kll_dynamic(feed, k, seed, &values) { spec.record_trial( - tallies.entry(format!("KLLDynamic/{feed:?}")).or_default(), + tallies.entry(format!("KllDynamic/{feed:?}")).or_default(), &format!("k={k} shape={shape} seed={seed:#x} n={N}"), truth.sorted(), &RANK_QS, @@ -473,16 +473,16 @@ fn kll_bulk_update_on_a_degenerate_slice_matches_the_loop() { let bytes = sk.to_asapv1().unwrap(); sk.update_batch(&[]); - assert_eq!(sk.count(), count, "an empty bulk_update changed the count"); + assert_eq!(sk.count(), count, "an empty update_batch changed the count"); assert_eq!( sk.quantile(0.5).to_bits(), median.to_bits(), - "an empty bulk_update moved the median" + "an empty update_batch moved the median" ); assert_eq!( sk.to_asapv1().unwrap(), bytes, - "an empty bulk_update changed the serialized state" + "an empty update_batch changed the serialized state" ); let mut one: Kll = Kll::with_k_and_seed(200, seed); @@ -492,7 +492,7 @@ fn kll_bulk_update_on_a_degenerate_slice_matches_the_loop() { assert_eq!( one.to_asapv1().unwrap(), once.to_asapv1().unwrap(), - "a one-element bulk_update diverged from one update" + "a one-element update_batch diverged from one update" ); } @@ -1022,7 +1022,7 @@ fn univmonq_frequency_and_f2_satisfy_the_count_sketch_bounds() { } let context = format!( - "UnivMonQConfig {{ levels: {}, width: {}, depth: {}, hash_seed: {} }}, \ + "UnivMonQConfig {{ layers: {}, top_layer_cols: {}, rows: {}, seed_index: {} }}, \ 40 separated heavy keys + uniform background (stream seed 3008)", config.layers, config.top_layer_cols, config.rows, config.seed_index ); @@ -1171,7 +1171,7 @@ fn univmonq_ordered_queries_satisfy_the_documented_cdf_and_rank_bounds() { let eta = 2.0 * e_h + p_hat_r * eps_r; let context = format!( - "{regime} trial {t}: hash_seed={} source_id={:#x}, n={n}, gate \ + "{regime} trial {t}: seed_index={} source_id={:#x}, n={n}, gate \ F2_hat/N^2={gate:.3e} vs 1/ordered_samples={threshold:.3e}, heavy set \ {} values, E_H={e_h:.6} (sum form), P_hat_R={p_hat_r:.6}, m_R={m_r}, \ epsilon_R={eps_r:.6} at delta={DELTA} -> eta {eta:.6}; \ @@ -2317,10 +2317,10 @@ mod documented_matrix { ); let dynamic = feed_kll_dynamic(feed, k, seed, &values) - .expect("both feeds are supported by KLLDynamic"); + .expect("both feeds are supported by KllDynamic"); spec.record_trial( &mut tally, - &format!("KLLDynamic {label}"), + &format!("KllDynamic {label}"), truth.sorted(), &DOC_QS, |q| dynamic.quantile(q), diff --git a/tests/e2e/topk.rs b/tests/e2e/topk.rs index 5c384c08..8ab7d1b0 100644 --- a/tests/e2e/topk.rs +++ b/tests/e2e/topk.rs @@ -36,13 +36,13 @@ fn heaps_satisfy_their_own_bounds_and_stay_heap_consistent() { // Each sketch against its own theorem. CountMinSpec::new(CM_ROWS, COLS).assert_contract( - "CMSHeap point estimate", + "CountMinSketchWithHeap point estimate", &truth, |k| cms_heap.estimate(&DataInput::I64(k)) as f64, &context, ); CountSketchSpec::new(CS_ROWS, COLS).assert_contract( - "CSHeap point estimate", + "CountSketchWithHeap point estimate", &truth, |k| cs_heap.estimate(&DataInput::I64(k)), &context, @@ -52,8 +52,12 @@ fn heaps_satisfy_their_own_bounds_and_stay_heap_consistent() { // the sketch is a bug regardless of any error bound. let kth_count = truth.top_k(TOP_K)[TOP_K - 1].1; for (label, items, cms) in [ - ("CMSHeap", cms_heap.heap().heap().to_vec(), true), - ("CSHeap", cs_heap.heap().heap().to_vec(), false), + ( + "CountMinSketchWithHeap", + cms_heap.heap().heap().to_vec(), + true, + ), + ("CountSketchWithHeap", cs_heap.heap().heap().to_vec(), false), ] { assert!(items.len() <= TOP_K, "{label} heap exceeded capacity"); let mut consistency = Tally::default(); @@ -150,8 +154,12 @@ mod documented_matrix { let kth_count = truth.top_k(top_k)[top_k - 1].1; for (label, items, cms) in [ - ("CMSHeap", cms_heap.heap().heap().to_vec(), true), - ("CSHeap", cs_heap.heap().heap().to_vec(), false), + ( + "CountMinSketchWithHeap", + cms_heap.heap().heap().to_vec(), + true, + ), + ("CountSketchWithHeap", cs_heap.heap().heap().to_vec(), false), ] { assert!( items.len() <= top_k, @@ -211,7 +219,10 @@ mod documented_matrix { ) }); } - bound.assert_none("CMSHeap / Count-Min bound at the heap's keys", &context); + bound.assert_none( + "CountMinSketchWithHeap / Count-Min bound at the heap's keys", + &context, + ); } else { let spec = CountSketchSpec::new(HEAP_ROWS, HEAP_COLS); let f2 = truth.f2(); @@ -229,7 +240,10 @@ mod documented_matrix { ) }); } - bound.assert_none("CSHeap / L2 bound at the heap's keys", &context); + bound.assert_none( + "CountSketchWithHeap / L2 bound at the heap's keys", + &context, + ); } consistency.assert_none(&format!("{label} heap/sketch consistency"), &context); diff --git a/tests/e2e/windows.rs b/tests/e2e/windows.rs index d1c88ec4..99cf19ac 100644 --- a/tests/e2e/windows.rs +++ b/tests/e2e/windows.rs @@ -74,13 +74,13 @@ fn full_span(eh: &ExponentialHistogram) -> (u64, u64) { fn every_eh_variant_selects_the_documented_merge_norm() { let l2_variants = [ ( - "COUNTL2HH", + "CountSketchWithL2Norm", asap_sketchlib::EhBucketSketch::CountSketchWithL2Norm( CountSketchWithL2Norm::with_dimensions(ROWS, COLS), ), ), ( - "UNIVMON", + "UnivMon", asap_sketchlib::EhBucketSketch::UnivMon(UnivMon::new(32, ROWS, COLS, 4)), ), ]; @@ -99,37 +99,37 @@ fn every_eh_variant_selects_the_documented_merge_norm() { let l1_variants: Vec<(&str, asap_sketchlib::EhBucketSketch)> = vec![ ( - "CM", + "CountMinSketch", asap_sketchlib::EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(ROWS, COLS), ), ), ( - "CS", + "CountSketch", asap_sketchlib::EhBucketSketch::CountSketch( CountSketch::, FastPath>::with_dimensions(ROWS, COLS), ), ), ( - "COCO", + "CocoSketch", asap_sketchlib::EhBucketSketch::CocoSketch(CocoSketch::init_with_size_and_seed( 512, 4, COCO_SEED, )), ), ( - "ELASTIC", + "ElasticSketch", asap_sketchlib::EhBucketSketch::ElasticSketch(ElasticSketch::with_heavy_buckets(512)), ), ( - "HLL", + "Hll", asap_sketchlib::EhBucketSketch::Hll(HyperLogLog::::new()), ), ( - "KLL", + "Kll", asap_sketchlib::EhBucketSketch::Kll(Kll::with_k_and_seed(200, 0x5EED_0100)), ), ( - "DDS", + "DdSketch", asap_sketchlib::EhBucketSketch::DdSketch(DdSketch::new(0.01)), ), ]; @@ -181,7 +181,7 @@ fn eh_count_min_variant_conforms_to_the_count_min_model_over_the_retained_window CountMinSketch::, FastPath>::with_dimensions(ROWS, COLS), )); CountMinSpec::new(ROWS, COLS).assert_contract( - "EHSketchList::CM", + "EhBucketSketch::CountMinSketch", &truth, |k| merged.query(&DataInput::U64(k as u64)).expect("CM query"), &ctx, @@ -198,7 +198,7 @@ fn eh_count_sketch_variant_conforms_to_the_l2_model_over_the_retained_window() { ROWS, COLS ))); CountSketchSpec::new(ROWS, COLS).assert_contract( - "EHSketchList::CS", + "EhBucketSketch::CountSketch", &truth, |k| merged.query(&DataInput::U64(k as u64)).expect("CS query"), &ctx, @@ -212,9 +212,13 @@ fn eh_countl2hh_variant_satisfies_the_l2_bound_over_the_retained_window() { CountSketchWithL2Norm::with_dimensions(ROWS, COLS), )); CountSketchSpec::new(ROWS, COLS).assert_contract( - "EHSketchList::COUNTL2HH", + "EhBucketSketch::CountSketchWithL2Norm", &truth, - |k| merged.query(&DataInput::U64(k as u64)).expect("L2HH query"), + |k| { + merged + .query(&DataInput::U64(k as u64)) + .expect("CountSketchWithL2Norm query") + }, &ctx, ); } @@ -259,7 +263,10 @@ fn eh_elastic_variant_stays_one_sided_over_the_retained_window() { format!("key f{k}: true {c}, reported {est} (must never read low)") }); } - tally.assert_none("EHSketchList::ELASTIC one-sided on heavy keys", &ctx); + tally.assert_none( + "EhBucketSketch::ElasticSketch one-sided on heavy keys", + &ctx, + ); } /// The merged Coco window holds exactly one unit of mass per retained event. @@ -270,7 +277,7 @@ fn eh_coco_variant_conserves_the_retained_mass() { CocoSketch::init_with_size_and_seed(1024, 4, COCO_SEED), )); let asap_sketchlib::EhBucketSketch::CocoSketch(coco) = &merged else { - panic!("a COCO histogram merges into a COCO payload"); + panic!("a CocoSketch histogram merges into a CocoSketch payload"); }; let mass: u64 = coco.recorded_flows().map(|(_, v)| v).sum(); assert_eq!( @@ -307,7 +314,7 @@ fn eh_hll_variant_satisfies_the_register_error_model_over_the_retained_window() // acceptance rule at n = 1 is exactly "this must pass", which is the honest // reading of a one-draw experiment. tally.assert_independent_binomial( - "EHSketchList::HLL / register error model", + "EhBucketSketch::Hll / register error model", spec.per_check_failure(), &format!( "k={EH_K} uniform n={N} domain=200000 seed={STREAM_SEED:#x}, retained span \ @@ -359,14 +366,14 @@ fn eh_kll_variant_satisfies_the_rank_error_characterization_over_the_retained_wi // The KLL payload answers `query(q)` as `quantile(q)`. spec.record_trial( &mut tally, - &format!("EHSketchList::KLL seed={seed:#x} span=[{tlo}, {thi}]"), + &format!("EhBucketSketch::Kll seed={seed:#x} span=[{tlo}, {thi}]"), trial_truth.sorted(), &QS, |q| trial_merged.query(&DataInput::F64(q)).expect("KLL query"), ); } tally.assert_independent_binomial( - "EHSketchList::KLL / maximum normalized rank error per compaction seed", + "EhBucketSketch::Kll / maximum normalized rank error per compaction seed", spec.trial_failure_probability, &format!( "k={EH_K} kll_k={KLL_K} uniform n={N} seed={STREAM_SEED:#x}, retained span \ @@ -406,7 +413,7 @@ fn eh_ddsketch_variant_satisfies_the_relative_value_error_contract_over_the_wind |q| merged.query(&DataInput::F64(q)).ok(), ); tally.assert_none( - "EHSketchList::DDS / relative value error", + "EhBucketSketch::DdSketch / relative value error", &format!( "alpha={ALPHA} k={EH_K} uniform n={N} seed={STREAM_SEED:#x}, retained span \ [{lo}, {hi}] with {} observations", @@ -569,7 +576,7 @@ fn tumbling_fold_cs_windows_are_exact_and_answers_satisfy_the_l2_bound() { } let active = tw.active_sketch(); spec.assert_contract( - "TumblingWindow active window", + "TumblingWindow active window", &active_truth, |k| active.estimate(&DataInput::U64(k as u64)) as f64, &format!( @@ -585,7 +592,7 @@ fn tumbling_fold_cs_windows_are_exact_and_answers_satisfy_the_l2_bound() { } let all = tw.query_all(); spec.assert_contract( - "TumblingWindow query_all", + "TumblingWindow query_all", &all_truth, |k| all.estimate(&DataInput::U64(k as u64)) as f64, &format!("all {TOTAL} observations across {} windows", TOTAL / WINDOW), @@ -599,7 +606,7 @@ fn tumbling_fold_cs_windows_are_exact_and_answers_satisfy_the_l2_bound() { } let recent = tw.query_recent(2); spec.assert_contract( - "TumblingWindow query_recent(2)", + "TumblingWindow query_recent(2)", &recent_truth, |k| recent.estimate(&DataInput::U64(k as u64)) as f64, "last three windows", @@ -702,42 +709,42 @@ fn every_eh_variant_can_merge_into_its_own_kind() { let variants: Vec<(&str, EhBucketSketch)> = vec![ ( - "CM", + "CountMinSketch", EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(3, 256), ), ), ( - "CS", + "CountSketch", EhBucketSketch::CountSketch(CountSketch::, FastPath>::with_dimensions( 3, 256, )), ), ( - "COCO", + "CocoSketch", EhBucketSketch::CocoSketch(CocoSketch::init_with_size_and_seed(256, 4, COCO_SEED)), ), ( - "COUNTL2HH", + "CountSketchWithL2Norm", EhBucketSketch::CountSketchWithL2Norm(CountSketchWithL2Norm::with_dimensions(3, 256)), ), - ("DDS", EhBucketSketch::DdSketch(DdSketch::new(0.01))), + ("DdSketch", EhBucketSketch::DdSketch(DdSketch::new(0.01))), ( - "ELASTIC", + "ElasticSketch", EhBucketSketch::ElasticSketch(ElasticSketch::with_heavy_buckets(256)), ), - ("HLL", EhBucketSketch::Hll(HyperLogLog::::new())), + ("Hll", EhBucketSketch::Hll(HyperLogLog::::new())), ( - "KLL", + "Kll", EhBucketSketch::Kll(Kll::with_k_and_seed(200, 0x5EED_0300)), ), ( - "UNIVMON", + "UnivMon", EhBucketSketch::UnivMon(UnivMon::new(32, 3, 256, 4)), ), #[cfg(feature = "experimental")] ( - "UNIFORM", + "Uniform", EhBucketSketch::Uniform(asap_sketchlib::UniformSampling::with_seed(0.5, 7)), ), ]; @@ -754,7 +761,7 @@ fn every_eh_variant_can_merge_into_its_own_kind() { } assert!( left.merge(&right).is_ok(), - "EHSketchList::{name} has no merge arm, so an ExponentialHistogram over it \ + "EhBucketSketch::{name} has no merge arm, so an ExponentialHistogram over it \ silently discards data on every bucket merge" ); } @@ -847,7 +854,7 @@ fn tumbling_fold_cms_hierarchical_merge_covers_every_observation() { let spec = CountMinSpec::new(ROWS, hierarchical.fold_cols()); spec.assert_contract( - "TumblingWindow query_all_hierarchical", + "TumblingWindow query_all_hierarchical", &truth, |k| hierarchical.estimate(&DataInput::U64(k as u64)) as f64, &format!( @@ -1239,7 +1246,7 @@ mod documented_matrix { fn eh_documented_key_input(id: u8) { let input = key_input(id); let context = format!( - "{} k={DOC_K} window={DOC_WINDOW} payload CountMin {DOC_ROWS}x{DOC_COLS} FastPath, \ + "{} k={DOC_K} window={DOC_WINDOW} payload CountMinSketch {DOC_ROWS}x{DOC_COLS} FastPath, \ stream spread over {SPAN} time units", input.context() ); @@ -1249,7 +1256,7 @@ mod documented_matrix { fn eh_documented_string_input(id: u8) { let input = string_input(id); let context = format!( - "{} k={DOC_K} window={DOC_WINDOW} payload CountMin {DOC_ROWS}x{DOC_COLS} FastPath, \ + "{} k={DOC_K} window={DOC_WINDOW} payload CountMinSketch {DOC_ROWS}x{DOC_COLS} FastPath, \ stream spread over {SPAN} time units", input.context() ); @@ -1351,7 +1358,7 @@ mod microscope { let spec = CountMinSpec::new(ROWS, COLS); spec.assert_contract( - "MicroCM/Over", + "MicroCm/Over", &truth, |key| sk.estimate_with(&DataInput::U64(key as u64), DeltaStrategy::Over), &format!("zipf(4096, 1.1) x {ITEMS}, window = {T} x {PER_SUB_WINDOW} items"), diff --git a/tests/e2e/wire.rs b/tests/e2e/wire.rs index cc497f12..596f017b 100644 --- a/tests/e2e/wire.rs +++ b/tests/e2e/wire.rs @@ -152,7 +152,7 @@ fn a_kll_dynamic_envelope_round_trips_and_names_its_kind() { sketch.update(&(v as f64)); } let bytes = sketch.to_asapv1().expect("encode"); - assert_kind(&bytes, 0x06, 0x01, "KLLDynamic"); + assert_kind(&bytes, 0x06, 0x01, "KllDynamic"); let decoded = KllDynamic::::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.count(), sketch.count(), "retained mass"); for q in [0.0f64, 0.1, 0.5, 0.9, 1.0] { @@ -180,13 +180,13 @@ fn the_heap_backed_matrix_envelopes_round_trip_and_name_their_kinds() { cs.update(&DataInput::U64(*k)); } - let cms_bytes = cms.to_asapv1().expect("encode CMSHeap"); - assert_kind(&cms_bytes, 0x03, 0x00, "CMSHeap"); + let cms_bytes = cms.to_asapv1().expect("encode CountMinSketchWithHeap"); + assert_kind(&cms_bytes, 0x03, 0x00, "CountMinSketchWithHeap"); let decoded_cms = CountMinSketchWithHeap::, FastPath>::from_asapv1(&cms_bytes).expect("decode"); - let cs_bytes = cs.to_asapv1().expect("encode CSHeap"); - assert_kind(&cs_bytes, 0x0a, 0x00, "CSHeap"); + let cs_bytes = cs.to_asapv1().expect("encode CountSketchWithHeap"); + assert_kind(&cs_bytes, 0x0a, 0x00, "CountSketchWithHeap"); let decoded_cs = CountSketchWithHeap::, RegularPath>::from_asapv1(&cs_bytes).expect("decode"); @@ -194,23 +194,23 @@ fn the_heap_backed_matrix_envelopes_round_trip_and_name_their_kinds() { assert_eq!( decoded_cms.estimate(&DataInput::U64(k)), cms.estimate(&DataInput::U64(k)), - "CMSHeap key {k} changed across the wire" + "CountMinSketchWithHeap key {k} changed across the wire" ); assert_eq!( decoded_cs.estimate(&DataInput::U64(k)), cs.estimate(&DataInput::U64(k)), - "CSHeap key {k} changed across the wire" + "CountSketchWithHeap key {k} changed across the wire" ); } assert_eq!( decoded_cms.heap().len(), cms.heap().len(), - "CMSHeap heap size changed across the wire" + "CountMinSketchWithHeap heap size changed across the wire" ); assert_eq!( decoded_cs.heap().len(), cs.heap().len(), - "CSHeap heap size changed across the wire" + "CountSketchWithHeap heap size changed across the wire" ); } @@ -255,7 +255,7 @@ fn an_eh_sketch_list_envelope_round_trips_and_names_its_kind() { payload.update(&DataInput::U64(*k)); } let bytes = payload.to_asapv1().expect("encode"); - assert_kind(&bytes, 0x14, 0x00, "EHSketchList"); + assert_kind(&bytes, 0x14, 0x00, "EhBucketSketch"); let decoded = EhBucketSketch::from_asapv1(&bytes).expect("decode"); assert_eq!( decoded.sketch_type(), @@ -292,7 +292,7 @@ fn a_hydra_envelope_round_trips_and_names_the_counter_it_carries() { } } let bytes = hydra.to_asapv1().expect("encode"); - assert_kind(&bytes, 0x07, 0x01, "Hydra"); + assert_kind(&bytes, 0x07, 0x01, "Hydra"); let decoded = Hydra::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.schema(), hydra.schema(), "schema"); for r in ["eu", "us", "apac"] { @@ -335,19 +335,19 @@ fn the_univmon_family_envelopes_round_trip_and_name_their_kinds() { "UnivMon cardinality" ); - let pyramid_bytes = pyramid.to_asapv1().expect("encode UnivMonPyramid"); - assert_kind(&pyramid_bytes, 0x11, 0x00, "UnivMonPyramid"); + let pyramid_bytes = pyramid.to_asapv1().expect("encode UnivMonTwoTier"); + assert_kind(&pyramid_bytes, 0x11, 0x00, "UnivMonTwoTier"); let decoded_pyramid = - UnivMonTwoTier::from_asapv1(&pyramid_bytes).expect("decode UnivMonPyramid"); + UnivMonTwoTier::from_asapv1(&pyramid_bytes).expect("decode UnivMonTwoTier"); assert_eq!( decoded_pyramid.estimate_l1(), pyramid.estimate_l1(), - "UnivMonPyramid L1" + "UnivMonTwoTier L1" ); assert_eq!( decoded_pyramid.estimate_l2(), pyramid.estimate_l2(), - "UnivMonPyramid L2" + "UnivMonTwoTier L2" ); let q_bytes = q.to_asapv1().expect("encode UnivMonQ"); @@ -412,10 +412,10 @@ fn the_matrix_and_quantile_envelopes_carry_their_answers_unchanged() { hll.update(&DataInput::U64(*k)); } - let cm_bytes = cm.to_asapv1().expect("encode CountMin"); - assert_kind(&cm_bytes, 0x02, 0x00, "CountMin"); - let cs_bytes = cs.to_asapv1().expect("encode Count"); - assert_kind(&cs_bytes, 0x04, 0x00, "Count"); + let cm_bytes = cm.to_asapv1().expect("encode CountMinSketch"); + assert_kind(&cm_bytes, 0x02, 0x00, "CountMinSketch"); + let cs_bytes = cs.to_asapv1().expect("encode CountSketch"); + assert_kind(&cs_bytes, 0x04, 0x00, "CountSketch"); let kll_bytes = kll.to_asapv1().expect("encode KLL"); assert_kind(&kll_bytes, 0x06, 0x00, "KLL"); let hll_bytes = hll.to_asapv1().expect("encode HyperLogLog"); @@ -432,12 +432,12 @@ fn the_matrix_and_quantile_envelopes_carry_their_answers_unchanged() { assert_eq!( decoded_cm.estimate(&DataInput::U64(k)), cm.estimate(&DataInput::U64(k)), - "CountMin key {k}" + "CountMinSketch key {k}" ); assert_eq!( decoded_cs.estimate(&DataInput::U64(k)), cs.estimate(&DataInput::U64(k)), - "Count key {k}" + "CountSketch key {k}" ); } for q in [0.0f64, 0.5, 1.0] { diff --git a/tests/hll_custom_precision.rs b/tests/hll_custom_precision.rs index dea1fc33..26bb3867 100644 --- a/tests/hll_custom_precision.rs +++ b/tests/hll_custom_precision.rs @@ -38,12 +38,12 @@ fn check_precision(lg_k: usize, cardinality: usize, toler assert_eq!( R::RANK_HASH_BITS, 64 - lg_k, - "REGISTER_BITS for lg_k={lg_k}" + "RANK_HASH_BITS for lg_k={lg_k}" ); assert_eq!( R::INDEX_MASK, (num_registers as u64) - 1, - "P_MASK for lg_k={lg_k}" + "INDEX_MASK for lg_k={lg_k}" ); let storage = R::default(); diff --git a/tests/pbt/coco.rs b/tests/pbt/coco.rs index 21feb405..4ba8520b 100644 --- a/tests/pbt/coco.rs +++ b/tests/pbt/coco.rs @@ -376,8 +376,8 @@ proptest! { .map(|&(_, val)| val) .sum(); let estimate = sketch.estimate(&key); - prop_assert_eq!(estimate, expected, "estimate_key({})", key); - prop_assert!(estimate <= total, "estimate_key({}) exceeds {}", key, total); + prop_assert_eq!(estimate, expected, "estimate({})", key); + prop_assert!(estimate <= total, "estimate({}) exceeds {}", key, total); } } diff --git a/tests/pbt/ensemble.rs b/tests/pbt/ensemble.rs index 5350cc41..961519bd 100644 --- a/tests/pbt/ensemble.rs +++ b/tests/pbt/ensemble.rs @@ -47,8 +47,8 @@ impl Kind { /// The label the layer reports for the family. fn label(self) -> &'static str { match self { - Kind::CountMinSketch => "CountMin", - Kind::Count => "Count", + Kind::CountMinSketch => "CountMinSketch", + Kind::Count => "CountSketch", Kind::Ertl | Kind::Classic | Kind::Hip => "HLL", } } @@ -354,7 +354,7 @@ proptest! { prop_assert_eq!( layer_answers(&named, i, *kind, &probes), layer_answers(expected, i, *kind, &probes), - "member {} ({:?}) under insert_at {:?}", i, kind, chosen + "member {} ({:?}) under update_at {:?}", i, kind, chosen ); } } diff --git a/tests/pbt/microscope.rs b/tests/pbt/microscope.rs index a03528a0..728bda42 100644 --- a/tests/pbt/microscope.rs +++ b/tests/pbt/microscope.rs @@ -857,7 +857,7 @@ proptest! { prop_assert!(refused.is_err(), "a grid with no bytes must be refused"); let err = refused.err().unwrap_or_default(); prop_assert!( - err.contains("does not match rows * cols * depth"), + err.contains("does not match rows * cols * bucket_len"), "unexpected refusal for an unfilled {}x{}x{} grid: {}", rows, cols, depth, err ); diff --git a/tests/pbt/octo_delta.rs b/tests/pbt/octo_delta.rs index 1347c851..734639cd 100644 --- a/tests/pbt/octo_delta.rs +++ b/tests/pbt/octo_delta.rs @@ -1017,7 +1017,7 @@ proptest! { for k in &keys { cm.update_emit_delta(&DataInput::U64(*k), OCTO_COUNT_MIN_DEFAULT_THRESHOLD, &mut |d| cm_deltas.push(d)); } - prop_assert!(!cm_deltas.is_empty(), "CM_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!cm_deltas.is_empty(), "OCTO_COUNT_MIN_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for d in &cm_deltas { prop_assert_eq!(d.count, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, "a Count-Min promotion carried {}", d.count); } @@ -1027,9 +1027,9 @@ proptest! { for k in &keys { cs.update_emit_delta(&DataInput::U64(*k), OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, &mut |d| cs_deltas.push(d)); } - prop_assert!(!cs_deltas.is_empty(), "COUNT_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!cs_deltas.is_empty(), "OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for d in &cs_deltas { - prop_assert_eq!(d.count.unsigned_abs(), OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, "a Count promotion carried {}", d.count); + prop_assert_eq!(d.count.unsigned_abs(), OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, "a CountSketch promotion carried {}", d.count); } let mut dd = DdSketchWorker::new(0.05); @@ -1037,7 +1037,7 @@ proptest! { for k in &keys { dd.add_emit_delta(f64::from((*k % 8) as u32), OCTO_DDSKETCH_DEFAULT_THRESHOLD, &mut |d| dd_deltas.push(d)); } - prop_assert!(!dd_deltas.is_empty(), "DD_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!dd_deltas.is_empty(), "OCTO_DDSKETCH_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for d in &dd_deltas { prop_assert_eq!(d.count, u64::from(OCTO_DDSKETCH_DEFAULT_THRESHOLD), "a DDSketch promotion carried {}", d.count); } @@ -1047,7 +1047,7 @@ proptest! { for k in &keys { coco.update_emit_delta(&flow(*k), OCTO_COCO_DEFAULT_THRESHOLD, &mut |d| coco_deltas.push(d)); } - prop_assert!(!coco_deltas.is_empty(), "COCO_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!coco_deltas.is_empty(), "OCTO_COCO_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for d in &coco_deltas { prop_assert_eq!(d.count, u64::from(OCTO_COCO_DEFAULT_THRESHOLD), "a Coco promotion carried {}", d.count); } @@ -1064,7 +1064,7 @@ proptest! { _ => None, }) .collect(); - prop_assert!(!heavy.is_empty(), "ELASTIC_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!heavy.is_empty(), "OCTO_ELASTIC_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for value in &heavy { prop_assert_eq!(*value, OCTO_ELASTIC_DEFAULT_THRESHOLD, "an Elastic promotion carried {}", value); } @@ -1082,7 +1082,7 @@ proptest! { .filter(|d| d.layer == 0) .map(|d| d.cell.count) .collect(); - prop_assert!(!top_layer.is_empty(), "UNIVMON_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!top_layer.is_empty(), "OCTO_UNIVMON_DEFAULT_THRESHOLD promoted nothing over {} inserts", n); for value in &top_layer { prop_assert_eq!(value.unsigned_abs(), OCTO_UNIVMON_DEFAULT_THRESHOLD, "a UnivMon promotion carried {}", value); } @@ -1099,11 +1099,11 @@ proptest! { max_hll_threshold(precision), 64 - precision, "the HLL threshold ceiling is not 64 minus the precision" ); - prop_assert!(OCTO_HLL_DEFAULT_THRESHOLD < 64 - precision, "HLL_PROMASK can never fire"); + prop_assert!(OCTO_HLL_DEFAULT_THRESHOLD < 64 - precision, "OCTO_HLL_DEFAULT_THRESHOLD can never fire"); prop_assert_eq!( hll_parent.registers_as_slice(), hll_worker.registers_as_slice(), - "HLL_PROMASK held a register improvement back" + "OCTO_HLL_DEFAULT_THRESHOLD held a register improvement back" ); } @@ -1123,7 +1123,7 @@ proptest! { for k in &keys { worker.update_emit_delta(&DataInput::U64(*k), OCTO_MAX_PROMOTION_THRESHOLD, &mut |d| deltas.push(d)); } - prop_assert!(!deltas.is_empty(), "MAX_PROMASK promoted nothing over {} inserts", n); + prop_assert!(!deltas.is_empty(), "OCTO_MAX_PROMOTION_THRESHOLD promoted nothing over {} inserts", n); for d in &deltas { prop_assert_eq!(d.count.unsigned_abs(), OCTO_MAX_PROMOTION_THRESHOLD, "a promotion carried {}", d.count); prop_assert!(i8::try_from(d.count).is_ok(), "{} does not fit the worker's counter", d.count); diff --git a/tests/sketches_go_parity_probe.rs b/tests/sketches_go_parity_probe.rs index fb352ca5..bf468563 100644 --- a/tests/sketches_go_parity_probe.rs +++ b/tests/sketches_go_parity_probe.rs @@ -96,7 +96,7 @@ fn sketches_count_fastpath_matches_go_count_sketch_envelope() { assert_eq!( got, want, - "sketches::Count, FastPath> envelope diverges from Go golden \ + "sketches::CountSketch, FastPath> envelope diverges from Go golden \ ({} bytes got vs {} bytes want)", got.len(), want.len(), @@ -157,13 +157,13 @@ fn sketches_countmin_fastpath_matches_go_count_min_envelope() { assert_eq!( got.len(), want.len(), - "CountMin envelope length: got {} want {}", + "CountMinSketch envelope length: got {} want {}", got.len(), want.len(), ); assert_eq!( got, want, - "sketches::CountMin, FastPath> envelope diverges from Go golden", + "sketches::CountMinSketch, FastPath> envelope diverges from Go golden", ); } @@ -214,7 +214,7 @@ fn sketches_hll_classic_matches_go_envelope() { ); assert_eq!( got, want, - "sketches::HyperLogLogImpl envelope diverges from Go golden", + "sketches::HyperLogLogImpl envelope diverges from Go golden", ); } From e3845514659c362e2eb24690a0e16a0c277decd3 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:32:09 +0000 Subject: [PATCH 02/11] refactor(names)!: Coco constructors take (rows, cols) CocoSketch::init_with_size(w, d) and init_with_size_and_seed(w, d, seed) become with_dimensions(rows, cols) and with_dimensions_and_seed(rows, cols, seed); CocoSketchWorker::new and with_seed take (rows, cols) too. Every call site swaps its two arguments. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- docs/api/api_coco.md | 34 +++++----- docs/e2e_testing_harness.md | 2 +- docs/tests.md | 32 ++++----- src/message_pack_format/envelope.rs | 2 +- src/sketch_framework/eh_bucket_sketch/wire.rs | 2 +- src/sketch_framework/octo.rs | 34 +++++----- src/sketches/coco.rs | 66 +++++++++---------- src/sketches/coco/wire.rs | 28 ++++---- tests/asapv1_golden.rs | 2 +- tests/e2e/heavy_hitters.rs | 24 +++---- tests/e2e/octo.rs | 6 +- tests/e2e/windows.rs | 8 +-- tests/e2e/wire.rs | 2 +- tests/pbt/coco.rs | 4 +- tests/pbt/octo_delta.rs | 10 +-- 16 files changed, 130 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9155d430..11a1c88a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,10 +66,10 @@ signals a backwards-compatible change. what the suite is for and what it has found; [`tests/PBT_COVERAGE.md`](tests/PBT_COVERAGE.md) lists every law as a relation, one section per sketch, with what it is checked against. -- **Seeded CocoSketch.** `Coco::init_with_size_and_seed(w, d, seed)` draws the +- **Seeded CocoSketch.** `CocoSketch::with_dimensions_and_seed(rows, cols, seed)` draws the tie-break and the election from a generator seeded with `seed`, so two sketches built with the same seed and fed the same inserts hold identical - tables. The OctoSketch pair has the same: `CocoWorkerSketch::with_seed`, + tables. The OctoSketch pair has the same: `CocoSketchWorker::with_seed`, `CocoOctoWorker::with_threshold_and_seed`, `CocoOctoAggregator::with_seed`, and `CocoOctoPlan::with_seed` / `with_threshold_and_seed`, which derive each worker's seed and the aggregator's from one seed. The unseeded constructors diff --git a/docs/api/api_coco.md b/docs/api/api_coco.md index f5aa2bf0..4015fbe0 100644 --- a/docs/api/api_coco.md +++ b/docs/api/api_coco.md @@ -18,24 +18,24 @@ subset aggregation. ```rust fn new() -> Self -fn init_with_size(w: usize, d: usize) -> Self -fn init_with_size_and_seed(w: usize, d: usize, seed: u64) -> Self +fn with_dimensions(rows: usize, cols: usize) -> Self +fn with_dimensions_and_seed(rows: usize, cols: usize, seed: u64) -> Self ``` -`new()` builds a `1024 x 4` table: 4096 buckets, 128 KiB, plus a heap allocation +`new()` builds a `4 x 1024` table: 4096 buckets, 128 KiB, plus a heap allocation per occupied bucket for its stored key. -`new` and `init_with_size` draw the tie-break and the election from the thread -generator. `init_with_size_and_seed` draws them from a generator seeded with +`new` and `with_dimensions` draw the tie-break and the election from the thread +generator. `with_dimensions_and_seed` draws them from a generator seeded with `seed`, so the table is a deterministic function of the seed and the insert sequence; a clone copies the generator's state along with the table. -Sizing: `d` is the number of arrays an insert scans, and the paper recommends -2 to 4 — larger `d` spreads small flows better at the cost of a longer scan per -packet. `w` is the buckets per array, and it is the parameter to raise: bucket -count drives the collision rate, and the sketch can attribute mass to at most -`w * d` distinct keys at once, so size it against the number of heavy flows you -expect to resolve. +Sizing: `rows` is the number of arrays an update scans (the paper's `d`), and the +paper recommends 2 to 4 — more rows spread small flows better at the cost of a +longer scan per packet. `cols` is the buckets per array (the paper's `w`), and it +is the parameter to raise: bucket count drives the collision rate, and the +sketch can attribute mass to at most `rows * cols` distinct keys at once, so size +it against the number of heavy flows you expect to resolve. ## Insert/Update @@ -43,7 +43,7 @@ expect to resolve. fn update_weighted(&mut self, key: &str, weight: u64) ``` -The SIGCOMM '21 update: the `d` mapped buckets are scanned for `key` first and a +The SIGCOMM '21 update: the `rows` mapped buckets are scanned for `key` first and a match absorbs `v` directly; otherwise the whole increment lands in the smallest of them and that bucket's key is replaced with `key` with probability `v / val`. When several buckets share that smallest value the paper draws one of them @@ -69,19 +69,19 @@ leaves a key in at most one bucket, so no key comes back twice. Step 4 is `SELECT g(k_F), SUM(Size) ... GROUP BY g(k_F)`, which is `group_by(g)`. Reach for `group_by` when you want the whole result table: it folds every -recorded flow onto its partial key in one `O(w * d)` pass, where asking +recorded flow onto its partial key in one `O(rows * cols)` pass, where asking `estimate_projected` for each partial key separately costs one such pass per group. A single known partial key is still cheaper through `estimate_projected`, which allocates nothing. -`estimate` is the paper's point query: the sum of the `d` mapped buckets -that currently hold `key`, in `O(d)`. Because every increment is attributed to +`estimate` is the paper's point query: the sum of the `rows` mapped buckets +that currently hold `key`, in `O(rows)`. Because every increment is attributed to exactly one bucket, summing it over the observed keys returns the total inserted mass. `estimate_projected` is the paper's partial-key query. Section 4.3 defines it as `SELECT g(k_F), SUM(Size) ... GROUP BY g(k_F)`, where `g` maps a full key to a -partial key; `project` is that `g`. The table is scanned in `O(w * d)` and a +partial key; `project` is that `g`. The table is scanned in `O(rows * cols)` and a bucket counts only when its stored full key projects exactly to `partial_key`. `estimate_with_udf` is the same scan behind a general predicate over the @@ -124,7 +124,7 @@ map every key to the same bucket. A wider table is refused on both sides. ```rust use asap_sketchlib::CocoSketch; -let mut sk = CocoSketch::init_with_size(1024, 4); +let mut sk = CocoSketch::with_dimensions(4, 1024); sk.update_weighted("region=us|id=1", 3); let _ = sk.estimate("region=us|id=1"); let _ = sk.estimate_projected("region=us", |full| { diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index 65f0dbd8..3e3e6d81 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -328,7 +328,7 @@ error against its bound. sketch appears only where the assertion holds for every draw. `Kll::with_k_and_seed`, `KllDynamic::with_k_and_seed`, `KllSketch::with_seed`, `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, - `CocoSketch::init_with_size_and_seed` and `CocoOctoPlan::with_seed` exist for this. + `CocoSketch::with_dimensions_and_seed` and `CocoOctoPlan::with_seed` exist for this. - An estimator whose guarantee is unbiasedness gets no per-key band: its accuracy is the mean over independent seeds, within standard errors of the truth (`specs::assert_unbiased_mean`). diff --git a/docs/tests.md b/docs/tests.md index a1d9522a..3244b4fc 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -791,18 +791,18 @@ Wire tests: [`src/sketches/coco/wire.rs`](../src/sketches/coco/wire.rs) | `a_key_occupies_at_most_one_bucket_per_row` | A key never gains a second home in the table. | After 64 inserts of one key into a `32x4` table, verifies exactly one bucket holds it and `estimate` returns `64`. | | `estimate_key_never_exceeds_the_inserted_mass` | Point queries stay inside the table mass. | Over 500 weighted inserts across 40 keys in an `8x2` table, verifies the table mass equals the inserted mass and no per-key estimate exceeds it. | | `merge_combines_tables_without_losing_counts` | Merge combines tables without losing counts. | Merge behavior preserves expected aggregate semantics and internal invariants. | -| `coco_round_trip_serialization` | The envelope frames a sketch under kind_id `0x0c 0x00`, and the state survives a round trip. | For an `init_with_size(8, 4)` table holding two weighted keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x0c 0x00`, and that the decode keeps `w` `8`, `d` `4`, every bucket's key and value, and an `estimate` of `521`. | +| `coco_round_trip_serialization` | The envelope frames a sketch under kind_id `0x0c 0x00`, and the state survives a round trip. | For an `with_dimensions(4, 8)` table holding two weighted keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x0c 0x00`, and that the decode keeps `cols` `8`, `rows` `4`, every bucket's key and value, and an `estimate` of `521`. | | `coco_rejects_foreign_kind_id` | Another sketch carrying `rows`/`cols` metadata is refused on its kind_id. | Verifies a `4x8` Count-Min envelope fails to decode as a `CocoSketch`. | | `coco_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `CocoMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `CocoMetadata`. | | `coco_metadata_rejects_a_missing_cols_key` | `cols` is required and can never be silently defaulted. | Encodes the other seven `CocoMetadata` fields as a named map with `cols` omitted and verifies it does not decode as `CocoMetadata`. | | `coco_rejects_zero_dimension_payload` | A zero dimension is a decode error, not a `Vector2D` panic. | Verifies a crafted `4x0` envelope with an empty payload fails rather than panicking in the `cols.ilog2()` mask derivation. | | `coco_rejects_dimension_length_mismatch` | The length check fires from the declared dimensions, before any allocation is sized from them. | Verifies a crafted envelope declaring `MATRIX_MAX_ROWS x 2^24` while carrying three buckets fails. | | `coco_rejects_mass_under_an_unoccupied_bucket` | An unoccupied bucket holds no mass. | Verifies a crafted `1x2` payload pairing a `nil` key with a value of `9` fails to decode. | -| `coco_rejects_serializing_a_geometry_mismatch` | The encode side refuses geometries its own decoder would reject. | Verifies a sketch whose `w` is set to `16` against an 8-wide table fails to serialize, and that an `init_with_size(8, 0)` sketch fails too. | -| `coco_empty_buckets_are_nil_and_never_collide_with_an_empty_key` | An unoccupied bucket is `nil`, not an empty string. | For an all-empty `init_with_size(4, 2)` table, verifies the payload holds eight `0xc0` and no `0xa0`, the decode matches cell for cell, and `recorded_flows` is empty; then that a table holding an inserted empty key emits an `0xa0`, decodes identically, lists one flow, and answers that key's `estimate` with `5`. | -| `coco_mixed_occupancy_round_trips` | A table mixing occupied and free buckets round-trips bucket for bucket. | For an `init_with_size(16, 3)` table fed ten weighted keys, verifies the fixture leaves both bucket states present and that the decode matches on every cell and on `estimate` for all ten keys. | -| `coco_decoded_sketch_reserializes_byte_identically` | The emitted order is the table's own index order, so a decode re-encodes exactly. | For an `init_with_size(16, 3)` table fed twelve weighted keys, verifies the decoded sketch re-serializes to the bytes it came from. | -| `coco_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For an `init_with_size(8, 4)` table over a hasher declaring its own `HashProfile`, verifies every cell round-trips, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | +| `coco_rejects_serializing_a_geometry_mismatch` | The encode side refuses geometries its own decoder would reject. | Verifies a sketch whose `cols` is set to `16` against an 8-wide table fails to serialize, and that an `with_dimensions(0, 8)` sketch fails too. | +| `coco_empty_buckets_are_nil_and_never_collide_with_an_empty_key` | An unoccupied bucket is `nil`, not an empty string. | For an all-empty `with_dimensions(2, 4)` table, verifies the payload holds eight `0xc0` and no `0xa0`, the decode matches cell for cell, and `recorded_flows` is empty; then that a table holding an inserted empty key emits an `0xa0`, decodes identically, lists one flow, and answers that key's `estimate` with `5`. | +| `coco_mixed_occupancy_round_trips` | A table mixing occupied and free buckets round-trips bucket for bucket. | For an `with_dimensions(3, 16)` table fed ten weighted keys, verifies the fixture leaves both bucket states present and that the decode matches on every cell and on `estimate` for all ten keys. | +| `coco_decoded_sketch_reserializes_byte_identically` | The emitted order is the table's own index order, so a decode re-encodes exactly. | For an `with_dimensions(3, 16)` table fed twelve weighted keys, verifies the decoded sketch re-serializes to the bytes it came from. | +| `coco_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For an `with_dimensions(4, 8)` table over a hasher declaring its own `HashProfile`, verifies every cell round-trips, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | | `coco_rejects_too_many_rows` | Row `i` hashes at seed index `i`, so the table carries the seed list's row bound. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | ### KMV @@ -1159,8 +1159,8 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `a_ddsketch_delta_from_a_finer_mapping_is_dropped_not_allocated` | A delta naming an index outside the parent's store is dropped, not grown into. | Applies `DdSketchBucketDelta` at index `i32::MAX / 2` and at `i32::MIN / 2`, each with a value of `4`, to a `DdSketch` at alpha `0.01` holding one sample of `50.0`, and verifies the store length is unchanged and `get_count()` is still `1`. | | `a_count_worker_refuses_hashes_built_for_a_different_geometry` | Hashes built for another row count panic rather than partly landing. | Verifies a `4x16` `CountSketchWorker` fed two-row hashes panics with a "one hash per row" message. | | `a_worker_refuses_hashes_built_for_a_different_geometry` | The Count-Min worker refuses the same mismatch. | Verifies a `4x16` `CountMinSketchWorker` fed two-row hashes panics with a "one hash per row" message. | -| `coco_worker_promotes_the_key_its_bucket_holds_and_clears_it` | A Coco bucket promotes its `` pair and clears. | Inserts `"flow::coco"` into a `CocoSketchWorker::new(64, 2)` at `OCTO_COCO_DEFAULT_THRESHOLD`, and verifies nothing promotes over the first `OCTO_COCO_DEFAULT_THRESHOLD - 1` inserts, that the next insert emits exactly one delta carrying that key and a value of `OCTO_COCO_DEFAULT_THRESHOLD`, and that every residual counter is `0`. | -| `coco_promotion_conserves_the_inserted_mass` | Promoted mass plus residual is the whole stream, and a flush hands over the rest. | Drives 5,000 inserts over the 200 keys `flow::{i % 200}` through a `CocoSketchWorker::new(16, 2)` into a `CocoOctoAggregator::new(16, 2)` at `OCTO_COCO_DEFAULT_THRESHOLD`, verifies the parent's recorded mass plus the worker's residual equals `5_000`, and that after `flush` the parent alone holds `5_000`. | +| `coco_worker_promotes_the_key_its_bucket_holds_and_clears_it` | A Coco bucket promotes its `` pair and clears. | Inserts `"flow::coco"` into a `CocoSketchWorker::new(2, 64)` at `OCTO_COCO_DEFAULT_THRESHOLD`, and verifies nothing promotes over the first `OCTO_COCO_DEFAULT_THRESHOLD - 1` inserts, that the next insert emits exactly one delta carrying that key and a value of `OCTO_COCO_DEFAULT_THRESHOLD`, and that every residual counter is `0`. | +| `coco_promotion_conserves_the_inserted_mass` | Promoted mass plus residual is the whole stream, and a flush hands over the rest. | Drives 5,000 inserts over the 200 keys `flow::{i % 200}` through a `CocoSketchWorker::new(2, 16)` into a `CocoOctoAggregator::new(16, 2)` at `OCTO_COCO_DEFAULT_THRESHOLD`, verifies the parent's recorded mass plus the worker's residual equals `5_000`, and that after `flush` the parent alone holds `5_000`. | | `a_seeded_coco_plan_replays_identically` | A seeded `CocoOctoPlan` replays to the same deltas and parent table. | Routes 20,000 inserts of `flow::{(i*i + 7i) % 512}` by parity through two workers of `CocoOctoPlan::with_seed(64, 2, 7)` into its aggregator, and verifies a second run emits the same per-worker deltas and leaves the same parent table bucket for bucket, while seed `9` emits different deltas. | | `a_seeded_coco_plan_gives_each_worker_its_own_stream` | Each worker of a seeded plan draws from its own generator. | Feeds the same 20,000 inserts to worker `0` twice and to worker `1` of `CocoOctoPlan::with_seed(64, 2, 7)`, and verifies worker `0` reproduces its deltas and residual while worker `1` differs. | | `elastic_worker_promotes_each_half_on_its_own_terms` | The heavy half, the light half and the keyed eviction spill all promote. | Alternates `"flow::a"` and `"flow::b"` over 1,000 inserts into an `ElasticSketchWorker::new(1, 2, 64)` at `OCTO_ELASTIC_DEFAULT_THRESHOLD` and verifies at least one `Heavy`, at least one `Light` and at least one `Evicted` delta; the counts and the delta contents are not checked. | @@ -1908,11 +1908,11 @@ Test file: [`tests/e2e/heavy_hitters.rs`](../tests/e2e/heavy_hitters.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `coco_dense_key_mass_is_unbiased_single_pass_and_merged` | Coco's dense-key estimates are unbiased, single-pass and after a merge. | Over 48 seeds, fills `CocoSketch::init_with_size_and_seed(128, 2, ..)` with 60,000 `zipf(1.1)` draws over a 2,048-key domain at seed `9_001` rendered as `flow::{k}`, and separately fills two tables with the parity halves and merges them; for each path, the mean over seeds of `estimate` summed over the keys of true count 25 or more must lie within 6.5 standard errors of their exact sum, with that band under 2% of it. | -| `coco_disjoint_prefix_families_partition_the_mass` | Two disjoint prefix families' queries partition the stream mass. | Inserts `aaa{i % 50}` at weight 7 for 3,000 rounds and `zzz{i % 30}` at weight 3 for 2,000 rounds into `CocoSketch::init_with_size_and_seed(256, 2, COCO_SEED)`, and verifies `estimate_substring("aaa")` plus the prefix UDF's estimate for `"zzz"` equals the 27,000 inserted units exactly. | -| `coco_point_queries_partition_the_inserted_mass` | A point query partitions the inserted mass exactly. | Inserts 40,000 `zipf(1.2)` draws over a 2,000-key domain at seed `4242`, each at weight 2, into `CocoSketch::init_with_size_and_seed(128, 3, COCO_SEED)`, and verifies `estimate` summed over every observed key equals the 80,000 inserted units exactly. | -| `coco_point_estimates_are_unbiased_under_heavy_eviction` | The mean point estimate over independent runs tracks the truth even when the target is often evicted. | Over 800 independent `CocoSketch::init_with_size_and_seed(32, 2, COCO_SEED + trial)` runs, each fed 200 `bg::{i}` flows at weight 10 with 20 `flow::target` inserts spread evenly through them, verifies the target reads `0` in more than 80 of the runs and that the mean of the 800 `estimate` readings lands in `[17.0, 23.0]` around the true 20 via `assert_between`. | -| `coco_recall_meets_the_papers_heavy_hitter_target` | A heavy hitter holding 1% of the traffic is recorded at least 99% of the time. | Over 200 runs of `CocoSketch::init_with_size_and_seed(900, 2, COCO_SEED + trial)`, each fed 5,000 `bg::{i}` flows at weight 1 with 51 `flow::heavy` inserts interleaved, verifies `recorded_flows()` contains `flow::heavy` in at least 99% of runs (`recorded/200 >= 0.99`). First checks the configuration itself reaches that operating point: `1 - (1 + 900 * 51/5000)^-2 >= 0.99`. | +| `coco_dense_key_mass_is_unbiased_single_pass_and_merged` | Coco's dense-key estimates are unbiased, single-pass and after a merge. | Over 48 seeds, fills `CocoSketch::with_dimensions_and_seed(2, 128, ..)` with 60,000 `zipf(1.1)` draws over a 2,048-key domain at seed `9_001` rendered as `flow::{k}`, and separately fills two tables with the parity halves and merges them; for each path, the mean over seeds of `estimate` summed over the keys of true count 25 or more must lie within 6.5 standard errors of their exact sum, with that band under 2% of it. | +| `coco_disjoint_prefix_families_partition_the_mass` | Two disjoint prefix families' queries partition the stream mass. | Inserts `aaa{i % 50}` at weight 7 for 3,000 rounds and `zzz{i % 30}` at weight 3 for 2,000 rounds into `CocoSketch::with_dimensions_and_seed(2, 256, COCO_SEED)`, and verifies `estimate_substring("aaa")` plus the prefix UDF's estimate for `"zzz"` equals the 27,000 inserted units exactly. | +| `coco_point_queries_partition_the_inserted_mass` | A point query partitions the inserted mass exactly. | Inserts 40,000 `zipf(1.2)` draws over a 2,000-key domain at seed `4242`, each at weight 2, into `CocoSketch::with_dimensions_and_seed(3, 128, COCO_SEED)`, and verifies `estimate` summed over every observed key equals the 80,000 inserted units exactly. | +| `coco_point_estimates_are_unbiased_under_heavy_eviction` | The mean point estimate over independent runs tracks the truth even when the target is often evicted. | Over 800 independent `CocoSketch::with_dimensions_and_seed(2, 32, COCO_SEED + trial)` runs, each fed 200 `bg::{i}` flows at weight 10 with 20 `flow::target` inserts spread evenly through them, verifies the target reads `0` in more than 80 of the runs and that the mean of the 800 `estimate` readings lands in `[17.0, 23.0]` around the true 20 via `assert_between`. | +| `coco_recall_meets_the_papers_heavy_hitter_target` | A heavy hitter holding 1% of the traffic is recorded at least 99% of the time. | Over 200 runs of `CocoSketch::with_dimensions_and_seed(2, 900, COCO_SEED + trial)`, each fed 5,000 `bg::{i}` flows at weight 1 with 51 `flow::heavy` inserts interleaved, verifies `recorded_flows()` contains `flow::heavy` in at least 99% of runs (`recorded/200 >= 0.99`). First checks the configuration itself reaches that operating point: `1 - (1 + 900 * 51/5000)^-2 >= 0.99`. | | `elastic_passes_frequency_and_merge_conformance` | Elastic clears the shared one-sided frequency and merge-equivalence batteries. | Runs `conformance::frequency_battery` and `conformance::merge_equivalence_battery` over `ElasticSketch::::with_heavy_buckets(256)` (256 heavy buckets on the default `3x4096` light layer), keying integers as `flow::{k}` and estimating through `query`, on 60,000 `zipf(1.1)` draws over a 2,048-key domain at seed `9_001`, with `FrequencySpec { one_sided: true, rel_tol: 0.0, abs_tol: e/4096 * 60_000 }` (about `39.8`); the one-sided battery also probes the absent key `i64::MIN` and requires `\|est\| <= abs_tol` there. Ingest is through `update` only, never overload mode. | | `elastic_tracks_hot_flows` | Three hot flows amid background chatter come back within a fifth of their true size. | Feeds 12,000 records into `ElasticSketch::with_heavy_buckets(64)` — `hot-alpha`, `hot-beta` and `hot-gamma` taking a tenth of the stream each, the rest `bg{i%977}` — and verifies `query` for each hot flow lands in `[0.80 * true, 1.20 * true]` via `assert_between`. | | `elastic_light_dimensions_set_the_error_on_evicted_flows` | The light layer's dimensions reach the estimate of a flow the heavy part evicted. | Inserts `flow::target` 50 times then 20,000 distinct `bg::{i}` flows into `ElasticSketch::with_heavy_and_light(8, rows, cols)`, asserting the target is no longer resident in `heavy`, once at `1x64` and once at `3x4096`; verifies both readings are at least the true `50`, that the `3x4096` reading is at most `100`, and that the `1x64` reading is at least four times the `3x4096` one. | @@ -1924,7 +1924,7 @@ Test file: [`tests/e2e/heavy_hitters.rs`](../tests/e2e/heavy_hitters.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `coco_group_by_partitions_exactly_the_mass_the_table_records` | `group_by` folds every recorded flow exactly once. | Over the tenant fixture — `CocoSketch::init_with_size_and_seed(2_048, 4, COCO_SEED)` fed 8 tenants x 64 flows named `tenant-{t:02}/flow-{f:04}` at weight `5 + i%3` — verifies `group_by(tenant_of)` sums to exactly the mass `recorded_flows()` holds, that the recorded mass is at most the inserted mass, that the projection is non-empty, and that every group key starts with `tenant-`. | +| `coco_group_by_partitions_exactly_the_mass_the_table_records` | `group_by` folds every recorded flow exactly once. | Over the tenant fixture — `CocoSketch::with_dimensions_and_seed(4, 2_048, COCO_SEED)` fed 8 tenants x 64 flows named `tenant-{t:02}/flow-{f:04}` at weight `5 + i%3` — verifies `group_by(tenant_of)` sums to exactly the mass `recorded_flows()` holds, that the recorded mass is at most the inserted mass, that the projection is non-empty, and that every group key starts with `tenant-`. | | `coco_projected_and_udf_partial_key_queries_agree_with_group_by` | The three partial-key query forms agree with the projection they implement. | For each of the 8 tenants in the same `4 x 2048` fixture, verifies `estimate_projected(tenant, tenant_of)` and `estimate_with_udf(tenant, \|full, partial\| tenant_of(full) == partial)` both return exactly the `group_by` value, and `estimate_substring(tenant)` returns at least it; also verifies `estimate_projected("tenant-99", tenant_of)` is `0`. | | `coco_per_key_estimates_sum_to_their_own_group` | A projected query equals the recorded flows it covers. | Folds `recorded_flows()` into per-tenant totals by hand over the same `4 x 2048` fixture and verifies `estimate_projected` returns exactly that total for every tenant present. | | `a_coco_bucket_reports_its_own_partial_key_membership` | A single bucket answers substring and predicate membership for the key it holds. | On a bare `CocoBucket::new()`, verifies `is_partial_key("tenant-00")` and `is_partial_key_with_udf` both reject; after `update_key("tenant-00/flow-0001")`, `add(7)` and `add(5)`, verifies substring containment accepts `tenant-00` and `flow-0001` and rejects `tenant-01`, that a tenant-projection predicate accepts `tenant-00` and rejects `flow-0001`, and that `val` is `12`. | @@ -2043,7 +2043,7 @@ Test file: [`tests/e2e/windows.rs`](../tests/e2e/windows.rs) | `eh_count_sketch_variant_conforms_to_the_l2_model_over_the_retained_window` | A Count Sketch payload satisfies the two-sided L2 bound over the retained span. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketch(CountSketch::, FastPath>::with_dimensions(3, 512))`; the full-span merge is judged by `CountSketchSpec::new(3, 512).assert_contract`, which requires `\|est - f\|` under `sqrt(kappa/512)*\|\|f_-i\|\|_2` at the simultaneous `kappa` (union-bounded to `delta = 1e-3` over every probed key, zero violations) and pins the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` violation rate at `P[Bin(3, 1/3) >= 2] = 7/27`. | | `eh_countl2hh_variant_satisfies_the_l2_bound_over_the_retained_window` | A `CountSketchWithL2Norm` payload keeps the L2 bound of the Count Sketch matrix underneath it. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketchWithL2Norm(CountSketchWithL2Norm::with_dimensions(3, 512))`; the full-span merge is held to `CountSketchSpec::new(3, 512).assert_contract` - the simultaneous L2 band at `delta = 1e-3` with zero violations and the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` rate pin - reading frequencies through `merged.query(&DataInput::U64(k))`. | | `eh_elastic_variant_stays_one_sided_over_the_retained_window` | The Elastic payload never reads a retained heavy key below its true count. | For `ELASTIC(1024)` inside `ExponentialHistogram::new(8, 1_000_000, ..)`, feeds `10_000` Zipf(1.1) draws over a `2_048` domain (seed `0x0E11_0001`) as `DataInput::String("f{k}")` at `t = 0..9_999`, merges the full retained span, and asserts `est >= c` for the truth's top `32` keys with zero violations. No upper bound is placed on the reported counts, and keys outside the top `32` are not probed. | -| `eh_coco_variant_conserves_the_retained_mass` | The merged Coco window holds exactly the retained mass. | Same EH, stream and merge over `COCO(CocoSketch::init_with_size_and_seed(1024, 4, COCO_SEED))`, and verifies the merged table's recorded mass equals the number of events in the retained span. | +| `eh_coco_variant_conserves_the_retained_mass` | The merged Coco window holds exactly the retained mass. | Same EH, stream and merge over `COCO(CocoSketch::with_dimensions_and_seed(4, 1024, COCO_SEED))`, and verifies the merged table's recorded mass equals the number of events in the retained span. | | `eh_hll_variant_satisfies_the_register_error_model_over_the_retained_window` | A merged HLL payload lands inside the register error model for the retained span's cardinality. | Feeds `10_000` uniform draws over `200_000` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Hll(HyperLogLog::::new()))`, merges the full retained span, and checks `merged.query(&DataInput::Str("card"))` against the exact distinct count of that span under `CardinalityConfidenceSpec::hll(14, 4.0)` - a relative band of `4 * 1.04 / 2^7 = 0.0325` at `m = 2^14` registers. This is a single trial, aggregated by `assert_independent_binomial` at `p = 6.3e-5`, so no violation is tolerated. | | `eh_kll_variant_satisfies_the_rank_error_characterization_over_the_retained_window` | A merged KLL payload's worst rank error stays inside the DataSketches characterization across independent compaction seeds. | Builds `12` separate `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Kll(Kll::with_k_and_seed(200, seed)))` instances with `seed = 0x5EED_0200 + t * 0x9E37_79B9_7F4A_7C15`, feeds each the same `10_000` uniform values over `1_000_000` (stream seed `0x0E11_0001`) at `t = 0..9_999`, merges each one's full retained span, and reduces each trial to its worst normalized rank error over the grid `[0.1, 0.25, 0.5, 0.75, 0.9]` read through `merged.query(&DataInput::F64(q))`; `KllRankSpec::datasketches(200)` supplies `eps = 2.446/200^0.9433 = 0.0165`, and the `12` trial outcomes are accepted by `assert_independent_binomial` at `p = 0.01`. | | `eh_ddsketch_variant_satisfies_the_relative_value_error_contract_over_the_window` | A merged DDSketch payload holds relative value error and retains every observation. | Feeds `10_000` values `1_000_000 + uniform[0, 9_000_000)` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::DdSketch(DdSketch::new(0.01)))`, merges the full retained span, and checks `merged.query(&DataInput::F64(q))` at `q` in `[0.1, 0.25, 0.5, 0.75, 0.9]` against the span's exact order statistics under `RelativeQuantileSpec::core(0.01)` (`CeilNearestRank` convention, tolerance `alpha` plus a few-ULP slack) with zero violations; then asserts `merged.query(&DataInput::Str("count"))` equals the span's observation count exactly. | @@ -2253,7 +2253,7 @@ Test file: [`tests/e2e/wire.rs`](../tests/e2e/wire.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | | `a_bloom_envelope_round_trips_on_both_paths_and_names_its_kind` | A Bloom envelope names kind_id `0x17 0x00` and preserves every answer on both hash paths. | Fills a `Bloom` and a `Bloom`, each `with_capacity(2_000, 0.01)`, with `DataInput::I64(0..2_000)`, verifies both envelopes carry a `kind_id_len` of `2` and the bytes `0x17 0x00`, that each decode reports all 2,000 members present, that the regular decode answers exactly as its source on the 2,000 non-members `10_000..12_000`, and that `inserted()` is unchanged. | -| `a_coco_envelope_round_trips_and_names_its_kind` | A Coco envelope names kind_id `0x0c 0x00` and preserves every per-key estimate. | Fills `CocoSketch::::init_with_size_and_seed(512, 4, 0xC0C0)` with the keys `flow-{k}` of a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the envelope's `kind_id_len` of `2` and bytes `0x0c 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | +| `a_coco_envelope_round_trips_and_names_its_kind` | A Coco envelope names kind_id `0x0c 0x00` and preserves every per-key estimate. | Fills `CocoSketch::::with_dimensions_and_seed(4, 512, 0xC0C0)` with the keys `flow-{k}` of a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the envelope's `kind_id_len` of `2` and bytes `0x0c 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | | `a_ddsketch_envelope_round_trips_and_names_its_kind` | A DDSketch envelope names kind_id `0x05 0x00` and preserves its quantile answers exactly. | Fills `DdSketch::new(0.01)` with `1.0 + v` over `uniform_u64(20_000, 1_000_000, 0x1DE7_0001)`, verifies the envelope's `kind_id_len` of `2` and bytes `0x05 0x00`, and that the decode matches the source on `get_count()`, `alpha()`, and `get_value_at_quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`. | | `a_hydra_envelope_round_trips_and_names_the_counter_it_carries` | A Hydra envelope's kind_id names the counter it carries, and the schema and cells survive. | Builds `Hydra::with_schema(4, 512, ["region", "user"], HydraCounter::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, updates each of the nine region/user pairs 40 times with `DataInput::Str("event")`, verifies the `kind_id_len` of `2` and bytes `0x07 0x01`, an unchanged `schema()`, and equal `query_key` answers under `HydraQuery::Frequency` for all nine fully specified keys. | | `a_kll_dynamic_envelope_round_trips_and_names_its_kind` | A KllDynamic envelope names kind_id `0x06 0x01` and re-encodes byte-identically. | Fills `KllDynamic::::with_k_and_seed(200, 0x5EED_9001)` from `uniform_u64(20_000, 1_000_000, 0xC0DE_0001)`, verifies the `kind_id_len` of `2` and bytes `0x06 0x01`, an unchanged `count()`, bit-identical `quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`, and that the decode re-serializes to the same bytes. | diff --git a/src/message_pack_format/envelope.rs b/src/message_pack_format/envelope.rs index a92970f9..fe5c9c1e 100644 --- a/src/message_pack_format/envelope.rs +++ b/src/message_pack_format/envelope.rs @@ -263,7 +263,7 @@ mod tests { univmon_bytes.update_weighted(key, 2); } ss_u64.update(&DataInput::U64(9)); - let mut coco: CocoSketch = CocoSketch::init_with_size(8, 2); + let mut coco: CocoSketch = CocoSketch::with_dimensions(2, 8); coco.update_weighted("flow::a", 3); let mut elastic: ElasticSketch = ElasticSketch::with_heavy_and_light(4, 2, 16); elastic.update("flow::a".to_string()); diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index 22a2ae5b..d19e5aff 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -277,7 +277,7 @@ pub(crate) mod tests { 3, 8 ))]; out.push(EhBucketSketch::CocoSketch( - crate::CocoSketch::init_with_size(16, 2), + crate::CocoSketch::with_dimensions(2, 16), )); out.push(EhBucketSketch::CountSketchWithL2Norm( CountSketchWithL2Norm::with_dimensions(3, 256), diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index 016d6130..509ef059 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -526,23 +526,23 @@ pub struct CocoSketchWorker { } impl CocoSketchWorker { - /// Creates a `d` x `w` worker table, empty and cleared, drawing from the - /// thread generator. The argument order is `CocoSketch::init_with_size`'s. - pub fn new(w: usize, d: usize) -> Self { - Self::with_rng(w, d, CocoRng::default()) + /// Creates a `rows` x `cols` worker table, empty and cleared, drawing from + /// the thread generator. The argument order is `CocoSketch::with_dimensions`'s. + pub fn new(rows: usize, cols: usize) -> Self { + Self::with_rng(rows, cols, CocoRng::default()) } /// As [`Self::new`], drawing from `seed`. - pub fn with_seed(w: usize, d: usize, seed: u64) -> Self { - Self::with_rng(w, d, CocoRng::seeded(seed)) + pub fn with_seed(rows: usize, cols: usize, seed: u64) -> Self { + Self::with_rng(rows, cols, CocoRng::seeded(seed)) } - fn with_rng(w: usize, d: usize, rng: CocoRng) -> Self { + fn with_rng(rows: usize, cols: usize, rng: CocoRng) -> Self { Self { - keys: vec![None; w * d], - counters: vec![0u8; w * d], - cols: w, - rows: d, + keys: vec![None; rows * cols], + counters: vec![0u8; rows * cols], + cols, + rows, rng, } } @@ -2245,7 +2245,7 @@ impl CocoOctoWorker { /// Creates a worker sharing `threshold` with its peers and the aggregator. pub fn with_threshold(w: usize, d: usize, threshold: OctoThreshold) -> Self { Self { - sketch: CocoSketchWorker::new(w, d), + sketch: CocoSketchWorker::new(d, w), threshold, } } @@ -2258,7 +2258,7 @@ impl CocoOctoWorker { seed: u64, ) -> Self { Self { - sketch: CocoSketchWorker::with_seed(w, d, seed), + sketch: CocoSketchWorker::with_seed(d, w, seed), threshold, } } @@ -2309,14 +2309,14 @@ impl CocoOctoAggregator { /// Creates an aggregator with a `d` x `w` parent table. pub fn new(w: usize, d: usize) -> Self { Self { - sketch: CocoSketch::init_with_size(w, d), + sketch: CocoSketch::with_dimensions(d, w), } } /// As [`Self::new`], with the parent table drawing from `seed`. pub fn with_seed(w: usize, d: usize, seed: u64) -> Self { Self { - sketch: CocoSketch::init_with_size_and_seed(w, d, seed), + sketch: CocoSketch::with_dimensions_and_seed(d, w, seed), } } } @@ -3368,7 +3368,7 @@ mod worker_tests { #[test] fn coco_worker_promotes_the_key_its_bucket_holds_and_clears_it() { - let mut worker = CocoSketchWorker::new(64, 2); + let mut worker = CocoSketchWorker::new(2, 64); let key = "flow::coco"; let mut deltas: Vec = Vec::new(); @@ -3391,7 +3391,7 @@ mod worker_tests { fn coco_promotion_conserves_the_inserted_mass() { // Every insert lands in exactly one bucket, so what the parent received // plus what the worker still holds is the whole stream. - let mut worker = CocoSketchWorker::new(16, 2); + let mut worker = CocoSketchWorker::new(2, 16); let mut parent = CocoOctoAggregator::new(16, 2); let inserted = 5_000u64; for i in 0..inserted { diff --git a/src/sketches/coco.rs b/src/sketches/coco.rs index 3e378495..15694b14 100644 --- a/src/sketches/coco.rs +++ b/src/sketches/coco.rs @@ -71,11 +71,11 @@ impl CocoRng { } /// Buckets per array in a default `CocoSketch`. Width sets the collision rate, and -/// `w * d` buckets at 32 bytes each set the table footprint. +/// `rows * cols` buckets at 32 bytes each set the table footprint. const DEFAULT_WIDTH: usize = 1024; /// Arrays in a default `CocoSketch`, inside the paper's recommended 2..=4. Every -/// insert scans the `d` buckets its key maps to. +/// update scans the `rows` buckets its key maps to. const DEFAULT_DEPTH: usize = 4; impl Default for CocoBucket { @@ -134,12 +134,12 @@ impl Default for CocoSketch { impl CocoSketch { pub fn new() -> Self { - CocoSketch::init_with_size(DEFAULT_WIDTH, DEFAULT_DEPTH) + CocoSketch::with_dimensions(DEFAULT_DEPTH, DEFAULT_WIDTH) } pub fn debug(&self) { - println!("w: {}", self.cols); - println!("d: {}", self.rows); + println!("cols: {}", self.cols); + println!("rows: {}", self.rows); for i in 0..self.rows { print!("[ "); for j in 0..self.cols { @@ -149,22 +149,22 @@ impl CocoSketch { } } - /// A `d` x `w` table whose draws come from the thread generator. - pub fn init_with_size(w: usize, d: usize) -> Self { - Self::with_rng(w, d, CocoRng::default()) + /// A `rows` x `cols` table whose draws come from the thread generator. + pub fn with_dimensions(rows: usize, cols: usize) -> Self { + Self::with_rng(rows, cols, CocoRng::default()) } - /// A `d` x `w` table whose draws come from `seed`. Two sketches built with - /// the same seed and fed the same inserts hold identical tables. - pub fn init_with_size_and_seed(w: usize, d: usize, seed: u64) -> Self { - Self::with_rng(w, d, CocoRng::seeded(seed)) + /// A `rows` x `cols` table whose draws come from `seed`. Two sketches built + /// with the same seed and fed the same updates hold identical tables. + pub fn with_dimensions_and_seed(rows: usize, cols: usize, seed: u64) -> Self { + Self::with_rng(rows, cols, CocoRng::seeded(seed)) } - fn with_rng(w: usize, d: usize, rng: CocoRng) -> Self { + fn with_rng(rows: usize, cols: usize, rng: CocoRng) -> Self { CocoSketch { - cols: w, - rows: d, - table: Vector2D::from_fn(d, w, |_, _| CocoBucket::default()), + cols, + rows, + table: Vector2D::from_fn(rows, cols, |_, _| CocoBucket::default()), rng, _hasher: PhantomData, } @@ -172,7 +172,7 @@ impl CocoSketch { /// Adds `v` to `key` using the paper's stochastic variance-optimized update. /// - /// The `d` mapped buckets are scanned for `key` first; a match absorbs `v` + /// The `rows` mapped buckets are scanned for `key` first; a match absorbs `v` /// directly. Otherwise the whole increment lands in the smallest of them, /// drawn uniformly at random when several share that smallest value, and /// that bucket's key is replaced with `key` with probability `v / val`. @@ -220,7 +220,7 @@ impl CocoSketch { } } - /// Frequency estimate for `key` as defined by the paper: the sum of the `d` + /// Frequency estimate for `key` as defined by the paper: the sum of the `rows` /// mapped buckets that currently hold `key`. pub fn estimate(&self, key: &str) -> u64 { if self.rows == 0 || self.cols == 0 { @@ -324,7 +324,7 @@ mod tests { #[test] fn insert_then_estimate_matches_full_value_for_partial_key() { // cover end-to-end flow of inserting a key and querying with a substring - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); let key = "user:1234"; coco.update_weighted(key, 3); @@ -338,7 +338,7 @@ mod tests { #[test] fn estimate_with_udf_allows_custom_partial_matching() { // ensure custom UDF matching logic aggregates only intended buckets - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); coco.update_weighted("region=us|id=1", 4); coco.update_weighted("region=eu|id=2", 6); @@ -363,8 +363,8 @@ mod tests { #[test] fn equal_seeds_build_identical_tables() { let build = |seed: u64| { - let mut left: CocoSketch = CocoSketch::init_with_size_and_seed(64, 2, seed); - let mut right: CocoSketch = CocoSketch::init_with_size_and_seed(64, 2, seed + 1); + let mut left: CocoSketch = CocoSketch::with_dimensions_and_seed(2, 64, seed); + let mut right: CocoSketch = CocoSketch::with_dimensions_and_seed(2, 64, seed + 1); for i in 0..20_000u64 { let key = format!("flow::{}", (i * i + 7 * i) % 512); if i % 2 == 0 { @@ -389,7 +389,7 @@ mod tests { for trial in 0..TRIALS { let mut coco: CocoSketch = - CocoSketch::init_with_size_and_seed(TEST_W, TEST_D, trial as u64); + CocoSketch::with_dimensions_and_seed(TEST_D, TEST_W, trial as u64); coco.update_weighted("flow::tie-probe", 1); let row = (0..TEST_D) .find(|i| { @@ -412,7 +412,7 @@ mod tests { #[test] fn the_three_queries_disagree_on_a_key_that_prefixes_another() { // substring containment is not a key projection: "k1" also matches "k10" - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); coco.update_weighted("k1", 7); coco.update_weighted("k10", 5); @@ -424,7 +424,7 @@ mod tests { #[test] fn estimate_projected_aggregates_full_keys_sharing_a_partial_key() { // the paper's figure 7: two full keys on one srcip sum to that srcip - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); coco.update_weighted("19.98.10.26|80", 521); coco.update_weighted("19.98.10.26|443", 520); coco.update_weighted("34.52.73.17|118", 856); @@ -439,7 +439,7 @@ mod tests { #[test] fn recorded_flows_yields_each_occupied_bucket_once() { - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); for i in 0..20u64 { coco.update_weighted(&format!("flow::{i}"), i + 1); } @@ -459,7 +459,7 @@ mod tests { #[test] fn group_by_agrees_with_per_key_projected_queries() { // the one-pass grouping and the per-key scan must never disagree - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); for i in 0..60u64 { coco.update_weighted(&format!("fam{}|item{i}", i % 5), i % 7 + 1); } @@ -478,7 +478,7 @@ mod tests { #[test] fn group_by_preserves_the_inserted_mass() { // an 8x2 table against far more keys forces eviction; mass still holds - let mut coco: CocoSketch = CocoSketch::init_with_size(8, 2); + let mut coco: CocoSketch = CocoSketch::with_dimensions(2, 8); let mut total = 0u64; for i in 0..400u64 { coco.update_weighted(&format!("fam{}|item{}", i % 6, i % 37), 3); @@ -491,7 +491,7 @@ mod tests { #[test] fn group_by_reproduces_the_papers_figure_seven() { - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); coco.update_weighted("19.98.10.26|80", 521); coco.update_weighted("19.98.10.26|443", 520); coco.update_weighted("34.52.73.17|118", 856); @@ -506,7 +506,7 @@ mod tests { fn a_key_occupies_at_most_one_bucket_per_row() { // the match scan covers every row before a victim is chosen, so a key // already resident in a later row never gains a second home - let mut coco: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); let key = "flow::single-home"; for _ in 0..64 { @@ -525,7 +525,7 @@ mod tests { fn estimate_key_never_exceeds_the_inserted_mass() { // biased replacement is unbiased in expectation but never invents mass // beyond what the whole table holds - let mut coco: CocoSketch = CocoSketch::init_with_size(8, 2); + let mut coco: CocoSketch = CocoSketch::with_dimensions(2, 8); let mut total = 0u64; for i in 0..500u64 { coco.update_weighted(&format!("k{}", i % 40), 3); @@ -545,8 +545,8 @@ mod tests { #[test] fn merge_combines_tables_without_losing_counts() { // verify merging replays entries so both sketches contribute to totals - let mut left: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); - let mut right: CocoSketch = CocoSketch::init_with_size(TEST_W, TEST_D); + let mut left: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); + let mut right: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); left.update_weighted("alpha:key", 7); right.update_weighted("beta:key", 11); diff --git a/src/sketches/coco/wire.rs b/src/sketches/coco/wire.rs index 748bcebf..0de335f9 100644 --- a/src/sketches/coco/wire.rs +++ b/src/sketches/coco/wire.rs @@ -107,7 +107,7 @@ fn check_geometry(rows: usize, cols: usize) -> Result<(), String> { Ok(()) } -/// Rejects a bucket layout no insert could produce. `update_weighted` scans all `d` +/// Rejects a bucket layout no insert could produce. `update_weighted` scans all `rows` /// mapped buckets before electing one, so a stored key sits in row `r` only at /// column `hash64_seeded(r, key) % cols`, and holds at most one bucket in the /// whole table. `keys` must already be `rows*cols` long and `cols` non-zero. @@ -253,7 +253,7 @@ mod tests { fn coco_rejects_too_many_rows() { let rows = MATRIX_MAX_ROWS + 1; assert!( - CocoSketch::::init_with_size(8, rows) + CocoSketch::::with_dimensions(rows, 8) .to_asapv1() .is_err(), "a table past MATRIX_MAX_ROWS must not serialize" @@ -274,7 +274,7 @@ mod tests { // The boundary itself is eligible. assert!( - CocoSketch::::init_with_size(8, MATRIX_MAX_ROWS) + CocoSketch::::with_dimensions(MATRIX_MAX_ROWS, 8) .to_asapv1() .is_ok() ); @@ -292,7 +292,7 @@ mod tests { #[test] fn coco_round_trip_serialization() { - let mut sketch: CocoSketch = CocoSketch::init_with_size(8, 4); + let mut sketch: CocoSketch = CocoSketch::with_dimensions(4, 8); sketch.update_weighted("19.98.10.26|80", 521); sketch.update_weighted("34.52.73.17|118", 856); @@ -496,14 +496,14 @@ mod tests { /// bytes the decoder would refuse. #[test] fn coco_rejects_serializing_a_geometry_mismatch() { - let mut sketch: CocoSketch = CocoSketch::init_with_size(8, 4); + let mut sketch: CocoSketch = CocoSketch::with_dimensions(4, 8); sketch.cols = 16; assert!( sketch.to_asapv1().is_err(), "a table whose bucket count disagrees with its geometry must not serialize" ); - let empty: CocoSketch = CocoSketch::init_with_size(8, 0); + let empty: CocoSketch = CocoSketch::with_dimensions(0, 8); assert!( empty.to_asapv1().is_err(), "a zero-dimension geometry must not serialize" @@ -516,7 +516,7 @@ mod tests { #[test] fn coco_rejects_serializing_a_hand_built_table() { let key = "flow::misplaced"; - let mut misplaced: CocoSketch = CocoSketch::init_with_size(2, 1); + let mut misplaced: CocoSketch = CocoSketch::with_dimensions(1, 2); let wrong = 1 - mapped_col(0, key, 2); misplaced.table.as_mut_slice()[wrong] = CocoBucket { full_key: Some(key.to_string()), @@ -529,7 +529,7 @@ mod tests { assert!(problem.contains("maps to column"), "got {problem}"); let (rows, cols) = (2, 2); - let mut twice: CocoSketch = CocoSketch::init_with_size(cols, rows); + let mut twice: CocoSketch = CocoSketch::with_dimensions(rows, cols); for row in 0..rows { twice.table.as_mut_slice()[row * cols + mapped_col(row, key, cols)] = CocoBucket { full_key: Some(key.to_string()), @@ -547,7 +547,7 @@ mod tests { /// key stays distinguishable from the buckets that hold nothing. #[test] fn coco_empty_buckets_are_nil_and_never_collide_with_an_empty_key() { - let all_empty: CocoSketch = CocoSketch::init_with_size(4, 2); + let all_empty: CocoSketch = CocoSketch::with_dimensions(2, 4); let encoded = all_empty.to_asapv1().expect("serialize"); let (_, _, payload) = envelope::split(&encoded).expect("split"); assert_eq!( @@ -560,7 +560,7 @@ mod tests { assert_eq!(cells(&all_empty), cells(&decoded)); assert!(decoded.recorded_flows().next().is_none()); - let mut with_empty_key: CocoSketch = CocoSketch::init_with_size(4, 2); + let mut with_empty_key: CocoSketch = CocoSketch::with_dimensions(2, 4); with_empty_key.update_weighted("", 5); let encoded = with_empty_key.to_asapv1().expect("serialize"); let (_, _, payload) = envelope::split(&encoded).expect("split"); @@ -574,7 +574,7 @@ mod tests { /// A table mixing occupied and free buckets round-trips bucket for bucket. #[test] fn coco_mixed_occupancy_round_trips() { - let mut sketch: CocoSketch = CocoSketch::init_with_size(16, 3); + let mut sketch: CocoSketch = CocoSketch::with_dimensions(3, 16); for i in 0..10u64 { sketch.update_weighted(&format!("flow::{i}"), i + 1); } @@ -594,7 +594,7 @@ mod tests { /// re-serializes to the bytes it came from. #[test] fn coco_decoded_sketch_reserializes_byte_identically() { - let mut sketch: CocoSketch = CocoSketch::init_with_size(16, 3); + let mut sketch: CocoSketch = CocoSketch::with_dimensions(3, 16); for i in 0..12u64 { sketch.update_weighted(&format!("fam{}|item{i}", i % 4), i % 5 + 1); } @@ -649,8 +649,8 @@ mod tests { #[test] fn coco_custom_hasher_profile_round_trips_and_is_self_describing() { // (a) A Coco built with a custom-profile hasher round-trips. - let mut alt: CocoSketch = CocoSketch::init_with_size(8, 4); - let mut std: CocoSketch = CocoSketch::init_with_size(8, 4); + let mut alt: CocoSketch = CocoSketch::with_dimensions(4, 8); + let mut std: CocoSketch = CocoSketch::with_dimensions(4, 8); alt.update_weighted("flow::42", 7); alt.update_weighted("flow::7", 3); std.update_weighted("flow::42", 7); diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 5e2d07cc..e3362ff7 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -1319,7 +1319,7 @@ const COCO_CELLS: [(usize, usize, &str, u64); 15] = [ fn coco_3x7_matches_golden() { let want = decode_hex(GOLDEN_COCO); - let mut sketch: CocoSketch = CocoSketch::init_with_size(7, 3); + let mut sketch: CocoSketch = CocoSketch::with_dimensions(3, 7); for (r, c, key, val) in COCO_CELLS { sketch.table[r][c] = CocoBucket { full_key: Some(key.to_string()), diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index 71a82dc7..e13b6a38 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -819,9 +819,9 @@ mod keyed_bucket { let (mut single, mut merged) = (Vec::new(), Vec::new()); for trial in 0..TRIALS { let seed = COCO_SEED + 3 * trial; - let mut one = CocoSketch::::init_with_size_and_seed(W, D, seed); - let mut left = CocoSketch::::init_with_size_and_seed(W, D, seed + 1); - let mut right = CocoSketch::::init_with_size_and_seed(W, D, seed + 2); + let mut one = CocoSketch::::with_dimensions_and_seed(D, W, seed); + let mut left = CocoSketch::::with_dimensions_and_seed(D, W, seed + 1); + let mut right = CocoSketch::::with_dimensions_and_seed(D, W, seed + 2); for (i, k) in stream.iter().enumerate() { let key = flow_key(*k); one.update_weighted(&key, 1); @@ -879,8 +879,8 @@ mod keyed_bucket { /// disjoint one together return the stream mass. #[test] fn coco_disjoint_prefix_families_partition_the_mass() { - let mut coco = CocoSketch::::init_with_size_and_seed( - 256, 2, COCO_SEED, + let mut coco = CocoSketch::::with_dimensions_and_seed( + 2, 256, COCO_SEED, ); let mut total = 0u64; for i in 0..3000u64 { @@ -908,9 +908,9 @@ mod keyed_bucket { #[test] fn coco_point_queries_partition_the_inserted_mass() { const COCO_BUCKETS: usize = 128; - let mut coco = CocoSketch::::init_with_size_and_seed( - COCO_BUCKETS, + let mut coco = CocoSketch::::with_dimensions_and_seed( 3, + COCO_BUCKETS, COCO_SEED, ); let mut truth: HashMap = HashMap::new(); @@ -950,9 +950,9 @@ mod keyed_bucket { let mut estimates: Vec = Vec::with_capacity(TRIALS); for trial in 0..TRIALS { - let mut coco = CocoSketch::::init_with_size_and_seed( - 32, + let mut coco = CocoSketch::::with_dimensions_and_seed( 2, + 32, COCO_SEED + trial as u64, ); let mut sent = 0u64; @@ -1016,9 +1016,9 @@ mod keyed_bucket { let mut recorded = 0usize; for trial in 0..TRIALS { - let mut coco = CocoSketch::::init_with_size_and_seed( - WIDTH, + let mut coco = CocoSketch::::with_dimensions_and_seed( DEPTH, + WIDTH, COCO_SEED + trial as u64, ); let mut sent = 0u64; @@ -1207,7 +1207,7 @@ mod partial_key_and_heavy_maintenance { fn filled_coco() -> (CocoSketch, u64) { let mut coco = - CocoSketch::::init_with_size_and_seed(2_048, 4, super::COCO_SEED); + CocoSketch::::with_dimensions_and_seed(4, 2_048, super::COCO_SEED); let mut inserted = 0u64; for (i, key) in coco_keys().iter().enumerate() { let weight = REPEATS + (i % 3) as u64; diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index b39753e8..1a0bf89b 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -3031,7 +3031,7 @@ mod heavy_hitters { let mut this = Self { children: Vec::with_capacity(workers), seen: vec![0; workers], - parent: CocoSketch::init_with_size_and_seed(w, d, seed), + parent: CocoSketch::with_dimensions_and_seed(d, w, seed), period, w, d, @@ -3049,7 +3049,7 @@ mod heavy_hitters { /// An empty child table on the next seed. fn fresh(&mut self) -> CocoSketch { self.next_seed += 1; - CocoSketch::init_with_size_and_seed(self.w, self.d, self.next_seed) + CocoSketch::with_dimensions_and_seed(self.d, self.w, self.next_seed) } fn insert(&mut self, index: usize, raw: u64) { @@ -4112,7 +4112,7 @@ mod heavy_hitters { COCO_SEED + 100 + seed, ); let mut ideal: CocoSketch = - CocoSketch::init_with_size_and_seed(w, d, COCO_SEED + 200 + seed); + CocoSketch::with_dimensions_and_seed(d, w, COCO_SEED + 200 + seed); let mut running: HashMap = HashMap::new(); let mut seen = 0u64; let stride = n / 8; diff --git a/tests/e2e/windows.rs b/tests/e2e/windows.rs index 99cf19ac..7dfc396c 100644 --- a/tests/e2e/windows.rs +++ b/tests/e2e/windows.rs @@ -112,8 +112,8 @@ fn every_eh_variant_selects_the_documented_merge_norm() { ), ( "CocoSketch", - asap_sketchlib::EhBucketSketch::CocoSketch(CocoSketch::init_with_size_and_seed( - 512, 4, COCO_SEED, + asap_sketchlib::EhBucketSketch::CocoSketch(CocoSketch::with_dimensions_and_seed( + 4, 512, COCO_SEED, )), ), ( @@ -274,7 +274,7 @@ fn eh_elastic_variant_stays_one_sided_over_the_retained_window() { fn eh_coco_variant_conserves_the_retained_mass() { let (merged, truth, ctx) = run_heavy_hitter_variant(asap_sketchlib::EhBucketSketch::CocoSketch( - CocoSketch::init_with_size_and_seed(1024, 4, COCO_SEED), + CocoSketch::with_dimensions_and_seed(4, 1024, COCO_SEED), )); let asap_sketchlib::EhBucketSketch::CocoSketch(coco) = &merged else { panic!("a CocoSketch histogram merges into a CocoSketch payload"); @@ -722,7 +722,7 @@ fn every_eh_variant_can_merge_into_its_own_kind() { ), ( "CocoSketch", - EhBucketSketch::CocoSketch(CocoSketch::init_with_size_and_seed(256, 4, COCO_SEED)), + EhBucketSketch::CocoSketch(CocoSketch::with_dimensions_and_seed(4, 256, COCO_SEED)), ), ( "CountSketchWithL2Norm", diff --git a/tests/e2e/wire.rs b/tests/e2e/wire.rs index 596f017b..8673525d 100644 --- a/tests/e2e/wire.rs +++ b/tests/e2e/wire.rs @@ -91,7 +91,7 @@ fn a_bloom_envelope_round_trips_on_both_paths_and_names_its_kind() { #[test] fn a_coco_envelope_round_trips_and_names_its_kind() { - let mut coco = CocoSketch::::init_with_size_and_seed(512, 4, 0xC0C0); + let mut coco = CocoSketch::::with_dimensions_and_seed(4, 512, 0xC0C0); for k in frequency_stream() { coco.update_weighted(&format!("flow-{k}"), 1); } diff --git a/tests/pbt/coco.rs b/tests/pbt/coco.rs index 4ba8520b..89e2b04b 100644 --- a/tests/pbt/coco.rs +++ b/tests/pbt/coco.rs @@ -79,7 +79,7 @@ fn crowded_pair(max: usize) -> impl Strategy { } fn filled(w: usize, d: usize, seed: u64, stream: &[(u64, u64)]) -> C { - let mut sketch = C::init_with_size_and_seed(w, d, seed); + let mut sketch = C::with_dimensions_and_seed(d, w, seed); for (id, v) in stream { sketch.update_weighted(&key_of(*id), *v); } @@ -166,7 +166,7 @@ proptest! { seed in any::(), stream in keys(200), ) { - let mut sketch = C::init_with_size_and_seed(width, depth, seed); + let mut sketch = C::with_dimensions_and_seed(depth, width, seed); for k in &stream { sketch.update_weighted(&format!("flow-{k}"), 1); } diff --git a/tests/pbt/octo_delta.rs b/tests/pbt/octo_delta.rs index 734639cd..fee04de4 100644 --- a/tests/pbt/octo_delta.rs +++ b/tests/pbt/octo_delta.rs @@ -583,7 +583,7 @@ proptest! { seed in any::(), keys in stream(300), ) { - let mut worker = CocoSketchWorker::with_seed(w, d, seed); + let mut worker = CocoSketchWorker::with_seed(d, w, seed); let mut deltas: Vec = Vec::new(); for k in &keys { worker.update_emit_delta(&flow(*k), tau, &mut |delta| deltas.push(delta)); @@ -619,7 +619,7 @@ proptest! { keys in stream(300), ) { let mut workers: Vec = (0..shards) - .map(|i| CocoSketchWorker::with_seed(w, d, seed.wrapping_add(i as u64 + 1))) + .map(|i| CocoSketchWorker::with_seed(d, w, seed.wrapping_add(i as u64 + 1))) .collect(); let mut parent = CocoOctoAggregator::with_seed(w, d, seed); for (i, k) in keys.iter().enumerate() { @@ -650,7 +650,7 @@ proptest! { n in 2usize..120, seed in any::(), ) { - let mut worker = CocoSketchWorker::with_seed(1, d, seed); + let mut worker = CocoSketchWorker::with_seed(d, 1, seed); let mut promoted = 0usize; for k in 0..n as u64 { worker.update_emit_delta(&flow(k), OCTO_MAX_PROMOTION_THRESHOLD, &mut |_| promoted += 1); @@ -677,7 +677,7 @@ proptest! { seed in any::(), keys in stream(300), ) { - let mut worker = CocoSketchWorker::with_seed(w, d, seed); + let mut worker = CocoSketchWorker::with_seed(d, w, seed); for k in &keys { let arrival = flow(*k); let mut promoted: Vec = Vec::new(); @@ -1042,7 +1042,7 @@ proptest! { prop_assert_eq!(d.count, u64::from(OCTO_DDSKETCH_DEFAULT_THRESHOLD), "a DDSketch promotion carried {}", d.count); } - let mut coco = CocoSketchWorker::with_seed(8, 2, seed); + let mut coco = CocoSketchWorker::with_seed(2, 8, seed); let mut coco_deltas: Vec = Vec::new(); for k in &keys { coco.update_emit_delta(&flow(*k), OCTO_COCO_DEFAULT_THRESHOLD, &mut |d| coco_deltas.push(d)); From 06de28ac827de6c07f6840175a152596829e1b46 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:34:36 +0000 Subject: [PATCH 03/11] refactor(names)!: CountSketchWithL2Norm gets its own module; drop the L2HH enum UnivMon and UnivMonTwoTier hold CountSketchWithL2Norm layers directly. The type and its ASAPv1 codec move to sketches::countsketch_l2_norm (countsketch_l2_norm.rs, countsketch_l2_norm/wire.rs). No wire change. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- docs/api/api_common.md | 2 +- docs/api/api_common_input.md | 14 +- docs/asapv1_wire_format.md | 6 +- docs/library_map.md | 8 +- docs/message_pack_format.md | 4 +- docs/tests.md | 4 +- src/common/input.rs | 64 +-- src/common/mod.rs | 4 +- .../native/countsketch_topk.rs | 4 +- src/sketch_framework/eh_bucket_sketch.rs | 2 +- src/sketch_framework/octo.rs | 9 +- src/sketch_framework/univmon.rs | 20 +- src/sketch_framework/univmon/wire.rs | 52 +-- src/sketch_framework/univmon_q/wire.rs | 9 +- src/sketch_framework/univmon_two_tier.rs | 28 +- src/sketch_framework/univmon_two_tier/wire.rs | 22 +- src/sketches/countsketch_heap.rs | 382 +----------------- src/sketches/countsketch_l2_norm.rs | 373 +++++++++++++++++ .../wire.rs} | 18 +- src/sketches/mod.rs | 6 +- tests/asapv1_golden.rs | 6 +- tests/e2e/octo.rs | 4 +- tests/pbt/univmon.rs | 6 +- 24 files changed, 491 insertions(+), 561 deletions(-) create mode 100644 src/sketches/countsketch_l2_norm.rs rename src/sketches/{countsketch_heap/l2hh_wire.rs => countsketch_l2_norm/wire.rs} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11a1c88a..9331e48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -171,7 +171,10 @@ signals a backwards-compatible change. - **`CountL2HH` -> `CountSketchWithL2Norm`** (breaking), with `update_weighted*`, `estimate*`, `l2()`, `l2_squared()`, `seed_index` and `with_dimensions_and_seed_index`; `l2hh_cell_for_row` -> - `CountSketchWithL2Norm::cell_for_row`. + `CountSketchWithL2Norm::cell_for_row`. It lives in module + `countsketch_l2_norm`, and the single-variant `L2HH` enum is gone: + `UnivMon` and `UnivMonTwoTier` `layer_sketches` hold `CountSketchWithL2Norm` + directly. - **Elastic and Coco** (breaking). `Elastic` -> `ElasticSketch`, `Coco` -> `CocoSketch`, `CocoWorkerSketch` -> `CocoSketchWorker`; `bktlen` -> `heavy_buckets`, `init_with_length` -> `with_heavy_buckets`, diff --git a/docs/api/api_common.md b/docs/api/api_common.md index 5f4fbd5c..387e51e7 100644 --- a/docs/api/api_common.md +++ b/docs/api/api_common.md @@ -37,7 +37,7 @@ src/common/ ```rust use asap_sketchlib::common::{ // Input layer - DataInput, OwnedDataInput, HeapEntry, L2HH, + DataInput, OwnedDataInput, HeapEntry, input_to_owned, heap_item_to_sketch_input, // Hash layer diff --git a/docs/api/api_common_input.md b/docs/api/api_common_input.md index 695ecccc..241a4420 100644 --- a/docs/api/api_common_input.md +++ b/docs/api/api_common_input.md @@ -15,7 +15,6 @@ Canonical input and ownership model shared across sketches/frameworks. - `HeapEntry` - `HydraQuery<'a>` - `HydraCounter` -- `L2HH` `HydraQuery` and `HydraCounter` are defined in `asap_sketchlib::common::input`. @@ -34,11 +33,6 @@ fn init_item(k: DataInput, count: C) -> Self ## Insert/Update ```rust -// L2HH -fn update_and_est(&mut self, key: &DataInput, value: i64) -> f64 -fn update_and_est_without_l2(&mut self, key: &DataInput, value: i64) -> f64 -fn clear(&mut self) - // HydraCounter fn update(&mut self, value: &DataInput, weight: Option) fn update_with_hash( @@ -52,9 +46,6 @@ fn update_with_hash( ## Query ```rust -// L2HH -fn l2(&self) -> f64 - // HydraCounter fn query(&self, query: &HydraQuery) -> Result ``` @@ -62,16 +53,13 @@ fn query(&self, query: &HydraQuery) -> Result ## Merge ```rust -// L2HH -fn merge(&mut self, other: &L2HH) - // HydraCounter fn merge(&mut self, other: &HydraCounter) -> Result<(), String> ``` ## Serialization -- `DataInput`, `OwnedDataInput`, `HeapEntry`, `HydraCounter`, and `L2HH` derive serde traits. +- `DataInput`, `OwnedDataInput`, `HeapEntry`, and `HydraCounter` derive serde traits. - `HydraQuery` is query-only and does not derive serde in current code. ## Variants (Core) diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 32e9e7fc..60a15f4b 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -865,7 +865,7 @@ Wire counter types are the base Count Sketch's: **`"i32"`, `"i64"` and `"f64"`** The same encode-side checks apply — the wire-representation duplicate-key check and the `cols` `u32` bound included — so the format never emits bytes it would refuse to read back. §3.7's `NaN` producer-side constraint holds here too: a heap holding the same `NaN` twice is reachable in memory and has no encoding. -`CountSketchWithL2Norm`, which also lives in `countsketch_heap.rs`, is a different algorithm with its own kind_id (§3.19) and is not serialized here. +`CountSketchWithL2Norm` (`countsketch_l2_norm.rs`) is a different algorithm with its own kind_id (§3.19) and is not serialized here. ### 3.11: Elastic payload (`0x0b 0x00`) @@ -1047,7 +1047,7 @@ UnivMon carries the hash-spec group — it hashes, and a consumer must reproduce `bucket_size` is the exact L1 and is not recoverable from the counters, so it is carried. Everything else UnivMon reports — `estimate_l2`, `estimate_entropy`, `estimate_cardinality` — is recomputed from the layers and appears nowhere. -**`L2HH` carries no variant tag.** `L2HH` is a single-variant enum (`COUNT(CountSketchWithL2Norm)`), and `kind_id` fixes the payload's structure, so the wire spells out the CountSketchWithL2Norm state directly with no tag — the same rule that keeps a variant tag out of every other payload. A second `L2HH` variant needs either a new metadata structural param naming the per-layer counter kind, as Count-Min does with `counter_type`, or a new `kind_id`; `0x10 0x00` and `0x11 0x00` are defined as all-`COUNT`. +**Every layer is a CountSketchWithL2Norm.** `kind_id` fixes the per-layer counter, so the wire spells out each layer's CountSketchWithL2Norm state directly with no tag — the same rule that keeps a variant tag out of every other payload. Another per-layer counter needs either a new metadata structural param naming it, as Count-Min does with `counter_type`, or a new `kind_id`. **Emitted order (cross-language contract).** Layers ascend, `0 .. layers`, in every array. Within a layer, heap entries are in descending `count`, ties broken by heap key order (§3.5). This is required, not cosmetic: `TopKHeap`'s array order follows the sift path and does not survive a rebuild, so an unordered payload would re-serialize to different bytes than it decoded from. The heap's digest index (`slots`, `positions`) is rebuilt from the entries, so no index reaches the wire and no crafted payload can point one out of bounds or into a cycle. @@ -1626,5 +1626,5 @@ Retire `portable` once the fixtures cover the rest. - **Q-DERIVED**: state a decoder can recompute exactly is **not carried**, and state it cannot is. Recomputed: DDSketch's `count` from its stores and `zero_count` (§3.8), `ExponentialHistogram`'s `l2_mass` and `merge_norm` (§3.17), Elastic's `heavy_buckets` from the declared bucket count (§3.11), every heap's digest index and array order, and every summary a sketch answers from its own state. Carried: DDSketch's `sum` / `min` / `max`, Elastic's `stale_copies`, UnivMon's `update_mode`, `candidate_complete` and `bucket_size`, UnivMon-Q's `ever_evicted`, `source_id` and `next_sequence`, UniformSampling's `priorities`, `total_seen` and `rng_state`, and CountSketchWithL2Norm's `l2`. Where a cached derived field exists in memory, the encoder rejects a value that disagrees with what the decoder would recompute. - **Q-NIL**: an absent slot is msgpack **nil** (`0xc0`), never a sentinel value. An empty string is a reachable key in Coco (§3.12) and a reachable flow id in Elastic (§3.11), so `""` cannot double as "free"; UnivMon-Q's absent extrema are nil for the same reason (§3.20). Go must model these as nullable, never as a zero value standing in for absent. - **Q-RNG**: RNG state is carried **resumable** when the generator is cheap to reproduce in another language, and opaque otherwise. UniformSampling's `rng_state` is one SplitMix64 word with published constants, so a decoded sampler continues the same draw sequence (§3.13); KLL's `coin` carries the compaction RNG's position and its `compaction_seed` is opaque state Go preserves without interpreting (Q-KLL-SEED). -- **Q-UNIVMON-SCOPE**: `L2HH` carries no variant tag — it is a single-variant enum and `kind_id` fixes the payload's structure — so `0x10 0x00` and `0x11 0x00` are defined as all-`COUNT`, and a second variant needs a new structural param or a new id. `UnivMonPool` is a `Vec` free list of scratch `UnivMon`s — not of pyramids — with nothing an observer could query, so it is not a wire kind and holds no id. +- **Q-UNIVMON-SCOPE**: every layer of `0x10 0x00` and `0x11 0x00` is a CountSketchWithL2Norm with no tag, and another per-layer counter needs a new structural param or a new id. `UnivMonPool` is a `Vec` free list of scratch `UnivMon`s — not of pyramids — with nothing an observer could query, so it is not a wire kind and holds no id. - **Encoding**: metadata + payload are both msgpack; payload is a positional array. Byte-level rules in Section 4. diff --git a/docs/library_map.md b/docs/library_map.md index dedc4a39..9d78eed2 100644 --- a/docs/library_map.md +++ b/docs/library_map.md @@ -3,18 +3,18 @@ ## Core Modules - **`src/common/`** - Foundation for all sketches ([api_common.md](./api/api_common.md)) - - `input.rs` - `DataInput` enum, `OwnedDataInput`, `HeapEntry`, framework enums (`HydraCounter`, `L2HH`, `HydraQuery`) + - `input.rs` - `DataInput` enum, `OwnedDataInput`, `HeapEntry`, framework enums (`HydraCounter`, `HydraQuery`) - `structures/` - High-performance data structures (`Vector1D`, `Vector2D`, `Vector3D`, `BitMatrix`, `CommonHeap`, `MatrixStorage`, `FixedMatrix`) - `heap.rs` - `TopKHeap` convenience wrapper for heavy hitter tracking - `hash.rs` - Hashing utilities (`hash_for_matrix`, `hash64_seeded`, `SEED_LIST`, `UNIVMON_BOTTOM_LAYER_SEED_INDEX`) plus `SketchHasher` for custom hasher injection and `DigestHasher` / `DigestBuildHasher` for maps keyed by an already-hashed digest - `hashspec.rs` - `HashSpec`/`derive_index`/`derive_sign` byte parity with `sketchlib-go`; `numerical.rs` - `NumericalValue`; `structure_utils.rs` - `Nitro`, `NitroContext`, `ToF64` - **`src/sketches/`** - Sketch implementations (status source: [apis.md](./apis.md)) - - `Ready` in API index: `countminsketch.rs`, `countsketch.rs`, `hll.rs`, `kll.rs`, `ddsketch.rs`, `countminsketch_heap.rs`, `countsketch_heap.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `space_saving.rs`, `bloom.rs`, `elastic.rs`, `coco.rs` + - `Ready` in API index: `countminsketch.rs`, `countsketch.rs`, `hll.rs`, `kll.rs`, `ddsketch.rs`, `countminsketch_heap.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `space_saving.rs`, `bloom.rs`, `elastic.rs`, `coco.rs` - `Unstable` in API index: `uniform.rs`, `kmv.rs` - Not in the API index: `count_min_fold.rs`, `count_sketch_fold.rs`, `kll_dynamic.rs`, `octo_delta.rs`, `mode.rs` (`RegularPath` / `FastPath`) - - A sketch on the ASAPv1 wire format uses a `.rs` (algorithm) + `/wire.rs` (serialization) split: `hll.rs`, `countminsketch.rs`, `countminsketch_heap.rs`, `countsketch.rs`, `countsketch_heap.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `kll.rs`, `kll_dynamic.rs`, `ddsketch.rs`, `elastic.rs`, `coco.rs`, `uniform.rs`, `kmv.rs`, `bloom.rs` and `space_saving.rs` (see [asapv1_wire_format.md](./asapv1_wire_format.md)) - - Three of those directories carry a second, shared wire module: `countminsketch_heap/heap_wire.rs` holds the top-k heap encoding CountMinSketchWithHeap and CountSketchWithHeap share, `countminsketch_space_saving/summary_wire.rs` holds the encoding CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving share, and `countsketch_heap/l2hh_wire.rs` holds CountSketchWithL2Norm's own encoding plus the layer sub-payload the UnivMon family inlines + - A sketch on the ASAPv1 wire format uses a `.rs` (algorithm) + `/wire.rs` (serialization) split: `hll.rs`, `countminsketch.rs`, `countminsketch_heap.rs`, `countsketch.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `kll.rs`, `kll_dynamic.rs`, `ddsketch.rs`, `elastic.rs`, `coco.rs`, `uniform.rs`, `kmv.rs`, `bloom.rs` and `space_saving.rs` (see [asapv1_wire_format.md](./asapv1_wire_format.md)) + - Two of those directories carry a second, shared wire module: `countminsketch_heap/heap_wire.rs` holds the top-k heap encoding CountMinSketchWithHeap and CountSketchWithHeap share, and `countminsketch_space_saving/summary_wire.rs` holds the encoding CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving share. `countsketch_l2_norm/wire.rs` also holds the layer sub-payload the UnivMon family inlines - **`src/sketch_framework/`** - Orchestration and serving layers (status source: [apis.md](./apis.md)) - `Ready` in API index: `hydra.rs`, `hashlayer.rs`, `univmon.rs`, `univmon_two_tier.rs`, `nitro.rs`, `eh.rs`, `eh_bucket_sketch.rs`, `octo.rs` diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 0257a9c9..54ebf0e9 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -76,7 +76,7 @@ beside `wire.rs`, declared `pub(crate)` so the other user can import it by path: - `src/sketches/countminsketch_heap/heap_wire.rs` — the top-k heap metadata schema, payload, `key_type` mapping, emitted order and heap rebuild that CountMinSketchWithHeap (`0x03 0x00`) and CountSketchWithHeap (`0x0a 0x00`) share. -- `src/sketches/countsketch_heap/l2hh_wire.rs` — CountSketchWithL2Norm (`0x19 0x00`), plus +- `src/sketches/countsketch_l2_norm/wire.rs` — CountSketchWithL2Norm (`0x19 0x00`), plus the `[counts, l2]` layer sub-payload the whole UnivMon family inlines. Converted: @@ -127,7 +127,7 @@ Converted: hash-spec group. - **KMV** (`src/sketches/kmv/wire.rs`, `0x0e 0x00`) — the retained digests, strictly ascending. -- **CountSketchWithL2Norm** (`src/sketches/countsketch_heap/l2hh_wire.rs`, `0x19 0x00`) — +- **CountSketchWithL2Norm** (`src/sketches/countsketch_l2_norm/wire.rs`, `0x19 0x00`) — `[counts, l2]`. - **Hydra** (`src/sketch_framework/hydra/wire.rs`, `0x07 0x00`-`0x07 0x04`) — one `kind_id` per counter variant; the cells' state tiles or nests one array. diff --git a/docs/tests.md b/docs/tests.md index 3244b4fc..fb64df6a 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -365,9 +365,9 @@ Delta tests: [`src/sketches/countsketch_heap/delta.rs`](../src/sketches/countske ### CountSketchWithL2Norm -Test file: [`src/sketches/countsketch_heap.rs`](../src/sketches/countsketch_heap.rs) +Test file: [`src/sketches/countsketch_l2_norm.rs`](../src/sketches/countsketch_l2_norm.rs) -Wire tests: [`src/sketches/countsketch_heap/l2hh_wire.rs`](../src/sketches/countsketch_heap/l2hh_wire.rs) +Wire tests: [`src/sketches/countsketch_l2_norm/wire.rs`](../src/sketches/countsketch_l2_norm/wire.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | diff --git a/src/common/input.rs b/src/common/input.rs index 7c2d1635..4eae6165 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -9,8 +9,8 @@ use std::{ }; use crate::{ - CountMinSketch, CountSketch, CountSketchWithL2Norm, Ertl, FastPath, HeapCount, HyperLogLog, - Kll, MatrixHash, UnivMon, Vector2D, + CountMinSketch, CountSketch, Ertl, FastPath, HeapCount, HyperLogLog, Kll, MatrixHash, UnivMon, + Vector2D, }; /// Input wrapper for sketch APIs (supports primitive and borrowed values). @@ -280,66 +280,6 @@ impl Hash for OwnedDataInput { } } -/// Counter wrapper for UnivMon (currently backed by CountSketchWithL2Norm). -#[derive(Clone, Debug, Serialize, Deserialize)] -pub enum L2HH { - /// Count-sketch-based heavy-hitter tracker with L2 support. - COUNT(CountSketchWithL2Norm), -} - -impl L2HH { - /// Updates the counter and returns the current estimate. - pub fn update_and_est(&mut self, key: &DataInput, value: i64) -> f64 { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.update_weighted_and_estimate(key, value), - } - } - - /// Updates the counter without refreshing the cached L2 value. - pub fn update_and_est_without_l2(&mut self, key: &DataInput, value: i64) -> f64 { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.fast_update_and_est_without_l2(key, value), - } - } - - /// Returns the current L2 estimate. - pub fn l2(&self) -> f64 { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.l2(), - } - } - - /// Returns the current frequency estimate without changing the sketch. - pub fn estimate(&self, key: &DataInput) -> f64 { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.estimate(key), - } - } - - /// Applies a counter delta promoted by an OctoSketch worker. - pub fn apply_delta(&mut self, delta: crate::octo_delta::CountSketchCellDelta) { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.apply_delta(delta), - } - } - - /// Merges another counter of the same kind into this one. - pub fn merge(&mut self, other: &L2HH) { - match (self, other) { - (L2HH::COUNT(self_count), L2HH::COUNT(other_count)) => { - self_count.merge(other_count); - } - } - } - - /// Resets all counters to zero without reallocating. - pub fn clear(&mut self) { - match self { - L2HH::COUNT(count_l2hh) => count_l2hh.clear(), - } - } -} - /// Query type for Hydra sketches. #[derive(Clone, Debug)] pub enum HydraQuery<'a> { diff --git a/src/common/mod.rs b/src/common/mod.rs index 099b9c22..db0099a7 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -43,9 +43,7 @@ pub use hash::{ hash64_seeded_owned, hash128_seeded, hash128_seeded_owned, }; pub use heap::{HeapCount, HeapCounter, TopKHeap}; -pub use input::{ - DataInput, HeapEntry, L2HH, OwnedDataInput, heap_item_to_sketch_input, input_to_owned, -}; +pub use input::{DataInput, HeapEntry, OwnedDataInput, heap_item_to_sketch_input, input_to_owned}; pub use numerical::NumericalValue; pub use precompute_hash::PRECOMPUTED_HASH; pub use precompute_sample::LN_ONE_MINUS_U_TABLE; diff --git a/src/message_pack_format/native/countsketch_topk.rs b/src/message_pack_format/native/countsketch_topk.rs index 9a00b89c..523ac0b2 100644 --- a/src/message_pack_format/native/countsketch_topk.rs +++ b/src/message_pack_format/native/countsketch_topk.rs @@ -1,7 +1,7 @@ -//! Native MessagePack codec impl for [`crate::sketches::countsketch_heap::CountSketchWithL2Norm`]. +//! Native MessagePack codec impl for [`crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm`]. use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::countsketch_heap::CountSketchWithL2Norm; +use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use crate::{HashProfile, SketchHasher}; impl MessagePackCodec for CountSketchWithL2Norm { diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index 31755766..55e0852e 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use super::super::sketches::*; use super::UnivMon; -use crate::sketches::countsketch_heap::CountSketchWithL2Norm; +use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; pub(crate) mod wire; diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index 509ef059..d0aceeaf 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -43,7 +43,8 @@ use crate::octo_delta::{ use crate::sketch_framework::univmon::UnivMonDeltaFidelity; use crate::sketches::coco::{CocoRng, CocoSketch}; use crate::sketches::countminsketch_heap::CountMinSketchWithHeap; -use crate::sketches::countsketch_heap::{CountSketchWithHeap, CountSketchWithL2Norm}; +use crate::sketches::countsketch_heap::CountSketchWithHeap; +use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use crate::sketches::elastic::{ElasticSketch, LAMBDA}; use crate::{ Classic, CountMinSketch, CountMinSketchCellDelta, CountSketch, CountSketchCellDelta, DataInput, @@ -4044,7 +4045,6 @@ mod runtime_tests { #[test] fn run_octo_univmon_reaches_the_deepest_layer() { - use crate::L2HH; // The structural half of `a_flat_threshold_starves_the_deep_univmon_layers`, // checked against the shipped pipeline rather than the replica that // test needs in order to run the flat counterfactual at all. @@ -4055,7 +4055,7 @@ mod runtime_tests { let result = run_octo(&inputs, &config(1), plan.clone(), || plan.aggregator(heap)); let deepest = layers - 1; - let L2HH::COUNT(inner) = &result.parent.sketch.layer_sketches[deepest]; + let inner = &result.parent.sketch.layer_sketches[deepest]; let nonzero = (0..inner.rows()) .flat_map(|r| (0..inner.cols()).map(move |c| (r, c))) .filter(|(r, c)| inner.as_storage().query_one_counter(*r, *c) != 0) @@ -4529,7 +4529,6 @@ mod runtime_tests { #[cfg(test)] mod univmon_tests { use super::*; - use crate::L2HH; const HEAP: usize = 32; const ROWS: usize = 3; @@ -4537,7 +4536,7 @@ mod univmon_tests { const LAYERS: usize = 6; fn layer_counters(sketch: &UnivMon, layer: usize) -> Vec { - let L2HH::COUNT(inner) = &sketch.layer_sketches[layer]; + let inner = &sketch.layer_sketches[layer]; (0..inner.rows()) .flat_map(|r| (0..inner.cols()).map(move |c| (r, c))) .map(|(r, c)| inner.as_storage().query_one_counter(r, c)) diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 5f0871ac..e6e8edec 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -28,9 +28,9 @@ use crate::common::{ DataInput, OwnedDataInput, SEED_LIST, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, hash64_seeded_owned, heap_item_to_sketch_input, }; -use crate::common::{L2HH, Vector1D}; +use crate::common::Vector1D; use crate::octo_delta::UnivMonCellDelta; -use crate::sketches::countsketch_heap::CountSketchWithL2Norm; +use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; @@ -53,7 +53,7 @@ pub(crate) enum UnivMonUpdateMode { /// UnivMon sketch pyramid for multi-metric stream estimation. pub struct UnivMon { /// Per-layer L2/heavy-hitter sketches. - pub layer_sketches: Vector1D, + pub layer_sketches: Vector1D, /// Per-layer heavy-hitter heaps. pub layer_heaps: Vector1D, /// Number of pyramid layers. @@ -138,12 +138,8 @@ impl UnivMon { layers <= UNIVMON_MAX_LAYERS, "layer count must be at most UNIVMON_MAX_LAYERS {UNIVMON_MAX_LAYERS}" ); - let sk_vec: Vec = (0..layers) - .map(|i| { - L2HH::COUNT(CountSketchWithL2Norm::with_dimensions_and_seed_index( - rows, cols, i, - )) - }) + let sk_vec: Vec = (0..layers) + .map(|i| CountSketchWithL2Norm::with_dimensions_and_seed_index(rows, cols, i)) .collect(); let hh_vec: Vec = (0..layers).map(|_| TopKHeap::new(heap_size)).collect(); @@ -268,7 +264,7 @@ impl UnivMon { #[inline(always)] fn update(&mut self, key: &DataInput, value: i64, bottom_layer_num: usize) { for i in 0..=bottom_layer_num { - let count = self.layer_sketches[i].update_and_est(key, value); + let count = self.layer_sketches[i].update_weighted_and_estimate(key, value); if !self.layer_heaps[i].update(key, count as i64) { self.candidate_complete[i] = false; } @@ -297,7 +293,7 @@ impl UnivMon { self.begin_update(weight, UnivMonUpdateMode::Terminal); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); let bottom_layer_num = self.find_bottom_layer_num(h, self.layers); - let count = self.layer_sketches[bottom_layer_num].update_and_est(key, weight); + let count = self.layer_sketches[bottom_layer_num].update_weighted_and_estimate(key, weight); if !self.layer_heaps[bottom_layer_num].update(key, count as i64) { self.candidate_complete[bottom_layer_num] = false; } @@ -573,7 +569,7 @@ mod tests { let deepest = UnivMon::new(4, 2, 16, UNIVMON_MAX_LAYERS); let seeds: Vec = (0..UNIVMON_MAX_LAYERS) .map(|layer| { - let L2HH::COUNT(counter) = &deepest.layer_sketches[layer]; + let counter = &deepest.layer_sketches[layer]; counter.seed_index() }) .collect(); diff --git a/src/sketch_framework/univmon/wire.rs b/src/sketch_framework/univmon/wire.rs index a3fc4d64..c2ff2bfb 100644 --- a/src/sketch_framework/univmon/wire.rs +++ b/src/sketch_framework/univmon/wire.rs @@ -17,7 +17,7 @@ //! //! Each layer is a `CountSketchWithL2Norm` plus an `TopKHeap`. Their state is written //! straight into UnivMon's own positional array through -//! [`l2hh_wire::layer_state`] and [`heap_entries`]; no layer carries an +//! [`l2_norm_wire::layer_state`] and [`heap_entries`]; no layer carries an //! envelope, a magic or a metadata map of its own. //! //! ## The hash profile is the crate default @@ -42,8 +42,10 @@ use crate::sketches::countminsketch_heap::heap_wire::{ EMPTY_KEY_TYPE, check_distinct_keys, check_empty_key_type, heap_entries, key_type_of, rebuild_heap, }; -use crate::sketches::countsketch_heap::l2hh_wire; -use crate::{DefaultXxHasher, HashProfile, L2HH, OwnedDataInput, TopKHeap, Vector1D}; +use crate::sketches::countsketch_l2_norm::wire as l2_norm_wire; +use crate::{ + CountSketchWithL2Norm, DefaultXxHasher, HashProfile, OwnedDataInput, TopKHeap, Vector1D, +}; use super::{UNIVMON_MAX_LAYERS, UnivMon, UnivMonUpdateMode}; @@ -187,7 +189,7 @@ pub(crate) struct PyramidState<'a> { /// layer's seed index is not its position, when a layer's heap holds the same /// wire key twice, or when the heaps hold more entries than `u32` can name. pub(crate) fn pyramid_state<'a>( - sketches: &'a Vector1D, + sketches: &'a Vector1D, heaps: &'a Vector1D, geometry: &[(usize, usize)], bucket_size: usize, @@ -223,7 +225,7 @@ pub(crate) fn pyramid_state<'a>( update_mode, }; for (layer, &(rows, cols)) in geometry.iter().enumerate() { - let L2HH::COUNT(counter) = &sketches[layer]; + let counter = &sketches[layer]; if (counter.rows(), counter.cols()) != (rows, cols) { return Err(RmpEncodeError::Syntax(format!( "ASAPv1 UnivMon envelope: layer {layer} is {}x{} against the declared {rows}x{cols}", @@ -237,7 +239,7 @@ pub(crate) fn pyramid_state<'a>( counter.seed_index() ))); } - let (counts, l2) = l2hh_wire::layer_state(counter)?; + let (counts, l2) = l2_norm_wire::layer_state(counter)?; state.counts.extend_from_slice(counts); state.l2.extend_from_slice(l2); let entries = heap_entries(&heaps[layer]); @@ -462,7 +464,13 @@ pub(crate) fn decode_pyramid( } /// The counters and heaps one decoded pyramid rebuilds into. -pub(crate) type DecodedLayers = (Vector1D, Vector1D, Vec, usize, u8); +pub(crate) type DecodedLayers = ( + Vector1D, + Vector1D, + Vec, + usize, + u8, +); /// Seats the decoded layers, rebuilding every counter matrix and heap index. /// @@ -491,7 +499,7 @@ pub(crate) fn rebuild_layers( let mut cells = 0usize; let mut accumulators = 0usize; for &(rows, cols) in geometry { - l2hh_wire::check_dimensions(rows, cols).map_err(complain)?; + l2_norm_wire::check_dimensions(rows, cols).map_err(complain)?; cells = cells .checked_add(rows.checked_mul(cols).ok_or_else(|| { complain(format!("UnivMon layer geometry {rows}x{cols} overflows")) @@ -533,13 +541,13 @@ pub(crate) fn rebuild_layers( for (layer, &(rows, cols)) in geometry.iter().enumerate() { let next_cell = cell + rows * cols; let next_accumulator = accumulator + rows; - sketches.push(L2HH::COUNT(l2hh_wire::rebuild_layer::( + sketches.push(l2_norm_wire::rebuild_layer::( rows, cols, layer, &decoded.counts[cell..next_cell], &decoded.l2[accumulator..next_accumulator], - )?)); + )?); cell = next_cell; accumulator = next_accumulator; let run: Vec<(OwnedDataInput, i64)> = entries @@ -973,10 +981,11 @@ mod tests { let count_sketch = crate::CountSketch::, RegularPath>::with_dimensions(3, 8) .to_asapv1() .expect("serialize Count Sketch"); - let count_l2hh = - crate::sketches::countsketch_heap::CountSketchWithL2Norm::::with_dimensions(2, 8) - .to_asapv1() - .expect("serialize CountSketchWithL2Norm"); + let count_l2hh = crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm::< + DefaultXxHasher, + >::with_dimensions(2, 8) + .to_asapv1() + .expect("serialize CountSketchWithL2Norm"); let pyramid = crate::UnivMonTwoTier::new(4, 1, 2, 8, 2, 4, 2) .to_asapv1() .expect("serialize UnivMonTwoTier"); @@ -1067,23 +1076,20 @@ mod tests { #[test] fn univmon_rejects_serializing_an_inconsistent_pyramid() { let mut wrong_layer = UnivMon::new(4, 2, 16, 3); - wrong_layer.layer_sketches[1] = L2HH::COUNT( - crate::sketches::countsketch_heap::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 16, 1), - ); + wrong_layer.layer_sketches[1] = + crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 16, 1); assert!(wrong_layer.to_asapv1().is_ok()); - wrong_layer.layer_sketches[1] = L2HH::COUNT( - crate::sketches::countsketch_heap::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 32, 1), - ); + wrong_layer.layer_sketches[1] = + crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 32, 1); assert!( wrong_layer.to_asapv1().is_err(), "a layer that is not the declared size must not serialize" ); let mut wrong_seed = UnivMon::new(4, 2, 16, 3); - wrong_seed.layer_sketches[1] = L2HH::COUNT( - crate::sketches::countsketch_heap::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 16, 5), - ); + wrong_seed.layer_sketches[1] = + crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 16, 5); assert!( wrong_seed.to_asapv1().is_err(), "a layer hashing at another layer's seed index must not serialize" diff --git a/src/sketch_framework/univmon_q/wire.rs b/src/sketch_framework/univmon_q/wire.rs index 0567f028..2422c80f 100644 --- a/src/sketch_framework/univmon_q/wire.rs +++ b/src/sketch_framework/univmon_q/wire.rs @@ -812,10 +812,11 @@ mod tests { let count_sketch = crate::CountSketch::, RegularPath>::with_dimensions(3, 8) .to_asapv1() .expect("serialize Count Sketch"); - let count_l2hh = - crate::sketches::countsketch_heap::CountSketchWithL2Norm::::with_dimensions(2, 8) - .to_asapv1() - .expect("serialize CountSketchWithL2Norm"); + let count_l2hh = crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm::< + DefaultXxHasher, + >::with_dimensions(2, 8) + .to_asapv1() + .expect("serialize CountSketchWithL2Norm"); let univmon = crate::UnivMon::new(4, 2, 8, 2) .to_asapv1() .expect("serialize UnivMon"); diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index fb46ba90..02a52456 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -11,9 +11,9 @@ use crate::common::{ DataInput, OwnedDataInput, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, hash64_seeded_owned, heap_item_to_sketch_input, }; -use crate::common::{L2HH, Vector1D}; +use crate::common::Vector1D; use crate::sketch_framework::univmon::{UNIVMON_MAX_LAYERS, UnivMonUpdateMode}; -use crate::sketches::countsketch_heap::CountSketchWithL2Norm; +use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use std::collections::{HashMap, HashSet}; mod wire; @@ -107,7 +107,7 @@ const DEFAULT_LAYERS: usize = 16; /// Optimized UnivMon variant with separate elephant and mouse layers. pub struct UnivMonTwoTier { /// Per-layer L2/heavy-hitter sketches. - pub layer_sketches: Vector1D, + pub layer_sketches: Vector1D, /// Per-layer heavy-hitter heaps. pub layer_heaps: Vector1D, /// Total number of layers. @@ -155,29 +155,27 @@ impl UnivMonTwoTier { layers <= UNIVMON_MAX_LAYERS, "layer count must be at most UNIVMON_MAX_LAYERS {UNIVMON_MAX_LAYERS}" ); - let sk_vec: Vec = if layers <= elephant_layers { + let sk_vec: Vec = if layers <= elephant_layers { (0..layers) .map(|i| { - L2HH::COUNT(CountSketchWithL2Norm::with_dimensions_and_seed_index( + CountSketchWithL2Norm::with_dimensions_and_seed_index( elephant_rows, elephant_cols, i, - )) + ) }) .collect() } else { (0..elephant_layers) .map(|i| { - L2HH::COUNT(CountSketchWithL2Norm::with_dimensions_and_seed_index( + CountSketchWithL2Norm::with_dimensions_and_seed_index( elephant_rows, elephant_cols, i, - )) + ) }) .chain((elephant_layers..layers).map(|i| { - L2HH::COUNT(CountSketchWithL2Norm::with_dimensions_and_seed_index( - mouse_rows, mouse_cols, i, - )) + CountSketchWithL2Norm::with_dimensions_and_seed_index(mouse_rows, mouse_cols, i) })) .collect() }; @@ -243,7 +241,7 @@ impl UnivMonTwoTier { let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); let bottom = self.find_bottom_layer_num(h); for i in 0..=bottom { - let count = self.layer_sketches[i].update_and_est(key, weight); + let count = self.layer_sketches[i].update_weighted_and_estimate(key, weight); if !self.layer_heaps[i].update(key, count as i64) { self.candidate_complete[i] = false; } @@ -258,7 +256,7 @@ impl UnivMonTwoTier { self.begin_update(weight, UnivMonUpdateMode::Terminal); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); let bottom = self.find_bottom_layer_num(h); - let count = self.layer_sketches[bottom].update_and_est(key, weight); + let count = self.layer_sketches[bottom].update_weighted_and_estimate(key, weight); if !self.layer_heaps[bottom].update(key, count as i64) { self.candidate_complete[bottom] = false; } @@ -659,8 +657,8 @@ mod tests { assert_eq!(left.total_weight, 1000); assert!(left.estimate_entropy().is_finite()); for level in 0..left.layers { - let L2HH::COUNT(merged_count) = &left.layer_sketches[level]; - let L2HH::COUNT(one_pass_count) = &one_pass.layer_sketches[level]; + let merged_count = &left.layer_sketches[level]; + let one_pass_count = &one_pass.layer_sketches[level]; assert_eq!( merged_count.as_storage().as_slice(), one_pass_count.as_storage().as_slice(), diff --git a/src/sketch_framework/univmon_two_tier/wire.rs b/src/sketch_framework/univmon_two_tier/wire.rs index 227305e2..147688e0 100644 --- a/src/sketch_framework/univmon_two_tier/wire.rs +++ b/src/sketch_framework/univmon_two_tier/wire.rs @@ -291,10 +291,10 @@ mod tests { use super::*; use crate::sketch_framework::univmon::UNIVMON_MAX_LAYERS; use crate::sketch_framework::univmon::wire::PyramidPayload; - use crate::sketches::countsketch_heap::CountSketchWithL2Norm; + use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use crate::{ - DEFAULT_CANONICAL_SEED_INDEX, DataInput, L2HH, MATRIX_MAX_ROWS, OwnedDataInput, - RegularPath, SketchHasher, Vector2D, + DEFAULT_CANONICAL_SEED_INDEX, DataInput, MATRIX_MAX_ROWS, OwnedDataInput, RegularPath, + SketchHasher, Vector2D, }; use rmp_serde::from_slice; @@ -373,8 +373,8 @@ mod tests { let encoded = pyramid.to_asapv1().expect("serialize"); let decoded = UnivMonTwoTier::from_asapv1(&encoded).expect("decode"); for layer in 0..pyramid.layers { - let L2HH::COUNT(original) = &pyramid.layer_sketches[layer]; - let L2HH::COUNT(rebuilt) = &decoded.layer_sketches[layer]; + let original = &pyramid.layer_sketches[layer]; + let rebuilt = &decoded.layer_sketches[layer]; let expected = if layer < 2 { (3, 16) } else { (2, 8) }; assert_eq!((rebuilt.rows(), rebuilt.cols()), expected); assert_eq!(rebuilt.seed_index(), layer); @@ -425,7 +425,7 @@ mod tests { let encoded = pyramid.to_asapv1().expect("serialize"); let decoded = UnivMonTwoTier::from_asapv1(&encoded).expect("decode"); for layer in 0..3 { - let L2HH::COUNT(rebuilt) = &decoded.layer_sketches[layer]; + let rebuilt = &decoded.layer_sketches[layer]; assert_eq!((rebuilt.rows(), rebuilt.cols()), (2, 16)); } assert_eq!(decoded.to_asapv1().expect("re-serialize"), encoded); @@ -806,18 +806,16 @@ mod tests { #[test] fn pyramid_rejects_serializing_an_inconsistent_layout() { let mut wrong_tier = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); - wrong_tier.layer_sketches[3] = L2HH::COUNT( - CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 16, 3), - ); + wrong_tier.layer_sketches[3] = + CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 16, 3); assert!( wrong_tier.to_asapv1().is_err(), "a mouse layer holding elephant dimensions must not serialize" ); let mut wrong_seed = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); - wrong_seed.layer_sketches[3] = L2HH::COUNT( - CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 8, 0), - ); + wrong_seed.layer_sketches[3] = + CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 8, 0); assert!( wrong_seed.to_asapv1().is_err(), "a layer hashing at another layer's seed index must not serialize" diff --git a/src/sketches/countsketch_heap.rs b/src/sketches/countsketch_heap.rs index e94a82fe..b9dcb285 100644 --- a/src/sketches/countsketch_heap.rs +++ b/src/sketches/countsketch_heap.rs @@ -1,26 +1,17 @@ -//! Top-k heavy-hitter variants of the Count Sketch family. +//! Top-k heavy-hitter variant of the Count Sketch family. //! -//! - [`CountSketchWithHeap`]: pairs a [`CountSketch`] sketch with an [`TopKHeap`] for automatic top-k tracking. -//! Every insertion updates both the frequency sketch and the heap, mirroring -//! the pattern used by `CountMinSketchWithHeap` but with Count Sketch (median estimator). -//! - [`CountSketchWithL2Norm`]: Count Sketch augmented with per-row L2 norm tracking for -//! heavy-hitter detection. +//! [`CountSketchWithHeap`] pairs a [`CountSketch`] sketch with a [`TopKHeap`] for automatic top-k tracking. +//! Every insertion updates both the frequency sketch and the heap, mirroring +//! the pattern used by `CountMinSketchWithHeap` but with Count Sketch (median estimator). -use serde::{Deserialize, Serialize}; -use std::marker::PhantomData; - -use crate::common::structures::matrix_storage::cols_mask_bits; -use crate::octo_delta::CountSketchCellDelta; use crate::sketches::countsketch::CountSketchCounter; use crate::{ CountSketch, DataInput, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, DefaultXxHasher, FastPath, FixedMatrix, HeapCounter, MatrixStorage, QuickMatrixI64, QuickMatrixI128, - RegularPath, SketchHasher, TopKHeap, Vector1D, Vector2D, compute_median_inline_f64, - heap_item_to_sketch_input, + RegularPath, SketchHasher, TopKHeap, Vector2D, heap_item_to_sketch_input, }; mod delta; -pub(crate) mod l2hh_wire; mod wire; pub use delta::CountSketchWithHeapDelta; @@ -90,8 +81,6 @@ impl CountSketchHeapCounter for f64 { } const DEFAULT_HEAP_SIZE: usize = 32; -const DEFAULT_ROWS: usize = 3; -const DEFAULT_COLS: usize = 4096; /// A Count Sketch paired with a top-k heavy-hitter heap. /// @@ -1015,364 +1004,3 @@ mod tests { ); } } - -// ===================================================================== -// CountSketchWithL2Norm — Count Sketch with per-row L2 norm tracking for -// heavy-hitter detection. Moved here from `count.rs` so the algorithm -// file stays focused on plain Count Sketch. -// ===================================================================== - -/// Count Sketch augmented with per-row L2 norm tracking for heavy-hitter detection. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "")] -pub struct CountSketchWithL2Norm { - counts: Vector2D, - row_l2_squared: Vector1D, - rows: usize, - cols: usize, - seed_index: usize, - #[serde(skip)] - _hasher: PhantomData, -} - -impl CountSketchWithL2Norm { - /// Checks that a `rows` x `cols` `CountSketchWithL2Norm` fits its one 128-bit key hash. - /// - /// Row `r` reads its column from bits `[r * mask_bits, (r + 1) * mask_bits)` - /// and its sign from bit `127 - r`, where `mask_bits = cols_mask_bits(cols)`. - /// Each row therefore claims `mask_bits + 1` bits, and the fields are in range - /// and pairwise disjoint (so every column and sign is an independent hash - /// draw) exactly while `rows * (mask_bits + 1) <= 128`. Past that, deep rows' - /// columns reuse shallow rows' sign bits, and from `rows * mask_bits >= 128` - /// the shift runs off the hash. - pub(crate) fn check_hash_bits(rows: usize, cols: usize) -> Result<(), String> { - let mask_bits = cols_mask_bits(cols.max(1)) as usize; - let bits_required = mask_bits.saturating_add(1).saturating_mul(rows); - if bits_required > 128 { - return Err(format!( - "CountSketchWithL2Norm rows={rows}, cols={cols} needs {bits_required} hash bits \ - ({mask_bits} column bits + 1 sign bit per row), past the 128 the key hash carries" - )); - } - Ok(()) - } - - /// Column and sign `CountSketchWithL2Norm` uses for `row`, given its 128-bit key hash. - /// - /// Shared with the OctoSketch worker sketches so a worker and its parent can - /// never disagree about which cell a key lands in. - #[inline(always)] - pub fn cell_for_row(hashed_val: u128, row: usize, cols: usize, mask_bits: u32) -> (usize, i64) { - let mask = (1u128 << mask_bits) - 1; - let hashed = (hashed_val >> (row * mask_bits as usize)) & mask; - let col = (hashed as usize) % cols; - let bit = ((hashed_val >> (127 - row)) & 1) as i64; - (col, -(1 - 2 * bit)) - } -} - -// Default CountSketchWithL2Norm configuration. -impl Default for CountSketchWithL2Norm { - fn default() -> Self { - Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) - } -} - -// CountSketchWithL2Norm constructors and operations. -impl CountSketchWithL2Norm { - /// Creates a sketch with the requested number of rows and columns. - pub fn with_dimensions(rows: usize, cols: usize) -> Self { - Self::with_dimensions_and_seed_index(rows, cols, 0) - } - - /// Creates a sketch with the requested dimensions and a custom seed offset. - /// - /// # Panics - /// - /// When `rows * (cols_mask_bits(cols) + 1) > 128`: the rows' column and - /// sign fields would overlap in the one 128-bit key hash. - pub fn with_dimensions_and_seed_index(rows: usize, cols: usize, seed_index: usize) -> Self { - if let Err(problem) = CountSketchWithL2Norm::check_hash_bits(rows, cols) { - panic!("{problem}"); - } - let mut sk = CountSketchWithL2Norm { - counts: Vector2D::unfilled(rows, cols), - row_l2_squared: Vector1D::filled(rows, 0), - rows, - cols, - seed_index, - _hasher: PhantomData, - }; - sk.counts.fill(0); - sk - } - - /// Number of rows in the sketch. - pub fn rows(&self) -> usize { - self.rows - } - - /// Number of columns in the sketch. - pub fn cols(&self) -> usize { - self.cols - } - - /// Exposes the backing matrix for inspection/testing. - pub fn as_storage(&self) -> &Vector2D { - &self.counts - } - - /// Mutable access used internally for testing scenarios. - pub fn as_storage_mut(&mut self) -> &mut Vector2D { - &mut self.counts - } - - /// Merges another sketch while asserting compatible dimensions. - pub fn merge(&mut self, other: &Self) { - assert_eq!( - (self.rows, self.cols), - (other.rows, other.cols), - "dimension mismatch while merging CountSketchWithL2Norm sketches" - ); - assert_eq!( - self.seed_index, other.seed_index, - "seed mismatch while merging CountSketchWithL2Norm sketches" - ); - - for i in 0..self.rows { - let mut row_l2 = 0_i128; - for j in 0..self.cols { - self.counts[i][j] += other.counts[i][j]; - let counter = self.counts[i][j] as i128; - row_l2 = row_l2.saturating_add(counter.saturating_mul(counter)); - } - self.row_l2_squared[i] = row_l2.min(i64::MAX as i128) as i64; - } - } - - /// Resets all counters and L2 accumulators to zero without reallocating. - pub fn clear(&mut self) { - self.counts.fill(0); - self.row_l2_squared.fill(0); - } - - /// Inserts with hash optimization - computes hash once and reuses it. - /// due to the limitation of seeds, use fast_insert only - pub fn update_weighted(&mut self, val: &DataInput, c: i64) { - let hashed_val = H::hash128_seeded(self.seed_index, val); - self.update_weighted_with_hash(hashed_val, c); - } - - /// Inserts with hash optimization using precomputed hash value. - pub fn update_weighted_with_hash(&mut self, hashed_val: u128, weight: i64) { - let mask_bits = self.counts.mask_bits(); - for i in 0..self.rows { - let (idx, sign_bit) = - CountSketchWithL2Norm::cell_for_row(hashed_val, i, self.cols, mask_bits); - let old_value = self.counts.query_one_counter(i, idx); - self.write_counter(i, idx, old_value + sign_bit * weight); - } - } - - /// Writes one counter and carries the row's L2 accumulator with it. - /// - /// Saturating i128 intermediate so extreme counts degrade gracefully - /// instead of wrapping (matches merge-path semantics, which also clamps to - /// [0, i64::MAX]: L2^2 can never go negative, even when a decrement follows - /// prior saturation). - #[inline(always)] - fn write_counter(&mut self, row: usize, col: usize, new_value: i64) { - let old_value = self.counts.query_one_counter(row, col); - self.counts[row][col] = new_value; - let old_l2 = self.row_l2_squared.as_slice()[row] as i128; - let new_l2 = old_l2 + (new_value as i128) * (new_value as i128) - - (old_value as i128) * (old_value as i128); - self.row_l2_squared[row] = new_l2.clamp(0, i64::MAX as i128) as i64; - } - - /// Seed offset this sketch hashes with; UnivMon gives each layer its own. - pub fn seed_index(&self) -> usize { - self.seed_index - } - - /// Column-mask width used to slice the 128-bit hash into per-row indices. - pub fn mask_bits(&self) -> u32 { - self.counts.mask_bits() - } - - /// Applies a counter delta promoted by an OctoSketch worker. - /// - /// The row's L2 accumulator is carried along exactly, so a parent fed only - /// deltas reports the same `l2` as one fed the raw stream. - pub fn apply_delta(&mut self, delta: CountSketchCellDelta) { - let (row, col) = (delta.row as usize, delta.col as usize); - let new_value = self.counts.query_one_counter(row, col) + delta.count as i64; - self.write_counter(row, col, new_value); - } - - /// Inserts without L2 update using precomputed hash value. - pub fn update_weighted_with_hash_without_l2(&mut self, hashed_val: u128, weight: i64) { - let mask_bits = self.counts.mask_bits() as usize; - let mask = (1u128 << mask_bits) - 1; - let mut shift_amount = 0; - let mut sign_bit_pos = 127; - - for i in 0..self.rows { - let hashed = (hashed_val >> shift_amount) & mask; - let idx = (hashed as usize) % self.cols; - let bit = ((hashed_val >> sign_bit_pos) & 1) as i64; - let sign_bit = -(1 - 2 * bit); - - self.counts[i][idx] += sign_bit * weight; - - shift_amount += mask_bits; - sign_bit_pos -= 1; - } - } - - /// Update and estimate with hash optimization. - /// due to the limitation of seeds, use fast_insert only - pub fn update_weighted_and_estimate(&mut self, val: &DataInput, weight: i64) -> f64 { - let hashed_val = H::hash128_seeded(self.seed_index, val); - self.update_weighted_with_hash(hashed_val, weight); - self.estimate_with_hash(hashed_val) - } - - /// Update and estimate without L2 with hash optimization. - /// due to the limitation of seeds, use fast_insert only - pub fn fast_update_and_est_without_l2(&mut self, val: &DataInput, c: i64) -> f64 { - let hashed_val = H::hash128_seeded(self.seed_index, val); - self.update_weighted_with_hash_without_l2(hashed_val, c); - self.estimate_with_hash(hashed_val) - } - - /// Returns the median estimate of the squared L2 norm across rows. - pub fn l2_squared(&self) -> f64 { - let mut values: Vec = self.row_l2_squared.as_slice()[..self.rows] - .iter() - .map(|&v| v as f64) - .collect(); - compute_median_inline_f64(&mut values) - } - - /// Returns the estimated L2 norm (square root of `l2_squared`). - pub fn l2(&self) -> f64 { - let l2 = self.l2_squared(); - l2.sqrt() - } - - /// Returns the frequency estimate with hash optimization. - /// due to the limitation of seeds, use fast_insert only - pub fn estimate(&self, val: &DataInput) -> f64 { - let hashed_val = H::hash128_seeded(self.seed_index, val); - self.estimate_with_hash(hashed_val) - } - - /// Returns the frequency estimate using precomputed hash value. - /// due to the limitation of seeds, use fast_insert only - pub fn estimate_with_hash(&self, hashed_val: u128) -> f64 { - let mask_bits = self.counts.mask_bits() as usize; - let mask = (1u128 << mask_bits) - 1; - let mut lst = Vec::with_capacity(self.rows); - let mut shift_amount = 0; - let mut sign_bit_pos = 127; - - for i in 0..self.rows { - let hashed = (hashed_val >> shift_amount) & mask; - let idx = (hashed as usize) % self.cols; - let bit = ((hashed_val >> sign_bit_pos) & 1) as i64; - let sign_bit = -(1 - 2 * bit); - let counter = self.counts.query_one_counter(i, idx); - lst.push((sign_bit * counter) as f64); - - shift_amount += mask_bits; - sign_bit_pos -= 1; - } - compute_median_inline_f64(&mut lst[..]) - } -} - -#[cfg(test)] -mod tests_count_l2_hh { - use super::*; - - /// 11 rows x (11 column bits + 1 sign bit) = 132 bits of one 128-bit - /// hash: the deep rows' columns would reuse the shallow rows' signs. - #[test] - #[should_panic(expected = "hash bits")] - fn countl2hh_rejects_column_bits_that_reach_the_sign_bits() { - let _: CountSketchWithL2Norm = - CountSketchWithL2Norm::with_dimensions_and_seed_index(11, 2048, 0); - } - - /// 10 x 2048 spends 120 bits and is accepted. - #[test] - fn countl2hh_accepts_the_widest_disjoint_geometry() { - let sketch: CountSketchWithL2Norm = - CountSketchWithL2Norm::with_dimensions_and_seed_index(10, 2048, 0); - assert_eq!((sketch.rows(), sketch.cols()), (10, 2048)); - } - - #[test] - fn countl2hh_estimates_and_l2_are_consistent() { - let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); - let key = DataInput::Str("gamma"); - - let est_after_first = sketch.update_weighted_and_estimate(&key, 5); - assert_eq!(est_after_first, 5.0); - - let est_after_second = sketch.update_weighted_and_estimate(&key, -2); - assert_eq!(est_after_second, 3.0); - - let l2 = sketch.l2(); - assert!(l2 >= 3.0, "expected non-trivial l2, got {l2}"); - } - - #[test] - fn countl2hh_merge_combines_frequency_vectors() { - let mut left: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); - let mut right: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); - let key = DataInput::U32(42); - - left.update_weighted(&key, 4); - assert_eq!(left.estimate(&key), 4.0); - right.update_weighted(&key, 9); - assert_eq!(right.estimate(&key), 9.0); - - left.merge(&right); - assert_eq!(left.estimate(&key), 13.0); - assert_eq!(left.l2(), 13.0); - } - - #[test] - fn countl2hh_round_trip_serialization() { - let mut sketch: CountSketchWithL2Norm = - CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 32, 7); - let key = DataInput::Str("serialize"); - - sketch.update_weighted(&key, 11); - sketch.update_weighted(&key, -3); - let base_est = sketch.estimate(&key); - let base_l2 = sketch.l2(); - - let encoded = sketch - .to_asapv1() - .expect("serialize CountSketchWithL2Norm into MessagePack"); - assert!(!encoded.is_empty(), "serialized bytes should not be empty"); - let data = encoded.clone(); - - let decoded: CountSketchWithL2Norm = CountSketchWithL2Norm::from_asapv1(&data) - .expect("deserialize CountSketchWithL2Norm from MessagePack"); - - assert_eq!(sketch.rows(), decoded.rows()); - assert_eq!(sketch.cols(), decoded.cols()); - assert!( - (decoded.estimate(&key) - base_est).abs() < f64::EPSILON, - "estimate changed after round trip" - ); - assert!( - (decoded.l2() - base_l2).abs() < f64::EPSILON, - "L2 changed after round trip" - ); - } -} diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs new file mode 100644 index 00000000..6950050b --- /dev/null +++ b/src/sketches/countsketch_l2_norm.rs @@ -0,0 +1,373 @@ +//! Count Sketch whose rows also keep a running sum of squared counters. +//! +//! [`CountSketchWithL2Norm`] is the per-layer sketch of `UnivMon` and +//! `UnivMonTwoTier`; its L2 norm sets their heavy-hitter threshold. + +use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; + +use crate::common::structures::matrix_storage::cols_mask_bits; +use crate::octo_delta::CountSketchCellDelta; +use crate::{ + DataInput, DefaultXxHasher, SketchHasher, Vector1D, Vector2D, compute_median_inline_f64, +}; + +pub(crate) mod wire; + +const DEFAULT_ROWS: usize = 3; +const DEFAULT_COLS: usize = 4096; + +/// Count Sketch augmented with per-row L2 norm tracking for heavy-hitter detection. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(bound = "")] +pub struct CountSketchWithL2Norm { + counts: Vector2D, + row_l2_squared: Vector1D, + rows: usize, + cols: usize, + seed_index: usize, + #[serde(skip)] + _hasher: PhantomData, +} + +impl CountSketchWithL2Norm { + /// Checks that a `rows` x `cols` `CountSketchWithL2Norm` fits its one 128-bit key hash. + /// + /// Row `r` reads its column from bits `[r * mask_bits, (r + 1) * mask_bits)` + /// and its sign from bit `127 - r`, where `mask_bits = cols_mask_bits(cols)`. + /// Each row therefore claims `mask_bits + 1` bits, and the fields are in range + /// and pairwise disjoint (so every column and sign is an independent hash + /// draw) exactly while `rows * (mask_bits + 1) <= 128`. Past that, deep rows' + /// columns reuse shallow rows' sign bits, and from `rows * mask_bits >= 128` + /// the shift runs off the hash. + pub(crate) fn check_hash_bits(rows: usize, cols: usize) -> Result<(), String> { + let mask_bits = cols_mask_bits(cols.max(1)) as usize; + let bits_required = mask_bits.saturating_add(1).saturating_mul(rows); + if bits_required > 128 { + return Err(format!( + "CountSketchWithL2Norm rows={rows}, cols={cols} needs {bits_required} hash bits \ + ({mask_bits} column bits + 1 sign bit per row), past the 128 the key hash carries" + )); + } + Ok(()) + } + + /// Column and sign `CountSketchWithL2Norm` uses for `row`, given its 128-bit key hash. + /// + /// Shared with the OctoSketch worker sketches so a worker and its parent can + /// never disagree about which cell a key lands in. + #[inline(always)] + pub fn cell_for_row(hashed_val: u128, row: usize, cols: usize, mask_bits: u32) -> (usize, i64) { + let mask = (1u128 << mask_bits) - 1; + let hashed = (hashed_val >> (row * mask_bits as usize)) & mask; + let col = (hashed as usize) % cols; + let bit = ((hashed_val >> (127 - row)) & 1) as i64; + (col, -(1 - 2 * bit)) + } +} + +// Default CountSketchWithL2Norm configuration. +impl Default for CountSketchWithL2Norm { + fn default() -> Self { + Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) + } +} + +// CountSketchWithL2Norm constructors and operations. +impl CountSketchWithL2Norm { + /// Creates a sketch with the requested number of rows and columns. + pub fn with_dimensions(rows: usize, cols: usize) -> Self { + Self::with_dimensions_and_seed_index(rows, cols, 0) + } + + /// Creates a sketch with the requested dimensions and a custom seed offset. + /// + /// # Panics + /// + /// When `rows * (cols_mask_bits(cols) + 1) > 128`: the rows' column and + /// sign fields would overlap in the one 128-bit key hash. + pub fn with_dimensions_and_seed_index(rows: usize, cols: usize, seed_index: usize) -> Self { + if let Err(problem) = CountSketchWithL2Norm::check_hash_bits(rows, cols) { + panic!("{problem}"); + } + let mut sk = CountSketchWithL2Norm { + counts: Vector2D::unfilled(rows, cols), + row_l2_squared: Vector1D::filled(rows, 0), + rows, + cols, + seed_index, + _hasher: PhantomData, + }; + sk.counts.fill(0); + sk + } + + /// Number of rows in the sketch. + pub fn rows(&self) -> usize { + self.rows + } + + /// Number of columns in the sketch. + pub fn cols(&self) -> usize { + self.cols + } + + /// Exposes the backing matrix for inspection/testing. + pub fn as_storage(&self) -> &Vector2D { + &self.counts + } + + /// Mutable access used internally for testing scenarios. + pub fn as_storage_mut(&mut self) -> &mut Vector2D { + &mut self.counts + } + + /// Merges another sketch while asserting compatible dimensions. + pub fn merge(&mut self, other: &Self) { + assert_eq!( + (self.rows, self.cols), + (other.rows, other.cols), + "dimension mismatch while merging CountSketchWithL2Norm sketches" + ); + assert_eq!( + self.seed_index, other.seed_index, + "seed mismatch while merging CountSketchWithL2Norm sketches" + ); + + for i in 0..self.rows { + let mut row_l2 = 0_i128; + for j in 0..self.cols { + self.counts[i][j] += other.counts[i][j]; + let counter = self.counts[i][j] as i128; + row_l2 = row_l2.saturating_add(counter.saturating_mul(counter)); + } + self.row_l2_squared[i] = row_l2.min(i64::MAX as i128) as i64; + } + } + + /// Resets all counters and L2 accumulators to zero without reallocating. + pub fn clear(&mut self) { + self.counts.fill(0); + self.row_l2_squared.fill(0); + } + + /// Inserts with hash optimization - computes hash once and reuses it. + /// due to the limitation of seeds, use fast_insert only + pub fn update_weighted(&mut self, val: &DataInput, c: i64) { + let hashed_val = H::hash128_seeded(self.seed_index, val); + self.update_weighted_with_hash(hashed_val, c); + } + + /// Inserts with hash optimization using precomputed hash value. + pub fn update_weighted_with_hash(&mut self, hashed_val: u128, weight: i64) { + let mask_bits = self.counts.mask_bits(); + for i in 0..self.rows { + let (idx, sign_bit) = + CountSketchWithL2Norm::cell_for_row(hashed_val, i, self.cols, mask_bits); + let old_value = self.counts.query_one_counter(i, idx); + self.write_counter(i, idx, old_value + sign_bit * weight); + } + } + + /// Writes one counter and carries the row's L2 accumulator with it. + /// + /// Saturating i128 intermediate so extreme counts degrade gracefully + /// instead of wrapping (matches merge-path semantics, which also clamps to + /// [0, i64::MAX]: L2^2 can never go negative, even when a decrement follows + /// prior saturation). + #[inline(always)] + fn write_counter(&mut self, row: usize, col: usize, new_value: i64) { + let old_value = self.counts.query_one_counter(row, col); + self.counts[row][col] = new_value; + let old_l2 = self.row_l2_squared.as_slice()[row] as i128; + let new_l2 = old_l2 + (new_value as i128) * (new_value as i128) + - (old_value as i128) * (old_value as i128); + self.row_l2_squared[row] = new_l2.clamp(0, i64::MAX as i128) as i64; + } + + /// Seed offset this sketch hashes with; UnivMon gives each layer its own. + pub fn seed_index(&self) -> usize { + self.seed_index + } + + /// Column-mask width used to slice the 128-bit hash into per-row indices. + pub fn mask_bits(&self) -> u32 { + self.counts.mask_bits() + } + + /// Applies a counter delta promoted by an OctoSketch worker. + /// + /// The row's L2 accumulator is carried along exactly, so a parent fed only + /// deltas reports the same `l2` as one fed the raw stream. + pub fn apply_delta(&mut self, delta: CountSketchCellDelta) { + let (row, col) = (delta.row as usize, delta.col as usize); + let new_value = self.counts.query_one_counter(row, col) + delta.count as i64; + self.write_counter(row, col, new_value); + } + + /// Inserts without L2 update using precomputed hash value. + pub fn update_weighted_with_hash_without_l2(&mut self, hashed_val: u128, weight: i64) { + let mask_bits = self.counts.mask_bits() as usize; + let mask = (1u128 << mask_bits) - 1; + let mut shift_amount = 0; + let mut sign_bit_pos = 127; + + for i in 0..self.rows { + let hashed = (hashed_val >> shift_amount) & mask; + let idx = (hashed as usize) % self.cols; + let bit = ((hashed_val >> sign_bit_pos) & 1) as i64; + let sign_bit = -(1 - 2 * bit); + + self.counts[i][idx] += sign_bit * weight; + + shift_amount += mask_bits; + sign_bit_pos -= 1; + } + } + + /// Update and estimate with hash optimization. + /// due to the limitation of seeds, use fast_insert only + pub fn update_weighted_and_estimate(&mut self, val: &DataInput, weight: i64) -> f64 { + let hashed_val = H::hash128_seeded(self.seed_index, val); + self.update_weighted_with_hash(hashed_val, weight); + self.estimate_with_hash(hashed_val) + } + + /// Update and estimate without L2 with hash optimization. + /// due to the limitation of seeds, use fast_insert only + pub fn fast_update_and_est_without_l2(&mut self, val: &DataInput, c: i64) -> f64 { + let hashed_val = H::hash128_seeded(self.seed_index, val); + self.update_weighted_with_hash_without_l2(hashed_val, c); + self.estimate_with_hash(hashed_val) + } + + /// Returns the median estimate of the squared L2 norm across rows. + pub fn l2_squared(&self) -> f64 { + let mut values: Vec = self.row_l2_squared.as_slice()[..self.rows] + .iter() + .map(|&v| v as f64) + .collect(); + compute_median_inline_f64(&mut values) + } + + /// Returns the estimated L2 norm (square root of `l2_squared`). + pub fn l2(&self) -> f64 { + let l2 = self.l2_squared(); + l2.sqrt() + } + + /// Returns the frequency estimate with hash optimization. + /// due to the limitation of seeds, use fast_insert only + pub fn estimate(&self, val: &DataInput) -> f64 { + let hashed_val = H::hash128_seeded(self.seed_index, val); + self.estimate_with_hash(hashed_val) + } + + /// Returns the frequency estimate using precomputed hash value. + /// due to the limitation of seeds, use fast_insert only + pub fn estimate_with_hash(&self, hashed_val: u128) -> f64 { + let mask_bits = self.counts.mask_bits() as usize; + let mask = (1u128 << mask_bits) - 1; + let mut lst = Vec::with_capacity(self.rows); + let mut shift_amount = 0; + let mut sign_bit_pos = 127; + + for i in 0..self.rows { + let hashed = (hashed_val >> shift_amount) & mask; + let idx = (hashed as usize) % self.cols; + let bit = ((hashed_val >> sign_bit_pos) & 1) as i64; + let sign_bit = -(1 - 2 * bit); + let counter = self.counts.query_one_counter(i, idx); + lst.push((sign_bit * counter) as f64); + + shift_amount += mask_bits; + sign_bit_pos -= 1; + } + compute_median_inline_f64(&mut lst[..]) + } +} + +#[cfg(test)] +mod tests_count_l2_hh { + use super::*; + + /// 11 rows x (11 column bits + 1 sign bit) = 132 bits of one 128-bit + /// hash: the deep rows' columns would reuse the shallow rows' signs. + #[test] + #[should_panic(expected = "hash bits")] + fn countl2hh_rejects_column_bits_that_reach_the_sign_bits() { + let _: CountSketchWithL2Norm = + CountSketchWithL2Norm::with_dimensions_and_seed_index(11, 2048, 0); + } + + /// 10 x 2048 spends 120 bits and is accepted. + #[test] + fn countl2hh_accepts_the_widest_disjoint_geometry() { + let sketch: CountSketchWithL2Norm = + CountSketchWithL2Norm::with_dimensions_and_seed_index(10, 2048, 0); + assert_eq!((sketch.rows(), sketch.cols()), (10, 2048)); + } + + #[test] + fn countl2hh_estimates_and_l2_are_consistent() { + let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); + let key = DataInput::Str("gamma"); + + let est_after_first = sketch.update_weighted_and_estimate(&key, 5); + assert_eq!(est_after_first, 5.0); + + let est_after_second = sketch.update_weighted_and_estimate(&key, -2); + assert_eq!(est_after_second, 3.0); + + let l2 = sketch.l2(); + assert!(l2 >= 3.0, "expected non-trivial l2, got {l2}"); + } + + #[test] + fn countl2hh_merge_combines_frequency_vectors() { + let mut left: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); + let mut right: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); + let key = DataInput::U32(42); + + left.update_weighted(&key, 4); + assert_eq!(left.estimate(&key), 4.0); + right.update_weighted(&key, 9); + assert_eq!(right.estimate(&key), 9.0); + + left.merge(&right); + assert_eq!(left.estimate(&key), 13.0); + assert_eq!(left.l2(), 13.0); + } + + #[test] + fn countl2hh_round_trip_serialization() { + let mut sketch: CountSketchWithL2Norm = + CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 32, 7); + let key = DataInput::Str("serialize"); + + sketch.update_weighted(&key, 11); + sketch.update_weighted(&key, -3); + let base_est = sketch.estimate(&key); + let base_l2 = sketch.l2(); + + let encoded = sketch + .to_asapv1() + .expect("serialize CountSketchWithL2Norm into MessagePack"); + assert!(!encoded.is_empty(), "serialized bytes should not be empty"); + let data = encoded.clone(); + + let decoded: CountSketchWithL2Norm = CountSketchWithL2Norm::from_asapv1(&data) + .expect("deserialize CountSketchWithL2Norm from MessagePack"); + + assert_eq!(sketch.rows(), decoded.rows()); + assert_eq!(sketch.cols(), decoded.cols()); + assert!( + (decoded.estimate(&key) - base_est).abs() < f64::EPSILON, + "estimate changed after round trip" + ); + assert!( + (decoded.l2() - base_l2).abs() < f64::EPSILON, + "L2 changed after round trip" + ); + } +} diff --git a/src/sketches/countsketch_heap/l2hh_wire.rs b/src/sketches/countsketch_l2_norm/wire.rs similarity index 98% rename from src/sketches/countsketch_heap/l2hh_wire.rs rename to src/sketches/countsketch_l2_norm/wire.rs index 31850b51..17981d3f 100644 --- a/src/sketches/countsketch_heap/l2hh_wire.rs +++ b/src/sketches/countsketch_l2_norm/wire.rs @@ -1,12 +1,12 @@ //! ASAPv1 wire serialization for [`CountSketchWithL2Norm`], plus the sub-payload the //! UnivMon family reuses. //! -//! Child submodule of [`crate::sketches::countsketch_heap`]: it holds the -//! metadata/payload DTOs, the kind_id constant and the `to_asapv1` / -//! `from_asapv1` impls, while the algorithm lives in the parent -//! module file. Being a descendant module, it reads the private `counts`, -//! `l2`, `row`, `col` and `seed_index` fields directly without widening any -//! field visibility. See `docs/asapv1_wire_format.md`. +//! Child submodule of [`crate::sketches::countsketch_l2_norm`]: it holds the +//! metadata/payload DTOs and the `to_asapv1` / `from_asapv1` impls, while the +//! algorithm lives in the parent module file. Being a descendant module, it +//! reads the private `counts`, `row_l2_squared`, `rows`, `cols` and +//! `seed_index` fields directly without widening any field visibility. See +//! `docs/asapv1_wire_format.md`. //! //! CountSketchWithL2Norm is one algorithm — a single kind_id `0x19 0x00`. The dimensions //! (`rows` / `cols`) and the seed-list index the sketch hashes with @@ -15,8 +15,8 @@ //! //! ## The shared sub-payload //! -//! [`CountSketchWithL2Norm`] is also the per-layer counter of `UnivMon`, `UnivMonTwoTier` -//! and every other `L2HH::COUNT`. Those sketches inline the same +//! [`CountSketchWithL2Norm`] is also the per-layer counter of `UnivMon` and +//! `UnivMonTwoTier`. Those sketches inline the same //! `(counts, l2)` state into their own positional arrays instead of nesting an //! envelope per layer, so [`layer_state`] and [`rebuild_layer`] are the one //! place that state is read and rebuilt. @@ -194,7 +194,7 @@ pub(crate) fn rebuild_layer( }) } -// Wire serialization for CountSketchWithL2Norm. `l2hh_wire` is a descendant of the sketch +// Wire serialization for CountSketchWithL2Norm. This module is a descendant of the sketch // module, so this impl reads the private fields directly. impl CountSketchWithL2Norm { /// Serializes the sketch into an ASAPv1 MessagePack envelope diff --git a/src/sketches/mod.rs b/src/sketches/mod.rs index f5950fde..67cf1861 100644 --- a/src/sketches/mod.rs +++ b/src/sketches/mod.rs @@ -31,7 +31,7 @@ //! appear?" These do not track which keys are heavy hitters. //! - [`countminsketch_heap`] / [`countsketch_heap`] — the same sketch paired //! with a min-heap that tracks the top-`k` heavy hitters as items stream in -//! ([`CountMinSketchWithHeap`], [`CountSketchWithHeap`] / [`CountSketchWithL2Norm`]). Use these when the question is +//! ([`CountMinSketchWithHeap`], [`CountSketchWithHeap`]). Use these when the question is //! "which keys are the most frequent?" rather than "how often did this //! specific key appear?" //! @@ -110,11 +110,13 @@ pub mod countminsketch_heap; pub use countminsketch_heap::{CountMinSketchWithHeap, CountMinSketchWithHeapDelta}; pub mod countsketch_heap; -pub use countsketch_heap::CountSketchWithL2Norm; pub use countsketch_heap::{ CountSketchHeapCounter, CountSketchWithHeap, CountSketchWithHeapDelta, heap_count_from_estimate, }; +pub mod countsketch_l2_norm; +pub use countsketch_l2_norm::CountSketchWithL2Norm; + pub mod countminsketch_space_saving; pub use countminsketch_space_saving::{CandidateCounter, CountMinSketchWithSpaceSaving}; diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index e3362ff7..a1d32764 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -26,7 +26,7 @@ use asap_sketchlib::{ CountSketchWithHeap, CountSketchWithHeapDelta, CountSketchWithL2Norm, CountSketchWithSpaceSaving, DataInput, DdSketch, DdSketchDelta, ElasticSketch, Ertl, FastPath, HeapCount, HeavyBucket, HllRegistersP14, Hydra, HyperLogLog, HyperLogLogDelta, - HyperLogLogHipP12, HyperLogLogHipP14, HyperLogLogP12, HyperLogLogP14, Kll, KllDynamic, L2HH, + HyperLogLogHipP12, HyperLogLogHipP14, HyperLogLogP12, HyperLogLogP14, Kll, KllDynamic, OwnedDataInput, RegularPath, SetAggregator, SetAggregatorDelta, TopKHeap, UnivMon, Vector2D, peek_kind_id, }; @@ -1467,7 +1467,7 @@ fn univmon_known(keys: [DataInput<'_>; 6]) -> UnivMon { fn univmon_counts(um: &UnivMon) -> Vec { (0..um.layers) .flat_map(|layer| { - let L2HH::COUNT(counter) = &um.layer_sketches[layer]; + let counter = &um.layer_sketches[layer]; counter.as_storage().as_slice().to_vec() }) .collect() @@ -2003,7 +2003,7 @@ fn univmon_optimized_l5_e2_3x8_1x4_h6_matches_golden() { ); assert_eq!(shape, (5, 2, 3, 8, 1, 4, 6, 70)); for (layer, rows) in pyramid_layers().iter().enumerate() { - let L2HH::COUNT(counter) = &decoded.layer_sketches[layer]; + let counter = &decoded.layer_sketches[layer]; let flat: Vec = rows.iter().flatten().map(|&v| i64::from(v)).collect(); assert_eq!(counter.as_storage().as_slice(), flat.as_slice()); let mut heap: Vec<(String, i64)> = decoded.layer_heaps[layer] diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index 1a0bf89b..b415592a 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -36,7 +36,7 @@ use asap_sketchlib::common::UNIVMON_BOTTOM_LAYER_SEED_INDEX; use asap_sketchlib::{ Classic, CountMinSketch, CountMinSketchCellDelta, CountSketch, CountSketchCellDelta, CountSketchWithL2NormWorker, DataInput, DdSketch, DdSketchWorker, Ertl, FastPath, - HllRegisterDelta, HyperLogLog, HyperLogLogP12, HyperLogLogP16, L2HH, + HllRegisterDelta, HyperLogLog, HyperLogLogP12, HyperLogLogP16, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, OCTO_HLL_DEFAULT_THRESHOLD, OctoAggregator, OctoPlan, OctoThreshold, OctoWorker, RegularPath, UnivMon, UnivMonCellDelta, UnivMonDeltaFidelity, UnivMonOctoAggregator, UnivMonOctoPlan, @@ -2816,7 +2816,7 @@ fn a_flat_threshold_starves_the_deep_univmon_layers() { }); let nonzero = |sketch: &UnivMon, layer: usize| -> usize { - let L2HH::COUNT(inner) = &sketch.layer_sketches[layer]; + let inner = &sketch.layer_sketches[layer]; (0..inner.rows()) .flat_map(|r| (0..inner.cols()).map(move |c| (r, c))) .filter(|(r, c)| inner.as_storage().query_one_counter(*r, *c) != 0) diff --git a/tests/pbt/univmon.rs b/tests/pbt/univmon.rs index 31386503..67b4b60c 100644 --- a/tests/pbt/univmon.rs +++ b/tests/pbt/univmon.rs @@ -17,8 +17,8 @@ use crate::support::{PROBE_KEYS, close, grid}; use asap_sketchlib::{ - CountSketchWithL2Norm, DataInput, HeapEntry, L2HH, OwnedDataInput, - UNIVMON_BOTTOM_LAYER_SEED_INDEX, UnivMon, hash64_seeded, hash128_seeded, + CountSketchWithL2Norm, DataInput, HeapEntry, OwnedDataInput, UNIVMON_BOTTOM_LAYER_SEED_INDEX, + UnivMon, hash64_seeded, hash128_seeded, }; use proptest::prelude::*; use std::collections::{HashMap, HashSet}; @@ -54,7 +54,7 @@ fn layer_keys(sketch: &UnivMon) -> Vec> { /// One layer's counter grid in row-major order. fn cells(sketch: &UnivMon, layer: usize) -> Vec { - let L2HH::COUNT(cs) = &sketch.layer_sketches[layer]; + let cs = &sketch.layer_sketches[layer]; grid(cs.rows(), cs.cols(), |r, c| { cs.as_storage().query_one_counter(r, c) }) From 1d8274105f769d41071b07cd5d8b09fbb4482a63 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:37:12 +0000 Subject: [PATCH 04/11] docs(names): follow-ups from the #209 implementation review Algorithm names in prose say Count-Min, KLL and Count Sketch again; stale old names in docs and comments (KLL constructors, EH bucket and Hydra variants, bare Count) follow the renames; the CHANGELOG write-verb bullet states the rule and the odd cases, and the Unreleased entries use current names. The crate doc qualifies the legacy DTOs with portable::, Uniform keeps a DEFAULT_SEED alias, and the Space-Saving debug assert says count again. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 104 +++++++++++++++------------- README.md | 2 +- docs/api/api_ehsketchlist.md | 38 +++++----- docs/api/api_hashlayer.md | 4 +- docs/api/api_microscope.md | 2 +- docs/api/api_octo.md | 4 +- docs/apis.md | 6 +- docs/asapv1_wire_format.md | 4 +- docs/e2e_testing_harness.md | 2 +- docs/tests.md | 36 +++++----- src/common/input.rs | 2 +- src/lib.rs | 9 +-- src/sketch_framework/hydra.rs | 2 +- src/sketches/countminsketch.rs | 38 +++++----- src/sketches/countminsketch_heap.rs | 2 +- src/sketches/countsketch_heap.rs | 2 +- src/sketches/kll.rs | 6 +- src/sketches/kll/wire.rs | 2 +- src/sketches/kll_dynamic.rs | 8 +-- src/sketches/mod.rs | 4 +- src/sketches/space_saving.rs | 2 +- src/sketches/uniform.rs | 3 +- tests/PBT_COVERAGE.md | 2 +- tests/TEST_COVERAGE.md | 10 +-- tests/conformance_kit.rs | 2 +- tests/e2e/octo.rs | 2 +- tests/e2e/windows.rs | 10 +-- 27 files changed, 157 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9331e48a..4af9833e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,11 @@ signals a backwards-compatible change. - **`TupleKey` and `KeyColumn`**, a heap key of typed columns (null, bool, `i64`, `f64`, string) encoded into one canonical byte string, so a GROUP BY - over several columns keeps one CMSHeap or CSHeap candidate per tuple under + over several columns keeps one CountMinSketchWithHeap or CountSketchWithHeap candidate per tuple under `key_type` `"bytes"`. The encoding is ASAPv1's tuple key (spec Section 2); `-0.0` encodes as `0.0` and every `NaN` as one `NaN`. - **ASAPv1 golden fixtures for KMV, Bloom, Space-Saving, UniformSampling, - UnivMon Optimized, UnivMon-Q, ExponentialHistogram and EHSketchList**, so + UnivMonTwoTier, UnivMon-Q, ExponentialHistogram and EhBucketSketch**, so every kind_id the spec marks implemented has one. The spec fixes Bloom's bit order within a word: column `c` is bit `c mod 64`. - **`Vector3D`**, filled in from a stub. A `rows x cols` grid whose every @@ -44,9 +44,9 @@ signals a backwards-compatible change. a stream of `(key, distinct_value)` pairs, how many distinct values has this key seen? A Count-Min-shaped grid whose every cell is a small HyperLogLog, held in one contiguous `Vector3D` register plane. `Hydra` with - `HydraCounter::HLL` answers the same query on a stable API, with one + `HydraCounter::Hll` answers the same query on a stable API, with one heap-allocated HLL per cell. -- **`MicroCM`** (`experimental` feature), a sliding-window frequency sketch +- **`MicroCm`** (`experimental` feature), a sliding-window frequency sketch following MicroscopeSketch (Zhao et al., KDD 2023). Each cell holds `T + 2` byte-wide pixels and one shared zoom exponent, so the counters stay narrow and the exponent buys the range instead of the counters being widened for @@ -77,40 +77,40 @@ signals a backwards-compatible change. payload is unchanged. - **`peek_kind_id(&[u8])`** returns an ASAPv1 envelope's `kind_id` after checking its magic, version, and framing, without decoding the sketch. -- **`CountMinDelta`**, the ASAPv1 Count-Min delta (`0x02 0x80`): built with +- **`CountMinSketchDelta`**, the ASAPv1 Count-Min delta (`0x02 0x80`): built with `new` from changed cells, serialized like a sketch, and added with - `CountMin::merge_delta`, which changes nothing when it fails. + `CountMinSketch::merge_delta`, which changes nothing when it fails. - **`sketches::CountSketchDelta`**, the ASAPv1 Count Sketch delta (`0x04 0x80`), and `f64` Count Sketch counters (`counter_type` `"f64"`): the delta is built with `new` from changed cells, serialized like a sketch, and added with - `Count::merge_delta`, which changes nothing when it fails. -- **`CMSHeapDelta`**, the ASAPv1 CMSHeap delta (`0x03 0x80`), and `f64` - CMSHeap insert and merge, with unrounded `f64` heap counts. -- **`CSHeapDelta`**, the ASAPv1 CSHeap delta (`0x0a 0x80`), and unrounded - `f64` heap counts for an `f64` CSHeap (`CsHeapCounter`). + `CountSketch::merge_delta`, which changes nothing when it fails. +- **`CountMinSketchWithHeapDelta`**, the ASAPv1 CountMinSketchWithHeap delta (`0x03 0x80`), and `f64` + CountMinSketchWithHeap insert and merge, with unrounded `f64` heap counts. +- **`CountSketchWithHeapDelta`**, the ASAPv1 CountSketchWithHeap delta (`0x0a 0x80`), and unrounded + `f64` heap counts for an `f64` CountSketchWithHeap (`CountSketchHeapCounter`). - **`HyperLogLogDelta`**, the ASAPv1 HLL Classic and Ertl-MLE deltas (`0x01 0x81`, `0x01 0x82`): built with `new` from changed registers, serialized like a sketch, and merged register-wise by `max` with `HyperLogLogImpl::merge_delta`. -- **ASAPv1 for `SetAggregator` (`0x08 0x00`) and `DeltaResult` (`0x08 0x80`)**: - `serialize_to_bytes` / `deserialize_from_bytes` on both types. -- **`DDSketchDelta`**, the ASAPv1 DDSketch delta (`0x05 0x80`): built with +- **ASAPv1 for `SetAggregator` (`0x08 0x00`) and `SetAggregatorDelta` (`0x08 0x80`)**: + `to_asapv1` / `from_asapv1` on both types. +- **`DdSketchDelta`**, the ASAPv1 DDSketch delta (`0x05 0x80`): built with `new` from changed buckets of both stores and a zero-count increment, - serialized like a sketch, and added with `DDSketch::merge_delta`, which + serialized like a sketch, and added with `DdSketch::merge_delta`, which changes nothing when it fails. - **`admission_rate`**, the optional ASAPv1 metadata key of a sampled HLL, Count-Min, Count Sketch or DDSketch state and its delta: set with `with_admission_rate(p)` and read with `admission_rate()` on - `HyperLogLogImpl`, `HyperLogLogHIPImpl`, `CountMin`, `Count` and `DDSketch`, + `HyperLogLogImpl`, `HyperLogLogHipImpl`, `CountMinSketch`, `CountSketch` and `DdSketch`, and set with `with_admission_rate(p)` on the four delta types. Estimates are of the admitted updates; a consumer divides them by `p`. States and deltas merge only at equal rates. A sketch nested in another kind's state must hold `p = 1` to serialize. -- **`CMSSpaceSaving` and `CSSpaceSaving`**: a Count-Min or Count Sketch +- **`CountMinSketchWithSpaceSaving` and `CountSketchWithSpaceSaving`**: a Count-Min or Count Sketch matrix beside a `SpaceSaving` summary of `capacity` counters. Every update reaches both; `top_k(k)` ranks the summary's keys by their matrix estimate. - `insert_many` gives the summary the weight rounded up, or nothing when it is - not positive (`SpaceSavingCounter`). Merge adds the matrices and merges the + `update_weighted` gives the summary the weight rounded up, or nothing when it is + not positive (`CandidateCounter`). Merge adds the matrices and merges the summaries, and panics on a dimension, admission-rate or capacity mismatch. ASAPv1 kinds `0x1e 0x00` and `0x1f 0x00` (spec §3.24, §3.25) for `i32`, `i64` and `f64` `Vector2D` matrices on either path, with no delta kind. @@ -127,22 +127,16 @@ signals a backwards-compatible change. `message_pack_format::portable::...`; `MessagePackCodec` from `message_pack_format`. `CountSketchDelta` at the root is the ASAPv1 Count Sketch delta. -- **Every sketch write is `update`** (breaking). `insert` -> `update`; - `insert_many` and weighted `insert` -> `update_weighted`; `bulk_insert`, - batch `insert_many`, `insert_batch`, `bulk_update` -> `update_batch`; - `bulk_insert_many` -> `update_weighted_batch`; `insert_with_hash`, - `insert_by_hash` -> `update_with_hash`; `bulk_insert_with_hashes`, - `insert_many_with_hashes` -> `update_batch_with_hashes`; - `bulk_insert_many_with_hashes` -> `update_weighted_batch_with_hashes`; - `insert_at*`/`bulk_insert_at*` -> `update_at*`/`update_batch_at*`; - `insert_heavy_only` -> `update_heavy_only`; `insert_cached_step` -> - `update_batch_cached_step`; `bulk_update_data_input` -> - `update_batch_data_input`; `tumbling_insert` -> `tumbling_update_weighted`; - `insert_emit_*` -> `update_emit_*`; `insert_hash_emit_delta` -> - `update_emit_delta_with_hash`; `insert_hashes_emit_delta`, - `insert_hashes_emit_keyed_delta` -> `update_emit_delta_with_row_hashes`, - `update_emit_keyed_delta_with_row_hashes`; `CocoBucket::add_v` -> `add`. - Weight parameters are named `weight`. Custom `sketch_catalog` and +- **Every sketch write is `update`** (breaking). A write is named + `update[_weighted][_batch][_at | _emit_delta | ...][_with_hash | _with_hashes | + _with_row_hashes][_with_threshold | _and_threshold]`: `insert` -> `update`, + `insert_many` -> `update_weighted`, `bulk_insert` -> `update_batch`, + `bulk_insert_many_with_hashes` -> `update_weighted_batch_with_hashes`, and so + on. The cases the rule does not spell out: `tumbling_insert` -> + `tumbling_update_weighted`, `insert_cached_step` -> + `update_batch_cached_step`, `insert_hashes_emit_*` -> + `update_emit_*_with_row_hashes`, `CocoBucket::add_v` -> `add`. Weight + parameters are named `weight`. Custom `sketch_catalog` and `TumblingWindowSketch` impls break. - **Acronyms are one word in type names** (breaking). `KLL`, `KLLDynamic`, `KLLConfig` -> `Kll`, `KllDynamic`, `KllConfig`; `DDSketch`, `DDSketchDelta` @@ -179,7 +173,11 @@ signals a backwards-compatible change. `CocoSketch`, `CocoWorkerSketch` -> `CocoSketchWorker`; `bktlen` -> `heavy_buckets`, `init_with_length` -> `with_heavy_buckets`, `init_with_dimensions` -> `with_heavy_and_light`, `query` -> `estimate`; - Coco `w`/`d` -> `cols`/`rows`, `estimate_key` -> `estimate`. + Coco `w`/`d` -> `cols`/`rows`, `estimate_key` -> `estimate`. Coco + constructors take `(rows, cols)`: `init_with_size(w, d)` -> + `with_dimensions(rows, cols)`, `init_with_size_and_seed(w, d, seed)` -> + `with_dimensions_and_seed(rows, cols, seed)`, and `CocoSketchWorker::new` / + `with_seed` swap their first two arguments the same way. - **KLL, CDF and DDSketch** (breaking). KLL `init`/`init_with_seed`/ `init_kll`/`init_kll_with_seed` -> `new`/`with_seed`/`with_k`/ `with_k_and_seed`; `wire_*` accessors -> `level_bounds_top_down()`, @@ -228,6 +226,12 @@ signals a backwards-compatible change. `CandidateCounter`; `KeepSmallest`/`KeepLargest` -> `RetainLargest`/ `RetainSmallest`; `KeyColumn::Str` -> `KeyColumn::String`. - **`DeltaResult` -> `SetAggregatorDelta`** (breaking). +- **Error, panic and label strings name the new types, methods and fields** + (breaking for code that matches on them). `sketch_type()` reports + `"CountMinSketch"`, `"CountSketch"` and `"CountSketchWithL2Norm"` for + `EnsembleSketch` and `EhBucketSketch`; UnivMon-Q config errors name + `layers`, `top_layer_cols`, `rows` and `heap_size`. ASAPv1 bytes are + unchanged. - **Hashing and seeds** (breaking). `hash_item64_seeded`/`hash_item128_seeded` -> `hash64_seeded_owned`/`hash128_seeded_owned`; seed-table index parameters are `seed_index`; `CANONICAL_HASH_SEED` -> @@ -250,16 +254,16 @@ signals a backwards-compatible change. `cell_counter_type`, `cell_mode`, `cell_precision`, `cell_k`, `cell_m`, `cell_item_type`, `cell_layers`, `cell_heap_size`, `cell_key_type`). Bytes with the old keys are rejected; the affected goldens are regenerated. -- **UnivMon Optimized and UnivMon-Q metadata keys follow the spec.** - UnivMon Optimized writes `layers`, `elephant_rows`, `elephant_cols`, +- **UnivMonTwoTier and UnivMon-Q metadata keys follow the spec.** + UnivMonTwoTier writes `layers`, `elephant_rows`, `elephant_cols`, `mouse_rows` and `mouse_cols`; UnivMon-Q writes `layers`, `top_layer_cols`, `cols_halving_period`, `rows` and `heap_size`. Bytes with the old keys are rejected. -- **An empty heap or summary must carry `key_type` `"u64"`.** CMSHeap, CSHeap, - their deltas, UnivMon, UnivMon Optimized, Hydra's UnivMon counter, - Space-Saving, CMSSpaceSaving and CSSpaceSaving reject one under any other +- **An empty heap or summary must carry `key_type` `"u64"`.** CountMinSketchWithHeap, CountSketchWithHeap, + their deltas, UnivMon, UnivMonTwoTier, Hydra's UnivMon counter, + Space-Saving, CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving reject one under any other name. -- **CMSHeap, CSHeap, their deltas, UnivMon, UnivMon Optimized and Hydra's +- **CountMinSketchWithHeap, CountSketchWithHeap, their deltas, UnivMon, UnivMonTwoTier and Hydra's UnivMon counter read a `"string"` key from msgpack `str` alone**; UTF-8 `bin` keys relabelled `"string"` are rejected. - **Space-Saving and UnivMon-Q ASAPv1 decodes reject entries out of emitted @@ -267,17 +271,17 @@ signals a backwards-compatible change. two `NaN` keys fail on both doors. - **`HyperLogLogImpl::merge_delta` returns `Result<(), String>`**, an `Err` when the delta's admission rate differs from the state's. -- **`EHSketchList::merge` and `EHBucket::to_merge` return `Result<(), String>`**, +- **`EhBucketSketch::merge` and `EhBucket::merge` return `Result<(), String>`**, passing the inner sketch's error through. -- **The derived serde form of `HyperLogLogImpl`, `HyperLogLogHIPImpl`, - `CountMin`, `Count` and `DDSketch` gains a trailing `admission_rate` field**, +- **The derived serde form of `HyperLogLogImpl`, `HyperLogLogHipImpl`, + `CountMinSketch`, `CountSketch` and `DdSketch` gains a trailing `admission_rate` field**, `1` when absent and refused outside `(0, 1]`. - **DDSketch has one ASAPv1 encoding**: `metadata_version` 1 with all eight payload fields, the negative store and zero count included. Bytes change for positive-only and empty sketches, and `metadata_version` 2 is rejected. -- **The CMSHeap and CSHeap ASAPv1 metadata key `k` is renamed `heap_size`**, - and the CMSHeap, CSHeap and UnivMon decoders reject heap entries out of - emitted order. `HHItem` and `HHHeap` take a count type, `i64` by default. +- **The CountMinSketchWithHeap and CountSketchWithHeap ASAPv1 metadata key `k` is renamed `heap_size`**, + and the CountMinSketchWithHeap, CountSketchWithHeap and UnivMon decoders reject heap entries out of + emitted order. `HeapEntry` and `TopKHeap` take a count type, `i64` by default. - **ASAPv1 decoders reject a block with bytes after its msgpack value, and a float field that is not msgpack float64** (an `f32` key: not float32). - **The ASAPv1 golden fixtures live in their own repo**, @@ -293,9 +297,9 @@ signals a backwards-compatible change. once for all of them instead of once per suite. `cargo test --test e2e` runs the set; a single suite is a filter, `cargo test --test e2e frequency::`. - Two doc comments described behaviour the code does not have, and now say - what it does: `HHHeap::update`'s return value reports whether the key took + what it does: `TopKHeap::update`'s return value reports whether the key took its place without displacing another, not whether every key offered so far - is retained; `TumblingWindow::insert` closes a period that saw no arrivals + is retained; `TumblingWindow::update_weighted` closes a period that saw no arrivals as an empty window like any other, and that window takes a place in `max_windows`. diff --git a/README.md b/README.md index bf316d24..76d99348 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ for &(event, true_count) in &events { ```rust use asap_sketchlib::Kll; -// Kll is a quantile sketch — it tracks the distribution of values so you can +// KLL is a quantile sketch — it tracks the distribution of values so you can // ask questions like "what is the median?" without storing every data point. let mut sketch = Kll::::default(); diff --git a/docs/api/api_ehsketchlist.md b/docs/api/api_ehsketchlist.md index e3138f8e..03dc5d6b 100644 --- a/docs/api/api_ehsketchlist.md +++ b/docs/api/api_ehsketchlist.md @@ -10,17 +10,17 @@ Unified enum wrapper for sketch payloads used by EH-style frameworks. ```rust enum EhBucketSketch { - CM(CountMinSketch, FastPath>), - COCO(CocoSketch), - COUNTL2HH(CountSketchWithL2Norm), - CS(CountSketch, FastPath>), - DDS(DdSketch), - ELASTIC(ElasticSketch), - HLL(HyperLogLog), + CountMinSketch(CountMinSketch, FastPath>), + CocoSketch(CocoSketch), + CountSketchWithL2Norm(CountSketchWithL2Norm), + CountSketch(CountSketch, FastPath>), + DdSketch(DdSketch), + ElasticSketch(ElasticSketch), + Hll(HyperLogLog), Kll(Kll), #[cfg(feature = "experimental")] - UNIFORM(UniformSampling), - UNIVMON(UnivMon), + Uniform(UniformSampling), + UnivMon(UnivMon), } ``` @@ -67,18 +67,18 @@ The ten nested kind_ids: | Variant | Nested kind_id | Registry name | Feature | | ------- | -------------- | ------------- | ------- | -| `CM` | `0x02 0x00` | Count-Min | default | -| `COCO` | `0x0c 0x00` | Coco | default | -| `COUNTL2HH` | `0x19 0x00` | CountSketchWithL2Norm | default | -| `CS` | `0x04 0x00` | Count Sketch | default | -| `DDS` | `0x05 0x00` | DDSketch | default | -| `ELASTIC` | `0x0b 0x00` | Elastic | default | -| `HLL` | `0x01 0x02` | HLL Ertl-MLE | default | +| `CountMinSketch` | `0x02 0x00` | Count-Min | default | +| `CocoSketch` | `0x0c 0x00` | Coco | default | +| `CountSketchWithL2Norm` | `0x19 0x00` | CountSketchWithL2Norm | default | +| `CountSketch` | `0x04 0x00` | Count Sketch | default | +| `DdSketch` | `0x05 0x00` | DDSketch | default | +| `ElasticSketch` | `0x0b 0x00` | Elastic | default | +| `Hll` | `0x01 0x02` | HLL Ertl-MLE | default | | `Kll` | `0x06 0x00` | KLL compact | default | -| `UNIFORM` | `0x0d 0x00` | UniformSampling | `experimental` | -| `UNIVMON` | `0x10 0x00` | UnivMon | default | +| `Uniform` | `0x0d 0x00` | UniformSampling | `experimental` | +| `UnivMon` | `0x10 0x00` | UnivMon | default | -Each id is pinned to one algorithm: `HLL` is Ertl-MLE, so `0x01 0x01` (Classic) +Each id is pinned to one algorithm: `Hll` is Ertl-MLE, so `0x01 0x01` (Classic) and `0x01 0x03` (HIP) are rejected, and `Kll` is compact, so `0x06 0x01` (dynamic) is rejected. The dispatch is the same in every build: a decoder built without `experimental` rejects `0x0d 0x00` with an error naming the variant and diff --git a/docs/api/api_hashlayer.md b/docs/api/api_hashlayer.md index db9113c8..bd12e17c 100644 --- a/docs/api/api_hashlayer.md +++ b/docs/api/api_hashlayer.md @@ -6,7 +6,7 @@ Status: `Ready` Group multiple compatible sketches that share a hash layout so the hash is computed once per insert and fanned out to every sketch in the ensemble. -Supports both frequency sketches (CountMinSketch, Count) and cardinality sketches +Supports both frequency sketches (CountMinSketch, CountSketch) and cardinality sketches (HyperLogLog variants). ### Quick Example @@ -79,7 +79,7 @@ Sketches are converted into `EnsembleSketch` via `From` impls, so you can use `. ```rust use asap_sketchlib::{ - CountMinSketch, Count, FastPath, Vector2D, DefaultXxHasher, + CountMinSketch, CountSketch, FastPath, Vector2D, DefaultXxHasher, HyperLogLog, Ertl, }; use asap_sketchlib::sketch_framework::hashlayer::{EnsembleSketch, HashSketchEnsemble}; diff --git a/docs/api/api_microscope.md b/docs/api/api_microscope.md index 5b8997b4..8dea65fd 100644 --- a/docs/api/api_microscope.md +++ b/docs/api/api_microscope.md @@ -1,4 +1,4 @@ -# API: MicroCM (MicroscopeSketch) +# API: MicroCm (Microscope sketch) Status: `Experimental` diff --git a/docs/api/api_octo.md b/docs/api/api_octo.md index b02a8933..dc4aadfb 100644 --- a/docs/api/api_octo.md +++ b/docs/api/api_octo.md @@ -432,8 +432,8 @@ copy, made once at the hand-off. CocoSketch and Elastic key on a `String`, so their plans render the input with `flow_key_string`, which renders every `DataInput` variant: numbers by `to_string`, bytes as lowercase hex. `EhBucketSketch` converts differently — -its `ELASTIC` arm renders bytes with `String::from_utf8_lossy`, and its -`COCO` arm drops every input that is not `Str` or `String`. The +its `ElasticSketch` arm renders bytes with `String::from_utf8_lossy`, and its +`CocoSketch` arm drops every input that is not `Str` or `String`. The `flow_key_string` rendering is what the aggregator stores, so it is what `CocoSketch::estimate` or `ElasticSketch::estimate` must be asked for. Their payloads carry no hashes: the heavy half of an Elastic insert can evict a key the diff --git a/docs/apis.md b/docs/apis.md index 2ede3d24..658fc2a2 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -4,9 +4,9 @@ This is the canonical API entry point for `asap_sketchlib`. ## Core Sketch APIs -- [CountMinSketch Sketch](./api/api_countmin.md) - `Ready` +- [Count-Min Sketch](./api/api_countmin.md) - `Ready` - Reference: Cormode & Muthukrishnan, "An Improved Data Stream Summary: The Count-Min Sketch and its Applications," J. Algorithms 2005. [https://www.cs.rutgers.edu/~muthu/cm-jal.pdf](https://www.cs.rutgers.edu/~muthu/cm-jal.pdf) -- [Count (`Count Sketch`)](./api/api_count_sketch.md) - `Ready` +- [Count Sketch](./api/api_count_sketch.md) - `Ready` - Reference: Charikar, Chen & Farach-Colton, "Finding Frequent Items in Data Streams," ICALP 2002. [https://dl.acm.org/doi/10.1007/3-540-45465-9_59](https://dl.acm.org/doi/10.1007/3-540-45465-9_59) - [HyperLogLog](./api/api_hyperloglog.md) - `Ready` - `Classic` variant: Flajolet, Fusy, Gandouet & Meunier, "HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm," AOFA 2007. [https://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf](https://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) @@ -34,7 +34,7 @@ This is the canonical API entry point for `asap_sketchlib`. - Reference: "On synopses for distinct-value estimation under multiset operations." [https://dl.acm.org/doi/10.1145/1247480.1247504](https://dl.acm.org/doi/10.1145/1247480.1247504) - [CountMinHll](./api/api_countminsketch_hll.md) - `Experimental` - Per-key distinct counts from a Count-Min grid of per-cell HyperLogLogs. Behind the `experimental` feature; `Hydra` with `HydraCounter::Hll` answers the same query on a stable API. -- [MicroCM (MicroscopeSketch)](./api/api_microscope.md) - `Experimental` +- [MicroCm (Microscope sketch)](./api/api_microscope.md) - `Experimental` - Sliding-window frequency estimation with adaptive zoom. Behind the `experimental` feature; not yet checked against the paper's published measurements. - Reference: Zhao et al., "MicroscopeSketch: Accurate Sliding Estimation Using Adaptive Zooming," KDD 2023. [https://dl.acm.org/doi/10.1145/3580305.3599432](https://dl.acm.org/doi/10.1145/3580305.3599432) diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 60a15f4b..ce84928e 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -92,7 +92,7 @@ erDiagram string input_encoding array seed_list u32 seed_index "hash-spec group, per-sketch; absent for a non-hashing sketch and for a fixed algorithmic index" - mixed structural_params "structural group, per-sketch: precision(HLL); rows+cols+counter_type+mode(CMS); k+m+item_type(Kll); one set per kind, Section 2" + mixed structural_params "structural group, per-sketch: precision(HLL); rows+cols+counter_type+mode(CMS); k+m+item_type(KLL); one set per kind, Section 2" } PAYLOAD { msgpack_array raw_state @@ -1120,7 +1120,7 @@ The four bucket arrays are parallel and equal-length. **The bucket count is not **`size` is genuine state.** It counts the updates folded into a bucket — 1 on creation, summed on merge — and is not recoverable from the sketch: an HLL bucket that saw the same key five times has `size = 5` and cardinality 1. -**`l2_mass` and `merge_norm` are derived and are not carried.** `compute_l2_mass(s)` is `s.eh_l2_mass().unwrap_or(0.0)`, and `eh_l2_mass` answers `Some` only for the `COUNTL2HH` and `UNIVMON` arms; both are pure functions of the sketch's decoded state, and both call sites that write the field assign exactly `compute_l2_mass(&bucket)`. `infer_merge_norm(&prototype)` reads `supports_norm` and is a pure function of the prototype's variant. So the decoder recomputes both, per the derived-field rule that keeps Count-Min's `l1` / `l2` off the wire, and the encoder rejects a histogram whose cached `l2_mass` disagrees with its sketch or whose `merge_norm` disagrees with its prototype. +**`l2_mass` and `merge_norm` are derived and are not carried.** `compute_l2_mass(s)` is `s.eh_l2_mass().unwrap_or(0.0)`, and `eh_l2_mass` answers `Some` only for the `CountSketchWithL2Norm` and `UnivMon` arms; both are pure functions of the sketch's decoded state, and both call sites that write the field assign exactly `compute_l2_mass(&bucket)`. `infer_merge_norm(&prototype)` reads `supports_norm` and is a pure function of the prototype's variant. So the decoder recomputes both, per the derived-field rule that keeps Count-Min's `l1` / `l2` off the wire, and the encoder rejects a histogram whose cached `l2_mass` disagrees with its sketch or whose `merge_norm` disagrees with its prototype. **The prototype carries its full state.** Nothing constrains it to be empty: `new(k, window, eh_type)` takes whatever the caller hands it, and `update_with` clones it for every new bucket without ever mutating it. A prototype seeded with data therefore seeds every future bucket, so it is carried as a full EhBucketSketch triple, the same shape a bucket uses. diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index 3e3e6d81..3bd350b9 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -52,7 +52,7 @@ DDSketch representative that violated the advertised α bound at bucket edges. None of these were caught by the existing unit tests. Tightening the bounds to the ones each family actually promises surfaced more: -a missing `ELASTIC` arm in `EhBucketSketch::merge` that silently discarded every +a missing `ElasticSketch` arm in `EhBucketSketch::merge` that silently discarded every bucket merge in an exponential histogram, HLL Classic's accuracy cliff at the linear-counting switchover, and a structural coupling in `ExponentialHistogramUnivMon` that makes its sketch-tier cardinality unrecoverable. diff --git a/docs/tests.md b/docs/tests.md index fb64df6a..138b6c66 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -51,7 +51,7 @@ Delta tests: [`src/sketches/countminsketch/delta.rs`](../src/sketches/countminsk | `count_min_counter_types_reject_each_other` | Each wire counter type refuses the others' bytes. | Verifies a `2x4` `f64` envelope fails to decode as both an `i32` and an `i64` sketch, and that an `i32` envelope fails to decode as an `f64` one. | | `count_min_rejects_too_many_rows` | More rows than the seed list has seeds is refused on both sides. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | | `estimate_does_not_clamp_i64_counts_above_i32_max` | An `i64` counter's estimate is not clamped at `i32::MAX`. | Inserts `Str("pkt_len")` with weight `35_000_000_000` into a `3x64` `CountMinSketch, RegularPath>` through `update_weighted` and verifies `estimate` is at or above that count and is not `i32::MAX as i64`. | -| `merge_requires_matching_dimensions` | Merging across row counts panics. | Verifies merging a `3x32` `CountMinSketch, RegularPath>` into a `2x32` one panics with "dimension mismatch while merging CountMinSketch sketches". | +| `merge_requires_matching_dimensions` | Merging across row counts panics. | Verifies merging a `3x32` `CountMinSketch, RegularPath>` into a `2x32` one panics with "dimension mismatch while merging Count-Min sketches". | | `decode_rejects_each_payload_rule` | A `0x02 0x80` payload breaking any delta rule fails to decode. | For a `2x3` `i64` delta, verifies `[[0, 5], [1, -1]]` decodes and that unequal lengths, a repeated index, index `6`, and a zero delta each fail. | | `decode_rejects_another_kind` | A delta decodes only under its own kind_id. | Verifies the `cms_i64_regular_2x3_delta` golden decodes and fails with its kind_id relabelled `0x02 0x00`. | | `new_rejects_a_geometry_past_the_seed_list` | A delta has the state's wire geometry. | Verifies `new` accepts `20` rows and refuses `21`. | @@ -93,7 +93,7 @@ Delta tests: [`src/sketches/countsketch/delta.rs`](../src/sketches/countsketch/d | `cs_metadata_rejects_f64_into_an_integer_sketch` | An `f64`-labelled envelope decodes into neither integer sketch. | Wraps metadata naming `counter_type: "f64"` around a valid 2x4 payload and verifies both the `i64` and the `i32` sketch reject it. | | `count_sketch_i32_round_trips_and_is_pinned_by_counter_type` | The `i32` wire config round-trips and its width is identity, not a detail. | Round-trips a `2x4` `Vector2D` sketch, then verifies its bytes differ from the numerically equal `i64` sketch's, that `i32` bytes fail to decode as `i64`, and that `i64` bytes fail to decode as `i32`. | | `countsketch_error_stays_rank_independent_across_frequency_deciles` | Mean absolute error does not track key frequency. | On a Zipf stream (`rows=5`, `cols=4096`, `domain=8192`, `exponent=1.1`, `N=200_000`, seed `1007`) into a `CountSketch, RegularPath>`, sorts the keys by true count into 10 equal deciles and verifies the largest decile mean `\|estimate - truth\|` is at most `3x` the smallest; a spread up to that 3x band passes. | -| `merge_requires_matching_dimensions` | Merging across row counts panics. | Verifies merging a `3x32` `CountSketch, RegularPath>` into a `2x32` one panics with "dimension mismatch while merging CountMinSketch sketches". | +| `merge_requires_matching_dimensions` | Merging across row counts panics. | Verifies merging a `3x32` `CountSketch, RegularPath>` into a `2x32` one panics with "dimension mismatch while merging Count Sketches". | | `f64_estimate_is_the_unrounded_median_and_takes_nan` | An `f64` sketch's estimate is not rounded, and a `NaN` cell does not panic. | Verifies `update_weighted(Str("alpha"), 1.5)` into a `3x64` `CountSketch, RegularPath>` estimates `1.5`, and that a `3x4` matrix of `NaN` estimates `NaN`. | | `decode_rejects_an_index_past_the_matrix` | A `0x04 0x80` index must be below `rows * cols`. | For a `2x4` `i64` delta, verifies index `7` decodes and index `8` fails. | | `decode_rejects_another_kind` | A delta decodes only under its own kind_id. | Verifies the `cs_i64_regular_2x4_delta` golden decodes and fails with its kind_id relabelled `0x04 0x00`. | @@ -164,7 +164,7 @@ Wire tests: [`src/sketches/kll/wire.rs`](../src/sketches/kll/wire.rs) | `kll_rejects_weighted_count_overflow` | A level layout whose weighted count overflows is refused rather than handed back. | Verifies a crafted envelope parking 16 items at compactor level 60, so that `16 * 2^60` overflows `usize` in `count()`, fails to decode. | | `kll_dynamic_kind_id_rejected_by_compact` | The dynamic kind_id is refused by the compact decoder. | Verifies a crafted envelope carrying valid compact metadata and payload under `0x06 0x01` fails to decode as a `Kll`. | | `cdf_cached_matches_uncached_across_lifecycle` | The memoized CDF agrees with the from-scratch rebuild at every lifecycle stage. | For `Kll::::with_seed(200, 8, 42)`, verifies `quantile_cached` equals `quantile` while empty at `{0.1, 0.5, 0.9}`, at `{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}` after 20,000 uniform samples in `[0, 1000)` (seed `0xACAC_0001`), and at `{0.1, 0.5, 0.9}` after merging a seed-43 sketch over 5,000 samples in `[-5000, -4000)` (seed `0xACAC_0002`); after `clear`, `quantile_cached(0.5)` is `0.0` and `count` is `0`. | -| `cdf_cache_invalidated_by_subsequent_updates` | Updates after a cached query invalidate the cache instead of serving a stale CDF. | Takes `quantile_cached(0.5)` on a `init_with_seed(200, 8, 42)` sketch over 10,000 uniform samples in `[0, 100)` (seed `0xACAC_0003`), then inserts `999_999.0` 10,000 times and verifies the cached median rises by more than `50.0`. | +| `cdf_cache_invalidated_by_subsequent_updates` | Updates after a cached query invalidate the cache instead of serving a stale CDF. | Takes `quantile_cached(0.5)` on a `with_seed(200, 8, 42)` sketch over 10,000 uniform samples in `[0, 100)` (seed `0xACAC_0003`), then inserts `999_999.0` 10,000 times and verifies the cached median rises by more than `50.0`. | | `kllwire_serde_rejects_crafted_dimensions` | The nested serde decoder refuses an out-of-range `k` before it sizes anything. | Encodes a `KLLWire` with no items, `levels` of `[0, 0]`, `k` of `usize::MAX`, `m` of `8`, `num_levels` `1`, and `Coin::from_seed(1)`, and verifies it does not decode as a `Kll` rather than overflowing `compute_max_capacity`. | | `from_portable_state_reproduces_source_exactly` | Reconstruction from portable state is bit-exact, not a lossy replay. | Extracts the compacted `items` and `levels` from a `Kll::::with_k(200)` sketch over 200,000 updates of `i * 0.0007 + 3.0`, and verifies `from_portable_state` reproduces the source's quantiles at `{0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1}` exactly; an empty reconstruction is verified to report `num_levels` `1` with `levels[0]` at `max_capacity`. | | `seeded_sketches_are_byte_identical` | Same seed and same input give byte-identical sketches. | Feeds 5,000 uniform samples in `[0, 1_000_000)` (seed `7`) to two `with_k_and_seed(200, 42)` sketches and verifies their serialized bytes are equal. | @@ -172,7 +172,7 @@ Wire tests: [`src/sketches/kll/wire.rs`](../src/sketches/kll/wire.rs) | `clear_preserves_seed_determinism` | `clear` re-seeds from the stored seed, so determinism survives a window rotation. | Burns 1,500 unrelated samples (seed `99`) through a `with_k_and_seed(200, 1234)` sketch, clears it, then feeds it and a fresh seed-1234 twin the same 3,000 samples (seed `13`) and verifies their serialized bytes are equal. | | `merge_into_empty_target_preserves_weight_issue_68_repro` | Merging into an empty target preserves the total weight exactly. | Builds a `k=200` source over `1..=1000`, checks its `count` falls in `980..=1020`, merges it into an empty `k=200` target, and verifies the target's `count` equals the source's exactly and its median lands in `475.0..=525.0`. | | `merge_two_nonempty_sketches_preserves_weight_and_quantiles` | Merging two non-empty sketches is a genuinely weighted merge, not just the empty-target special case. | Builds one `k=200` sketch over 50,000 uniform samples in `[0, 1_000_000)` (seed `0xA11CE`) and another over 50,000 Zipf samples (8,192 distinct, exponent `1.1`, seed `0xB0B`), verifies each pre-merge `count` is within 3% of `50_000`, that the merged `count` is within 3% of their sum, and that `assert_quantiles_within_error` holds at `{0, 0.1, 0.25, 0.5, 0.75, 0.9, 1}` within a rank tolerance of `0.03` against the sorted union. | -| `bulk_update_equivalent_to_loop_and_empty_is_noop` | `update_batch` is the loop it replaces, and an empty batch changes nothing. | Verifies `update_batch(&[])` on a seed-42 `init_with_seed(200, 8, 42)` sketch holding `[1.0, 2.0, 3.0]` leaves `count`, `quantile(0.5)`, and the serialized bytes untouched; that 20,000 uniform samples in `[0, 1000)` (seed `0xBEEF_1234`) through `update_batch` match the one-at-a-time loop on `count`, bytes, and quantiles `{0.1, 0.5, 0.9}` for two seed-99 sketches; that 100 `DataInput::F64` values through `update_batch_data_input` match the loop's bytes at seed 77; and that `update_batch_data_input` over `[F64(1.0), String("x"), F64(2.0)]` errors with `count` left at `1`. | +| `bulk_update_equivalent_to_loop_and_empty_is_noop` | `update_batch` is the loop it replaces, and an empty batch changes nothing. | Verifies `update_batch(&[])` on a seed-42 `with_seed(200, 8, 42)` sketch holding `[1.0, 2.0, 3.0]` leaves `count`, `quantile(0.5)`, and the serialized bytes untouched; that 20,000 uniform samples in `[0, 1000)` (seed `0xBEEF_1234`) through `update_batch` match the one-at-a-time loop on `count`, bytes, and quantiles `{0.1, 0.5, 0.9}` for two seed-99 sketches; that 100 `DataInput::F64` values through `update_batch_data_input` match the loop's bytes at seed 77; and that `update_batch_data_input` over `[F64(1.0), String("x"), F64(2.0)]` errors with `count` left at `1`. | ### KllDynamic @@ -185,7 +185,7 @@ Wire tests: [`src/sketches/kll_dynamic/wire.rs`](../src/sketches/kll_dynamic/wir | `test_data_input_api` | `KllDynamic` accepts numeric `DataInput` and rejects non-numeric input. | Inserts `I32`, `I64`, `F64`, `F32`, and `U32` values through `update_data_input`, checks median query lies between `20.0` and `40.2`, and verifies string input returns error `Kll sketch only accepts numeric inputs`. | | `test_forced_compact` | Small-capacity KllDynamic triggers compaction and keeps median in valid compacted outcomes. | With `KllDynamic::new(3,3)` and typed inserts `[10,20,30,40,50]`, asserts median query is one of `{30.0, 40.0}` under forced compaction. | | `test_no_compact` | Larger-capacity KllDynamic avoids compaction for small stream and returns exact median. | With `KllDynamic::with_k(8)` and typed inserts `[10,20,30,40,50]`, asserts median query equals `30.0`. | -| `merge_preserves_quantiles_within_tolerance` | Merging two KllDynamic sketches preserves quantiles within 2% rank tolerance. | Splits 10,000 uniform samples (`1,000,000..10,000,000`, seed `0xC0FFEE`) across two `k=200` sketches by index parity, merges, and checks quantiles `{0,0.1,0.25,0.5,0.75,0.9,1}` remain within `q +/- 0.02` truth bounds. | +| `merge_preserves_quantiles_within_tolerance` | Merging two KLL dynamic sketches preserves quantiles within 2% rank tolerance. | Splits 10,000 uniform samples (`1,000,000..10,000,000`, seed `0xC0FFEE`) across two `k=200` sketches by index parity, merges, and checks quantiles `{0,0.1,0.25,0.5,0.75,0.9,1}` remain within `q +/- 0.02` truth bounds. | | `cdf_handles_empty_sketch` | Empty KllDynamic CDF queries return zero-valued defaults. | For empty `KllDynamic::::with_k(64)`, asserts `cdf.normalized_rank(123.0) == 0.0`, `cdf.value_at(0.5) == 0.0`, and `cdf.value_at_interpolated(0.5) == 0.0`. | | `kll_dynamic_round_trip_rmp` | RMP round trip preserves KllDynamic structure, packed data, and queried quantiles. | Serializes/deserializes `KllDynamic::with_k(256)` after 5,000 uniform updates (`0..1,000,000`, seed `0xDEAD_BEEF`), verifies non-empty bytes, core fields and packed arrays (`levels`, `items`) are identical, and CDF queries at `{0,0.1,0.25,0.5,0.75,0.9,1}` match within `f64::EPSILON`. | | `generic_kll_dynamic_i64_sanity` | Generic `KllDynamic` path works for non-`f64` numeric types. | Builds `KllDynamic`, inserts `1..=20_000` through the typed `update(&T)` API, checks approximate count and p50/p90 quantiles, and confirms MessagePack round-trip preserves weighted count. | @@ -195,7 +195,7 @@ Wire tests: [`src/sketches/kll_dynamic/wire.rs`](../src/sketches/kll_dynamic/wir | `kll_dynamic_item_type_cross_rejection` | The metadata `item_type` pins the element type. | Verifies an `f64` `KllDynamic` envelope fails to decode as a `KllDynamic`. | | `merge_into_empty_target_preserves_weight_issue_68_repro` | Merging into an empty target preserves the total weight exactly. | Builds a `k=200` source over `1..=1000`, checks its `count` falls in `980..=1020`, merges it into an empty `k=200` target, and verifies the target's `count` equals the source's exactly and its median lands in `475.0..=525.0`. | | `merge_two_nonempty_sketches_preserves_weight_and_quantiles` | Merging two non-empty sketches is a genuinely weighted merge, not just the empty-target special case. | Builds one `k=200` sketch over 50,000 uniform samples in `[0, 1_000_000)` (seed `0xA11CE`) and another over 50,000 Zipf samples (8,192 distinct, exponent `1.1`, seed `0xB0B`), verifies each pre-merge `count` is within 3% of `50_000`, that the merged `count` is within 3% of their sum, and that `assert_quantiles_within_error` holds at `{0, 0.1, 0.25, 0.5, 0.75, 0.9, 1}` within a rank tolerance of `0.03` against the sorted union. | -| `merge_handles_operand_level_that_is_not_a_single_sorted_run` | A merge sorts an operand level that is a concatenation of separately-sorted chunks. | Hand-builds a `KllDynamic::::new(50, 4)` operand whose level 1 is `[30.0, 40.0, 10.0, 20.0]` - two ascending chunks out of order - with `levels` `[0, 4, 5]` and `num_levels` `2`, checks `capacity_for_level(1)` is at least `4` so the level is never itself compacted, merges it into an empty `init(50, 4)` target, and verifies every level above 0 is a single ascending run afterwards. | +| `merge_handles_operand_level_that_is_not_a_single_sorted_run` | A merge sorts an operand level that is a concatenation of separately-sorted chunks. | Hand-builds a `KllDynamic::::new(50, 4)` operand whose level 1 is `[30.0, 40.0, 10.0, 20.0]` - two ascending chunks out of order - with `levels` `[0, 4, 5]` and `num_levels` `2`, checks `capacity_for_level(1)` is at least `4` so the level is never itself compacted, merges it into an empty `new(50, 4)` target, and verifies every level above 0 is a single ascending run afterwards. | | `bulk_update_equivalent_to_loop` | `update_batch` tracks the loop it replaces. | Feeds 10,000 uniform samples in `[0, 1000)` (seed `0xCAFE_1234`) to two `with_k(200)` sketches, one value at a time and one through `update_batch`, and verifies their counts agree within 5% relative; then that 100 `DataInput::F64` values through `update_batch_data_input` give the same `count` as the loop. Only counts are compared - quantiles and bytes are not, since the wall-clock `Coin` makes them non-deterministic. | ### DDSketch @@ -1014,8 +1014,8 @@ Wire tests: [`src/sketch_framework/univmon_q/wire.rs`](../src/sketch_framework/u | `univmon_q_rejects_serializing_an_inconsistent_state` | The encode side refuses states its own decoder would reject. | Verifies a level whose `PackedCountSketch` is not the config's size, a `count`/`min`/`max` triple the algorithm cannot reach, and a candidate table over its declared capacity each fail to serialize. | | `univmon_q_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the fourteen `UnivMonQMetadata` fields plus a `bogus_field`, and the same fields with `counter_type` omitted, and verifies neither decodes as `UnivMonQMetadata`. | | `exact_l1_and_occurrence_entropy_for_known_distribution` | A two-value distribution's L1 and entropy come back exactly. | For a 8-level, 4096-wide, depth-5 sketch with 1 candidate, 100 ordered samples and `seed_index` `17` over 80 copies of `0.0` and 20 of `1.0`, verifies `estimate_l1` is `100.0`, that `estimate_entropy_from_samples` and `estimate_entropy` both land within `1e-12` of the exact `-(0.8 ln 0.8 + 0.2 ln 0.2)`, and that `prepare_queries` reports the same L1 and entropy. | -| `candidate_eviction_history_distinguishes_full_from_truncated` | A candidate table that is merely full has not evicted anything. | On a `Level::new(32, 1, 64, 2)` - width 32, depth 1, 64-bit counters, 2 candidates - verifies `ever_evicted` stays false after the two updates that fill the table, flips on the third key, and is reset by `clear`. | -| `candidate_merge_uses_zero_error_for_full_complete_summary` | A merge re-scores the union from the merged sketch and keeps the heaviest. | Merges a 2-candidate `Level` holding key 3 at 2 into one holding key 1 at 100 and key 2 at 1, and verifies the merge sets `ever_evicted` because the 3-key union exceeds the capacity, and that `candidate_scores` reads exactly `100` for key 1 and `2` for key 3 with key 2 dropped; no `error` field is inspected. | +| `candidate_eviction_history_distinguishes_full_from_truncated` | A candidate table that is merely full has not evicted anything. | On a `Layer::new(32, 1, 64, 2)` - 32 columns, 1 row, 64-bit counters, 2 candidates - verifies `ever_evicted` stays false after the two updates that fill the table, flips on the third key, and is reset by `clear`. | +| `candidate_merge_uses_zero_error_for_full_complete_summary` | A merge re-scores the union from the merged sketch and keeps the heaviest. | Merges a 2-candidate `Layer` holding key 3 at 2 into one holding key 1 at 100 and key 2 at 1, and verifies the merge sets `ever_evicted` because the 3-key union exceeds the capacity, and that `candidate_scores` reads exactly `100` for key 1 and `2` for key 3 with key 2 dropped; no `error` field is inspected. | | `asapv1_envelope_preserves_candidate_eviction_history` | The per-level eviction flag travels on the wire. | For a 2-level, 1-candidate sketch over `[1.0, 2.0, 3.0]` that has evicted on some level, verifies the decoded sketch's per-level `ever_evicted` vector equals the source's. | | `l2_heavy_candidate_survives_a_diffuse_tail` | An item above twice the L2 threshold stays recoverable under 50,000 distinct competitors. | In a 2-level, 4096-wide, depth-5 sketch with 32 candidates and no ordered samples, searches for a value that samples to the terminal level, adds it 200 times, then adds 50,000 further distinct values that also sample there; verifies `200` exceeds twice `sqrt(F2/32)` for the exact `200^2 + 50_000`, that the level has evicted, that `candidate_scores` still holds the target key, and that `logical_heavy_sets_from(&recover_candidates())` lists it. | | `exact_for_tiny_frequency_vector` | A seven-element stream is answered exactly by every query. | For the 8-level, 128-wide, depth-5, 32-candidate, 32-sample config over `[5, 1, 2, 2, 9, 5, 5]`, verifies `rank(2.0)` and `estimate_rank_from_g_sum(2.0)` are `Some(3)`, `rank(5.0)` is `Some(6)`, quantiles `0.0`/`0.5`/`1.0` are `1.0`/`5.0`/`9.0`, `estimate_frequency(5.0)` is `3`, `estimate_cardinality` is `4.0`, `estimate_f2` is `15.0`, the linear `estimate_g_sum` is `7.0`, `heavy_hitters(1)` is `[(5.0, 3)]`, and that `prepare_queries` reproduces all of those plus `count`, `quantiles`, and `cdf`. | @@ -1056,7 +1056,7 @@ Wire tests: [`src/sketch_framework/eh/wire.rs`](../src/sketch_framework/eh/wire. | test_name | test_description | what_is_tested | | --- | --- | --- | -| `constructor_infers_merge_norm` | Constructor infers merge norm. | Verifies constructor infers `SketchNorm::L1` for CM payload and `SketchNorm::L2` for `COUNTL2HH` payload. | +| `constructor_infers_merge_norm` | Constructor infers merge norm. | Verifies constructor infers `SketchNorm::L1` for the `CountMinSketch` payload and `SketchNorm::L2` for the `CountSketchWithL2Norm` payload. | | `l1_merge_invariant_same_size` | L1 merge invariant same size. | Under repeated updates with `k=2`, verifies L1 merge policy compacts buckets so `bucket_count < 10`. | | `l2_merge_invariant_sum_l22` | L2 merge invariant sum l2_squared. | With `k=1` and weighted updates, verifies L2 merge rule keeps bucket count bounded (`bucket_count <= 2`). | | `merge_recomputes_l2_mass` | Merge recomputes L2 mass. | After L2 merges, verifies bounded bucket count (`<=2`) and non-negative recomputed `l2_mass` for every payload bucket. | @@ -1085,10 +1085,10 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | test_name | test_description | what_is_tested | | --- | --- | --- | -| `insert_routes_to_countl2hh_and_univmon` | Insert routes to countl2hh and univmon. | Verifies variant routing by checking `COUNTL2HH` estimate `>=9` after 9 inserts and `UNIVMON` `bucket_size=6` after 6 inserts. | +| `insert_routes_to_countl2hh_and_univmon` | Insert routes to countl2hh and univmon. | Verifies variant routing by checking `CountSketchWithL2Norm` estimate `>=9` after 9 inserts and `UnivMon` `total_weight=6` after 6 inserts. | | `count_sketch_insert_and_query_round_trip` | Count insert and query round trip. | Confirms the `CountSketch` variant updates/query path by inserting one key and verifying returned estimate is at least `1.0`. | | `ddsketch_insert_and_quantile_query_round_trip` | DDSketch insert and quantile query round trip. | Inserts `10,20,30` into DDSketch variant and verifies queried median (`q=0.5`) lies within `[10.0, 30.0]`. | -| `supports_norm_whitelist_is_enforced` | Supports norm whitelist is enforced. | Validates norm capability matrix: `CM/CS/DDS` support `L1` only, while `COUNTL2HH/UNIVMON` support `L2` only. | +| `supports_norm_whitelist_is_enforced` | Supports norm whitelist is enforced. | Validates norm capability matrix: `CountMinSketch`/`CountSketch`/`DdSketch` support `L1` only, while `CountSketchWithL2Norm`/`UnivMon` support `L2` only. | | `eh_sketch_list_round_trip_serialization` | The envelope frames a union under kind_id `0x14 0x00`, and the state survives a round trip. | For a `3x8` fast-path Count-Min variant holding one key, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x14 0x00`, and that the decode keeps a `sketch_type` of `CountMinSketch` and answers that key as the source does. | | `eh_sketch_list_every_variant_round_trips` | Every variant this build carries round-trips and keeps its type. | For each populated variant, verifies the decode reports the same `sketch_type` and re-serializes to the bytes it came from. | | `eh_sketch_list_kind_ids_are_build_independent` | The ten nested kind_ids and their registry names are the same in every build. | Verifies `variant_name` maps each of the ten ids to its name and `0xff 0xff` to `None`, and that every populated variant emits the id the table gives it. | @@ -1194,7 +1194,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `flushing_twice_is_harmless` | A second flush changes nothing. | Inserts 5,000 values into a `DdOctoPlan` runtime at alpha `0.01` on 2 workers, flushes twice, and verifies `finish` reports a count of `5_000`. | | `a_flushed_count_min_parent_matches_a_single_pass_exactly` | A finished Count-Min parent equals a single-threaded pass cell for cell. | Runs 40,000 inputs `U64(i % 512)` through a `3x1024` `CountMinSketchOctoPlan` on 4 workers and verifies all 3,072 counters equal a single-threaded reference's. | | `octo_runtime_close_is_idempotent` | Closing twice is allowed. | Calls `close` twice on a 2-worker `HllOctoPlan` runtime that saw no inserts and verifies `finish` estimates `0`. | -| `octo_runtime_insert_after_close_panics` | An insert after `close` panics. | Verifies inserting `U64(1)` into a closed 2-worker `HllOctoPlan` runtime panics with "cannot insert after runtime has been closed". | +| `octo_runtime_insert_after_close_panics` | An insert after `close` panics. | Verifies inserting `U64(1)` into a closed 2-worker `HllOctoPlan` runtime panics with "cannot update after runtime has been closed". | | `octo_runtime_empty_stream_finishes` | A runtime that saw nothing still finishes. | Verifies `finish` on a 4-worker `HllOctoPlan` runtime with no inserts estimates `0`. | | `octo_runtime_live_read_handle_tracks_the_aggregator` | A read handle reaches the live parent before the run is finished. | Inserts 64 inputs into a 2-worker counting runtime and polls the read handle, verifying it reads `0` before any insert, never goes backwards, never exceeds `64`, reaches exactly `64` inside a 10-second deadline, and that `finish` then reports `64`. | | `octo_runtime_close_preserves_queued_items` | `close` does not discard what is already queued. | Inserts 257 inputs into a 4-worker counting runtime, calls `close`, and verifies `finish` reports `257`. | @@ -1243,7 +1243,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `the_read_handle_observes_a_monotone_prefix_of_the_final_state` | The live read handle is monotone and reaches the live total before `finish`. | Under `--features octo-runtime`, drives `U64(0..4_000)` through an `OctoRuntime` over `SumPlan` on `2` workers, and verifies the handle reads `0` before the first insert, that every reading sampled at each 500th insert is non-decreasing and at most `4_000`, that it then reaches exactly `4_000` inside a 10-second `Instant`-based deadline while the runtime is still open, and that `finish` reports `4_000`. | | `reading_through_a_stale_handle_panics` | Reading a handle after `finish` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "Octo runtime has been finished")]` test that takes a `read_handle()` from a 2-worker `SumPlan` runtime, calls `finish()`, then calls `with_parent` on the stale handle. | | `close_is_idempotent_and_preserves_already_queued_work` | Closing twice is allowed and drains what was queued. | Under `--features octo-runtime`, inserts `U64(0..3_001)` into a 4-worker `SumPlan` runtime, calls `close()` twice, and verifies `finish` reports a per-worker total of exactly `3_001`. | -| `inserting_after_close_panics` | An insert after `close` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "cannot insert after runtime has been closed")]` test that closes a 2-worker `SumPlan` runtime and then inserts `U64(1)`. | +| `inserting_after_close_panics` | An insert after `close` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "cannot update after runtime has been closed")]` test that closes a 2-worker `SumPlan` runtime and then inserts `U64(1)`. | | `a_finished_run_carries_only_count_mins_own_one_sided_error` | A flushed Count-Min run has only Count-Min's own one-sided error left. | Under `--features octo-runtime`, runs 200,000 zipf(1.1) keys, domain `4_096`, seed `9_501`, through `CountMinSketchOctoPlan::new(5, 4096)` on `4` workers and verifies for every one of the 4,096 keys that the estimate is at or above the exact `FreqTruth` count and at most `exact + (e / 4096) * N` above it. | | `run_octo_hll_cardinality_error_stays_within_three_sigma` | The HLL runtime is register-exact and its cardinality is within three sigma. | Under `--features octo-runtime`, runs the 200,000 distinct keys `U64(0..200_000)` through `HllOctoPlan` on `4` workers and verifies the parent's registers equal a single-threaded reference's exactly, then that the relative error of `estimate()` against `200_000` is under `0.025`. | | `run_octo_count_frequencies_track_a_zipf_stream` | A flushed Count-sketch run holds the Count-sketch L2 bound on the heavy keys. | Under `--features octo-runtime`, runs 200,000 zipf(1.1) keys, domain `2_048`, seed `9_502`, through `CountSketchOctoPlan::new(5, 4096)` on `4` workers, then for the truth's top `64` keys collects any estimate whose absolute error exceeds `3 * \|\|f\|\|_2 / sqrt(4096)` and asserts the violation list is empty. | @@ -2038,17 +2038,17 @@ Test file: [`tests/e2e/windows.rs`](../tests/e2e/windows.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `every_eh_variant_selects_the_documented_merge_norm` | Each `EhBucketSketch` payload selects the merge norm its enclosing histogram uses. | For `COUNTL2HH` (`CountSketchWithL2Norm::with_dimensions(3, 512)`) and `UNIVMON` (`UnivMon::new(32, 3, 512, 4)`) asserts `supports_norm(SketchNorm::L2)` and `!supports_norm(SketchNorm::L1)`, and that `ExponentialHistogram::new(8, 1_000_000, proto).merge_norm` is `SketchNorm::L2`; for `CM` and `CS` (`Vector2D` `FastPath` at `3x512`), `COCO(512, 4)`, `ELASTIC(512)`, `HLL`, `Kll(k=200, seed 0x5EED_0100)` and `DDS(alpha=0.01)` asserts `supports_norm(SketchNorm::L1)` and `merge_norm == SketchNorm::L1`. | +| `every_eh_variant_selects_the_documented_merge_norm` | Each `EhBucketSketch` payload selects the merge norm its enclosing histogram uses. | For `CountSketchWithL2Norm` (`CountSketchWithL2Norm::with_dimensions(3, 512)`) and `UnivMon` (`UnivMon::new(32, 3, 512, 4)`) asserts `supports_norm(SketchNorm::L2)` and `!supports_norm(SketchNorm::L1)`, and that `ExponentialHistogram::new(8, 1_000_000, proto).merge_norm` is `SketchNorm::L2`; for `CountMinSketch` and `CountSketch` (`Vector2D` `FastPath` at `3x512`), `CocoSketch(4, 512)`, `ElasticSketch(512)`, `Hll`, `Kll(k=200, seed 0x5EED_0100)` and `DdSketch(alpha=0.01)` asserts `supports_norm(SketchNorm::L1)` and `merge_norm == SketchNorm::L1`. | | `eh_count_min_variant_conforms_to_the_count_min_model_over_the_retained_window` | A Count-Min payload satisfies Count-Min's theorem over the histogram's full retained span. | Feeds `10_000` Zipf(1.1) keys over a `2_048` domain (seed `0x0E11_0001`), one per timestamp `t = 0..9_999`, into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, merges the full retained span with `query_interval(payload.first().min_time, payload.last().max_time)`, and runs `CountMinSpec::new(3, 512).assert_contract` over the exact truth for that span: one-sidedness `est >= f` with zero violations, the simultaneous budget `b*(N-f)/w` with `b = (D/1e-3)^(1/3)` and zero violations, and the marginal `e*(N-f)/w` as a rate pin at `e^-3 = 0.0498`. | | `eh_count_sketch_variant_conforms_to_the_l2_model_over_the_retained_window` | A Count Sketch payload satisfies the two-sided L2 bound over the retained span. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketch(CountSketch::, FastPath>::with_dimensions(3, 512))`; the full-span merge is judged by `CountSketchSpec::new(3, 512).assert_contract`, which requires `\|est - f\|` under `sqrt(kappa/512)*\|\|f_-i\|\|_2` at the simultaneous `kappa` (union-bounded to `delta = 1e-3` over every probed key, zero violations) and pins the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` violation rate at `P[Bin(3, 1/3) >= 2] = 7/27`. | | `eh_countl2hh_variant_satisfies_the_l2_bound_over_the_retained_window` | A `CountSketchWithL2Norm` payload keeps the L2 bound of the Count Sketch matrix underneath it. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketchWithL2Norm(CountSketchWithL2Norm::with_dimensions(3, 512))`; the full-span merge is held to `CountSketchSpec::new(3, 512).assert_contract` - the simultaneous L2 band at `delta = 1e-3` with zero violations and the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` rate pin - reading frequencies through `merged.query(&DataInput::U64(k))`. | -| `eh_elastic_variant_stays_one_sided_over_the_retained_window` | The Elastic payload never reads a retained heavy key below its true count. | For `ELASTIC(1024)` inside `ExponentialHistogram::new(8, 1_000_000, ..)`, feeds `10_000` Zipf(1.1) draws over a `2_048` domain (seed `0x0E11_0001`) as `DataInput::String("f{k}")` at `t = 0..9_999`, merges the full retained span, and asserts `est >= c` for the truth's top `32` keys with zero violations. No upper bound is placed on the reported counts, and keys outside the top `32` are not probed. | -| `eh_coco_variant_conserves_the_retained_mass` | The merged Coco window holds exactly the retained mass. | Same EH, stream and merge over `COCO(CocoSketch::with_dimensions_and_seed(4, 1024, COCO_SEED))`, and verifies the merged table's recorded mass equals the number of events in the retained span. | +| `eh_elastic_variant_stays_one_sided_over_the_retained_window` | The Elastic payload never reads a retained heavy key below its true count. | For `ElasticSketch(1024)` inside `ExponentialHistogram::new(8, 1_000_000, ..)`, feeds `10_000` Zipf(1.1) draws over a `2_048` domain (seed `0x0E11_0001`) as `DataInput::String("f{k}")` at `t = 0..9_999`, merges the full retained span, and asserts `est >= c` for the truth's top `32` keys with zero violations. No upper bound is placed on the reported counts, and keys outside the top `32` are not probed. | +| `eh_coco_variant_conserves_the_retained_mass` | The merged Coco window holds exactly the retained mass. | Same EH, stream and merge over `CocoSketch(CocoSketch::with_dimensions_and_seed(4, 1024, COCO_SEED))`, and verifies the merged table's recorded mass equals the number of events in the retained span. | | `eh_hll_variant_satisfies_the_register_error_model_over_the_retained_window` | A merged HLL payload lands inside the register error model for the retained span's cardinality. | Feeds `10_000` uniform draws over `200_000` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Hll(HyperLogLog::::new()))`, merges the full retained span, and checks `merged.query(&DataInput::Str("card"))` against the exact distinct count of that span under `CardinalityConfidenceSpec::hll(14, 4.0)` - a relative band of `4 * 1.04 / 2^7 = 0.0325` at `m = 2^14` registers. This is a single trial, aggregated by `assert_independent_binomial` at `p = 6.3e-5`, so no violation is tolerated. | | `eh_kll_variant_satisfies_the_rank_error_characterization_over_the_retained_window` | A merged KLL payload's worst rank error stays inside the DataSketches characterization across independent compaction seeds. | Builds `12` separate `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Kll(Kll::with_k_and_seed(200, seed)))` instances with `seed = 0x5EED_0200 + t * 0x9E37_79B9_7F4A_7C15`, feeds each the same `10_000` uniform values over `1_000_000` (stream seed `0x0E11_0001`) at `t = 0..9_999`, merges each one's full retained span, and reduces each trial to its worst normalized rank error over the grid `[0.1, 0.25, 0.5, 0.75, 0.9]` read through `merged.query(&DataInput::F64(q))`; `KllRankSpec::datasketches(200)` supplies `eps = 2.446/200^0.9433 = 0.0165`, and the `12` trial outcomes are accepted by `assert_independent_binomial` at `p = 0.01`. | | `eh_ddsketch_variant_satisfies_the_relative_value_error_contract_over_the_window` | A merged DDSketch payload holds relative value error and retains every observation. | Feeds `10_000` values `1_000_000 + uniform[0, 9_000_000)` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::DdSketch(DdSketch::new(0.01)))`, merges the full retained span, and checks `merged.query(&DataInput::F64(q))` at `q` in `[0.1, 0.25, 0.5, 0.75, 0.9]` against the span's exact order statistics under `RelativeQuantileSpec::core(0.01)` (`CeilNearestRank` convention, tolerance `alpha` plus a few-ULP slack) with zero violations; then asserts `merged.query(&DataInput::Str("count"))` equals the span's observation count exactly. | | `eh_univmon_variant_reports_the_exact_l1_over_the_retained_window` | A merged UnivMon payload reports L1 exactly and L2 inside the AMS second-moment band of its own grid. | Feeds `10_000` Zipf(1.1) keys over a `2_048` domain (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::UnivMon(UnivMon::new(32, 5, 2_048, 8)))`, merges the full retained span, and asserts `query(&DataInput::Str("l1"))` equals the span's exact total mass; then asserts `query(&DataInput::Str("l2"))` lies in `[\|\|f\|\|_2 * sqrt(1-b), \|\|f\|\|_2 * sqrt(1+b)]` with `b = SecondMomentSpec::new(5, 2_048).relative_bound() = sqrt(2*3/2048) = 0.0541`, i.e. about `-2.7%/+2.7%` on the norm. The bound asserted is the terminal layer's, not a composed constant for the recurrence. | -| `every_eh_variant_can_merge_into_its_own_kind` | Every mergeable `EhBucketSketch` variant has a merge arm, so bucket consolidation cannot silently drop a payload. | For `CM` and `CS` at `3x256`, `COCO(256, 4)`, `COUNTL2HH` at `3x256`, `DDS(0.01)`, `ELASTIC(256)`, `HLL`, `Kll(200, seed 0x5EED_0300)`, `UNIVMON(32, 3, 256, 4)` and - under `--features experimental` - `UNIFORM(UniformSampling::with_seed(0.5, 7))`, feeds each side `64` distinct `DataInput::String` keys and `64` distinct `DataInput::F64` values (`k0..k63` / `1.0..64.0` on the left, `k64..k127` / `100.0..163.0` on the right) and asserts `left.merge(&right).is_ok()`. Only the `Ok` status is asserted; the merged contents are not read back. | +| `every_eh_variant_can_merge_into_its_own_kind` | Every mergeable `EhBucketSketch` variant has a merge arm, so bucket consolidation cannot silently drop a payload. | For `CountMinSketch` and `CountSketch` at `3x256`, `CocoSketch(4, 256)`, `CountSketchWithL2Norm` at `3x256`, `DdSketch(0.01)`, `ElasticSketch(256)`, `Hll`, `Kll(200, seed 0x5EED_0300)`, `UnivMon(32, 3, 256, 4)` and - under `--features experimental` - `Uniform(UniformSampling::with_seed(0.5, 7))`, feeds each side `64` distinct `DataInput::String` keys and `64` distinct `DataInput::F64` values (`k0..k63` / `1.0..64.0` on the left, `k64..k127` / `100.0..163.0` on the right) and asserts `left.merge(&right).is_ok()`. Only the `Ok` status is asserted; the merged contents are not read back. | | `eh_uniform_sampling_variant_reports_exact_retention_bookkeeping` | The reservoir payload's retention counters are exact over the merged span and its samples were all observed. | Gated on `#[cfg(feature = "experimental")]`. Feeds `10_000` uniform values over `1_000_000` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Uniform(UniformSampling::with_seed(0.1, 0x5A_9101)))`, merges the full retained span, and asserts `query(&DataInput::Str("total_seen"))` equals `hi - lo + 1` exactly, that `query(&DataInput::Str("len"))` is greater than `0` and at most `total_seen`, and that every sample index `0..len` returns a value whose bit pattern appears in the span. The retained count is not held against the `0.1` rate's budget beyond `<= total_seen`. | ### Windows: Exponential Histogram Semantics @@ -2132,7 +2132,7 @@ Test file: [`tests/e2e/matrix_instances.rs`](../tests/e2e/matrix_instances.rs) | `csheap_{regular_path_instances_satisfy_the_l2_bound, fast_path_instances_conform_to_the_l2_model}` | Every `CountSketchWithHeap` instance carries the L2 bound and keeps its heap consistent with the sketch. | One shared body per storage: three `CountSketchWithHeap` sketches over `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, whole stream to one and even/odd halves to two `merge`d shards, then `CountSketchSpec::new(rows, cols)` over both - simultaneous `sqrt(kappa/w)*\|\|f_-i\|\|_2` at the union-bounded `kappa` for `delta = 1e-3` with zero tolerance, marginal band at `kappa = 3` rate-pinned - and `assert_heap_matches_sketch` with `heap_count_from_estimate` as the estimate-to-entry map, requiring every heap entry's `count` to equal `heap_count_from_estimate(estimate)` and at most one entry to sit below the true k-th count. Nine instances each: `Vector2D`, `Vector2D`, `DefaultMatrixI32`, `DefaultMatrixI64`, `DefaultMatrixI128` from `Default` at `3x4096`, `FixedMatrix`, `QuickMatrixI64`, `QuickMatrixI128` from `Default` at `5x2048`, and `Vector2D` built explicitly by `CountSketchWithHeap::new(3, 4096, 32)` because it has no `Default`. Generated as `csheap_regular_path_instances_satisfy_the_l2_bound` (`RegularPath`) and `csheap_fast_path_instances_conform_to_the_l2_model` (`FastPath`). | | `countmin_both_paths_are_exact_on_a_collision_free_workload` | On a collision-free workload both hashing paths return exact counts on every storage backend. | Inserts the eight well-separated keys `1`, `7`, `4_242`, `90_210`, `1_000_003`, `2_147_483_647`, `4_294_967_311`, `9_007_199_254_740_993` at multiplicities `10, 20, ..., 80` into a `RegularPath` and a `FastPath` `CountMinSketch` over the same storage, and requires both `estimate` calls to equal the true count exactly for every key, with zero violations. Run over all ten storages: `Vector2D`, `Vector2D`, `Vector2D`, `Vector2D`, `FixedMatrix`, `DefaultMatrixI32`, `QuickMatrixI64`, `QuickMatrixI128`, `DefaultMatrixI64`, `DefaultMatrixI128`. | | `countsketch_both_paths_are_exact_on_a_collision_free_workload` | The signed family is likewise exact on both paths when nothing collides. | Same eight keys at multiplicities `10, 20, ..., 80` into a `RegularPath` and a `FastPath` `CountSketch` over the same storage, requiring both `estimate` values to equal the true count exactly with zero violations, over nine storages: `Vector2D`, `Vector2D`, `Vector2D`, `FixedMatrix`, `DefaultMatrixI32`, `QuickMatrixI64`, `QuickMatrixI128`, `DefaultMatrixI64`, `DefaultMatrixI128`. | -| `cmsheap_inert_instances_construct_and_report_their_geometry` | The `CountMinSketchWithHeap` instances whose counter type fails the insert bound still construct and answer their geometry. | For each of six instances, verifies `rows()`, `cols()`, `cms().rows()` and an empty `heap()`: `CountMinSketchWithHeap::, _>::new(3, 4096, 32)` on both paths at `3x4096`, `CountMinSketchWithHeap::::default()` on both paths at `5x2048`, and `CountMinSketchWithHeap::::default()` on both paths at `3x4096`. This is constructibility only: the test does not assert that `update`, `update_weighted`, `estimate` or `merge` are uncallable, and adding such an impl would leave every assertion here passing. | +| `cmsheap_inert_instances_construct_and_report_their_geometry` | The `CountMinSketchWithHeap` instances whose counter type fails the insert bound still construct and answer their geometry. | For each of six instances, verifies `rows()`, `cols()`, `count_min_sketch().rows()` and an empty `heap()`: `CountMinSketchWithHeap::, _>::new(3, 4096, 32)` on both paths at `3x4096`, `CountMinSketchWithHeap::::default()` on both paths at `5x2048`, and `CountMinSketchWithHeap::::default()` on both paths at `3x4096`. This is constructibility only: the test does not assert that `update`, `update_weighted`, `estimate` or `merge` are uncallable, and adding such an impl would leave every assertion here passing. | | `csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping` | An `i128` `CountSketchWithHeap` holds mass past `i64::MAX` while its heap entry saturates at the ceiling. | For each of six instances, inserts `update_weighted(U64(0xC0FF_EE01), 1 << 40)` and requires `estimate` to equal `2^40` exactly and the heap entry to carry the same `i64`; then, on a fresh sketch, `update_weighted` of `i64::MAX as i128 * 4` and requires `estimate >= i64::MAX as f64`, `heap_count_from_estimate(estimate) == i64::MAX`, and the heap entry to read exactly `i64::MAX` rather than a wrapped negative; then merges a second sketch carrying the same overflow weight and requires the heap entry to stay at `i64::MAX`. Instances: `CountSketchWithHeap::, _>::new(3, 4096, 32)` on both paths, and `QuickMatrixI128` and `DefaultMatrixI128` from `Default` on both paths. | | `countmin_counter_widths_carry_the_mass_their_type_allows` | Each `CountMinSketch` counter width carries a count the next narrower one cannot. | On `3x64` `RegularPath` sketches keyed by `U64(0xFEED_FACE)`: `Vector2D` returns `i32::MAX - 1` after `update_weighted` of that weight; `Vector2D` returns `i32::MAX as i64 * 4`; `Vector2D` returns `i64::MAX as i128 * 4`. Then merges two `Vector2D` `3x64` sketches each holding `i64::MAX` and requires the result to be exactly `i64::MAX as i128 * 2`. | | `countsketch_counter_widths_carry_signed_mass_in_both_directions` | Each `CountSketch` counter width reaches the negative end of its range without wrapping. | On `3x64` `Vector2D<_>` `RegularPath` sketches keyed by `U64(0xDEAD_BEEF)`: `i32` cancels `+i32::MAX/2` against `-(i32::MAX/2)` to exactly `0.0`; `i64` reads `i32::MAX as i64 * 4` and, after a further `-2x` that weight, exactly its negation; `i128` reads `i64::MAX as i128 * 4` and, after a further `-2x`, exactly its negation. | diff --git a/src/common/input.rs b/src/common/input.rs index 4eae6165..542e2f20 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -283,7 +283,7 @@ impl Hash for OwnedDataInput { /// Query type for Hydra sketches. #[derive(Clone, Debug)] pub enum HydraQuery<'a> { - /// Query for frequency of a specific item (for CountMinSketch, Count, etc.) + /// Query for frequency of a specific item (for CountMinSketch, CountSketch, etc.) Frequency(DataInput<'a>), /// Query for quantile/CDF at a threshold (for KLL, DDSketch, etc.) Quantile(f64), diff --git a/src/lib.rs b/src/lib.rs index 94174645..e14c1f95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,10 +9,11 @@ //! - [`sketch_framework`]: higher-level composition layers such as Hydra, //! UnivMon, tumbling windows, and batch/parallel execution helpers. //! - [`message_pack_format`]: MessagePack/proto wire format shared with -//! `sketchlib-go`. Owns the wire-DTO sketch types consumed by the ASAP -//! query engine (`CountMinSketch`, `CountSketch`, `KllSketch`, `HllSketch`, -//! `DdSketch`, `HydraKllSketch`, `CountMinSketchWithHeap`, -//! `SetAggregator`, `SetAggregatorDelta`). +//! `sketchlib-go`. `message_pack_format::portable` owns the legacy wire-DTO +//! sketch types consumed by the ASAP query engine (`portable::{CountMinSketch, +//! CountSketch, KllSketch, HllSketch, DdSketch, HydraKllSketch, +//! CountMinSketchWithHeap}`); `SetAggregator` and `SetAggregatorDelta` are +//! re-exported at the crate root. //! //! Most users can start with the crate-root re-exports such as [`DataInput`], //! [`CountMinSketch`], [`HyperLogLog`], [`Kll`], and [`DdSketch`]. Reach for the diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 30db6df7..c1b98298 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -1138,7 +1138,7 @@ mod tests { #[test] fn test_merge_counters() { - // Test merging two CountMinSketch sketches via the Hydra wrapper + // Test merging two Count-Min sketches via the Hydra wrapper let mut c1 = cm_counter(); let mut c2 = cm_counter(); diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 9d00a625..33a5dc3b 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -56,49 +56,49 @@ pub struct CountMinSketch< admission_rate: f64, } -// Default CountMinSketch sketch for Vector2D (RegularPath). +// Default Count-Min sketch for Vector2D (RegularPath). impl Default for CountMinSketch, RegularPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (FastPath). +// Default Count-Min sketch for Vector2D (FastPath). impl Default for CountMinSketch, FastPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (RegularPath). +// Default Count-Min sketch for Vector2D (RegularPath). impl Default for CountMinSketch, RegularPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (FastPath). +// Default Count-Min sketch for Vector2D (FastPath). impl Default for CountMinSketch, FastPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (RegularPath). +// Default Count-Min sketch for Vector2D (RegularPath). impl Default for CountMinSketch, RegularPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (FastPath). +// Default Count-Min sketch for Vector2D (FastPath). impl Default for CountMinSketch, FastPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) } } -// Default CountMinSketch sketch for Vector2D (RegularPath and FastPath). +// Default Count-Min sketch for Vector2D (RegularPath and FastPath). impl Default for CountMinSketch, RegularPath> { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS) @@ -111,84 +111,84 @@ impl Default for CountMinSketch, FastPath> { } } -// Default CountMinSketch sketch for FixedMatrix (RegularPath). +// Default Count-Min sketch for FixedMatrix (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(FixedMatrix::default()) } } -// Default CountMinSketch sketch for FixedMatrix (FastPath). +// Default Count-Min sketch for FixedMatrix (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(FixedMatrix::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI32 (RegularPath). +// Default Count-Min sketch for DefaultMatrixI32 (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI32::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI32 (FastPath). +// Default Count-Min sketch for DefaultMatrixI32 (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI32::default()) } } -// Default CountMinSketch sketch for QuickMatrixI64 (RegularPath). +// Default Count-Min sketch for QuickMatrixI64 (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(QuickMatrixI64::default()) } } -// Default CountMinSketch sketch for QuickMatrixI64 (FastPath). +// Default Count-Min sketch for QuickMatrixI64 (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(QuickMatrixI64::default()) } } -// Default CountMinSketch sketch for QuickMatrixI128 (RegularPath). +// Default Count-Min sketch for QuickMatrixI128 (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(QuickMatrixI128::default()) } } -// Default CountMinSketch sketch for QuickMatrixI128 (FastPath). +// Default Count-Min sketch for QuickMatrixI128 (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(QuickMatrixI128::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI64 (RegularPath). +// Default Count-Min sketch for DefaultMatrixI64 (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI64::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI64 (FastPath). +// Default Count-Min sketch for DefaultMatrixI64 (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI64::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI128 (RegularPath). +// Default Count-Min sketch for DefaultMatrixI128 (RegularPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI128::default()) } } -// Default CountMinSketch sketch for DefaultMatrixI128 (FastPath). +// Default Count-Min sketch for DefaultMatrixI128 (FastPath). impl Default for CountMinSketch { fn default() -> Self { CountMinSketch::from_storage(DefaultMatrixI128::default()) diff --git a/src/sketches/countminsketch_heap.rs b/src/sketches/countminsketch_heap.rs index 5ef61a86..3ecdb884 100644 --- a/src/sketches/countminsketch_heap.rs +++ b/src/sketches/countminsketch_heap.rs @@ -488,7 +488,7 @@ mod tests { #[test] fn insert_many_updates_estimate_and_heap() { - // Verifies batched insert_many updates both estimate and heap entry count. + // Verifies update_weighted updates both estimate and heap entry count. let mut sh = CountMinSketchWithHeap::, RegularPath>::new(3, 1024, 4); let key = DataInput::Str("many"); sh.update_weighted(&key, 11); diff --git a/src/sketches/countsketch_heap.rs b/src/sketches/countsketch_heap.rs index b9dcb285..07e6e7ff 100644 --- a/src/sketches/countsketch_heap.rs +++ b/src/sketches/countsketch_heap.rs @@ -561,7 +561,7 @@ mod tests { #[test] fn insert_many_updates_estimate_and_heap() { - // Verifies insert_many updates estimate and heap count consistently. + // Verifies update_weighted updates estimate and heap count consistently. let mut sh = CountSketchWithHeap::, RegularPath>::new(5, 2048, 4); let key = DataInput::Str("many"); sh.update_weighted(&key, 17); diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 0eef240d..8a2abf58 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -292,7 +292,7 @@ impl Kll { Self::new(k as usize, 8) } - /// `with_k` with an explicit RNG seed. See `init_with_seed`. + /// `with_k` with an explicit RNG seed. See `with_seed`. pub fn with_k_and_seed(k: i32, seed: u64) -> Self { Self::with_seed(k as usize, 8, seed) } @@ -731,7 +731,7 @@ impl Kll { // -- Lifecycle ----------------------------------------------------------- /// Resets the sketch to its empty initial state, keeping the same `k` and `m` parameters. - /// If the sketch was constructed with an explicit seed (`init_with_seed` / + /// If the sketch was constructed with an explicit seed (`with_seed` / /// `with_k_and_seed`), the coin is re-seeded from that seed so determinism /// survives `clear()` (and therefore window rotation in stateful aggregators). pub fn clear(&mut self) { @@ -1000,7 +1000,7 @@ impl<'de, T: NumericalValue + Deserialize<'de>> Deserialize<'de> for Kll { // round-tripped sketch keeps its current coin state but // would re-randomize from time on a subsequent clear(). // Callers that need clear()-determinism after a deserialize - // should rebuild via init_with_seed. + // should rebuild via with_seed. seed: None, capacity_cache: [0; CAPACITY_CACHE_LEN], top_height: 0, diff --git a/src/sketches/kll/wire.rs b/src/sketches/kll/wire.rs index f3ad8080..7cbe63c9 100644 --- a/src/sketches/kll/wire.rs +++ b/src/sketches/kll/wire.rs @@ -5,7 +5,7 @@ //! variants, the [`KllWireItem`] marker trait, and the `to_asapv1` / //! `from_asapv1` impls for the compact [`Kll`]. Being a descendant //! module, it reads the sketch's private fields (`self.items`, `self.levels`, -//! `self.co`, …) and constructs the struct directly without widening any field +//! `self.coin`, …) and constructs the struct directly without widening any field //! visibility. The sibling `kll_dynamic::wire` reuses these DTOs for the dynamic //! variant. See `docs/asapv1_wire_format.md` §3. //! diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index bbfdc65b..9311d656 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -73,12 +73,12 @@ impl Default for KllDynamic { } impl KllDynamic { - /// Creates a KllDynamic sketch with the given `k` and `m` parameters. + /// Creates a KLL dynamic sketch with the given `k` and `m` parameters. pub fn new(k: usize, m: usize) -> Self { Self::init_internal(k, m, Coin::new(), None) } - /// Creates a KllDynamic sketch with an explicit RNG seed for the + /// Creates a KLL dynamic sketch with an explicit RNG seed for the /// compaction coin. Two sketches built with the same seed and fed the same /// input produce identical state, which is what makes reproducible-replay /// scenarios and deterministic accuracy tests possible. The seed is also @@ -114,7 +114,7 @@ impl KllDynamic { s } - /// Creates a KllDynamic sketch with default `m` and the provided `k`. + /// Creates a KLL dynamic sketch with default `m` and the provided `k`. pub fn with_k(k: i32) -> Self { Self::new(k as usize, 8) } @@ -775,7 +775,7 @@ mod tests { ); } - // General case: merging two NON-empty KllDynamic sketches must still + // General case: merging two NON-empty KLL dynamic sketches must still // preserve total count (within the same inherent rounding budget as // ordinary inserts) and produce quantile estimates consistent with a // reference built from the union of both inputs' raw data. Guards diff --git a/src/sketches/mod.rs b/src/sketches/mod.rs index 67cf1861..1c1b8d14 100644 --- a/src/sketches/mod.rs +++ b/src/sketches/mod.rs @@ -22,7 +22,7 @@ //! sketches map values to rows and columns. Several specialized sketches in //! this module are feature-gated behind `experimental`. //! -//! ## `*sketch` vs `*sketch_topk` files +//! ## `*sketch` vs `*sketch_heap` files //! //! For each Count-Min / Count Sketch style algorithm we ship two files: //! @@ -36,7 +36,7 @@ //! specific key appear?" //! //! The split keeps the core sketch lean for callers that only need point -//! queries, while the `_topk` variants compose the sketch with heap +//! queries, while the `_heap` variants compose the sketch with heap //! bookkeeping for heavy-hitter workloads. pub mod coco; diff --git a/src/sketches/space_saving.rs b/src/sketches/space_saving.rs index 475e04cd..f40a9143 100644 --- a/src/sketches/space_saving.rs +++ b/src/sketches/space_saving.rs @@ -196,7 +196,7 @@ impl SpaceSaving { let lowest = self.buckets[self.bucket_head].count; debug_assert!( self.discarded_max <= lowest, - "the ceiling {} sits above the lowest live weight {lowest}", + "the ceiling {} sits above the lowest live count {lowest}", self.discarded_max ); self.discarded_max = lowest; diff --git a/src/sketches/uniform.rs b/src/sketches/uniform.rs index 1b9661dc..b3350dd9 100644 --- a/src/sketches/uniform.rs +++ b/src/sketches/uniform.rs @@ -5,6 +5,7 @@ use crate::DataInput; mod wire; const GOLDEN_GAMMA: u64 = 0x9E37_79B9_7F4A_7C15; +const DEFAULT_SEED: u64 = GOLDEN_GAMMA; const MIX_MULTIPLIER_1: u64 = 0xBF58_476D_1CE4_E5B9; const MIX_MULTIPLIER_2: u64 = 0x94D0_49BB_1331_11EB; @@ -30,7 +31,7 @@ pub struct UniformSampling { impl UniformSampling { pub fn new(sample_rate: f64) -> Self { - Self::with_seed(sample_rate, GOLDEN_GAMMA) + Self::with_seed(sample_rate, DEFAULT_SEED) } pub fn with_seed(sample_rate: f64, seed: u64) -> Self { diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index b8529533..352bfd1d 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -63,7 +63,7 @@ Compared against: an exact `HashMap` count, columns via the public `hash_for_mat - paths - `cells(S(xs)) == cells(S(perm(xs)))` - `update_weighted(k, n) == n x update(k)`, cell for cell - - `update_weighted_batch_with_hashes == insert_many`, cell for cell + - `update_weighted_batch_with_hashes == update_weighted`, cell for cell - `DefaultMatrixI32`, `QuickMatrixI64` == `Vector2D`, cell for cell, at their fixed geometries - estimator - `estimate(k)` never drops on a non-negative insert; all 32 keys probed after every insert diff --git a/tests/TEST_COVERAGE.md b/tests/TEST_COVERAGE.md index ed6be6ce..cc080157 100644 --- a/tests/TEST_COVERAGE.md +++ b/tests/TEST_COVERAGE.md @@ -129,7 +129,7 @@ Feature-gated behind `--features experimental`. ## e2e/frequency -- CountMinSketch sketch +- Count-Min sketch - both vector based and fixed matrix based - both FastPath and RegularPath - size, combination of row and column: @@ -529,7 +529,7 @@ Feature-gated behind `--features experimental`. ## e2e/heavy_hitters -- CountMinSketch sketch +- Count-Min sketch - both vector based and fixed matrix based - both FastPath and RegularPath - size, combination of row and column: @@ -968,7 +968,7 @@ Feature-gated behind `--features experimental`. ## e2e/nitro - NitroBatch - - target sketches: CountMinSketch (row 5, col 2048, FastPath) and Count sketch (row 5, col 2048, FastPath) + - target sketches: CountMinSketch (row 5, col 2048, FastPath) and CountSketch (row 5, col 2048, FastPath) - Configuration (sampling rate): 1.0, 0.5 - Input: one key repeated 100K times, so the rescaling of the estimator is what the bound measures - error bound: @@ -1099,7 +1099,7 @@ Feature-gated behind `--features experimental`. - every variant selects the documented merge norm, and can merge into its own kind - buckets past the window expire, the retained span is reported, and expiry follows the window length the histogram was last given - a custom bucket update matches repeated inserts -- MicroCM (feature-gated behind `--features experimental`) +- MicroCm (feature-gated behind `--features experimental`) - Configuration: row 4, col 1024, T=8, c=2, count-based clock of 1000 items per sub-window - Input: 40K zipf i64, s=1.1, key-size=4096 - reference window: `DeltaStrategy::Over` charges sub-windows `n-T ..= n` in full, and a count-based clock puts item `i` in sub-window `i / L`, so the exact window is items `[(n-T) * L, seen)` -- no tolerance is spent on granularity @@ -1111,7 +1111,7 @@ Feature-gated behind `--features experimental`. ## conformance_kit -- MicroCM (feature-gated behind `--features experimental`) +- MicroCm (feature-gated behind `--features experimental`) - Configuration: row 4, col 4096, T=64, c=2, count-based clock of 1000 items per sub-window - Input: the kit's shared 60K zipf i64 stream, s=1.1, key-size=2048 - batteries: `frequency_battery` with `one_sided: true`, and `merge_equivalence_battery` diff --git a/tests/conformance_kit.rs b/tests/conformance_kit.rs index 17816dad..99ede9c3 100644 --- a/tests/conformance_kit.rs +++ b/tests/conformance_kit.rs @@ -792,7 +792,7 @@ fn uniform_sampling_at_full_rate_passes_quantile_conformance() { .assert_ok(); } -// --------------------------------------------------------------- MicroCM +// --------------------------------------------------------------- MicroCm /// MicroscopeSketch's Count-Min grid, run at a configuration where it is /// exact. diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index b415592a..3079ee1c 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -1,5 +1,5 @@ //! E2E OctoSketch suite: delta-promotion conservation invariants for the -//! CountMinSketch, Count and HyperLogLog child/parent protocol, and the +//! CountMinSketch, CountSketch and HyperLogLog child/parent protocol, and the //! `octo-runtime` multi-threaded pipeline checked against single-threaded //! replays of the same round-robin partition. //! diff --git a/tests/e2e/windows.rs b/tests/e2e/windows.rs index 7dfc396c..d283211e 100644 --- a/tests/e2e/windows.rs +++ b/tests/e2e/windows.rs @@ -47,7 +47,7 @@ const COCO_SEED: u64 = 0xC0C0; /// Matrix dimensions for the counter-backed variants, chosen so their bounds /// are meaningful at `N` updates over `DOMAIN` keys. -/// UnivMon layer geometry used by the `UNIVMON` payload below. Its L2 band is +/// UnivMon layer geometry used by the `UnivMon` payload below. Its L2 band is /// derived from these, so a change here moves the band with it. const UNIVMON_ROWS: usize = 5; const UNIVMON_COLS: usize = 2_048; @@ -65,7 +65,7 @@ fn full_span(eh: &ExponentialHistogram) -> (u64, u64) { // ------------------------------------------------- Norm policy per variant -/// Which merge rule each payload selects. `COUNTL2HH` and `UNIVMON` carry an +/// Which merge rule each payload selects. `CountSketchWithL2Norm` and `UnivMon` carry an /// L2 mass and are merged by it; every other variant has no L2 mass to read /// and falls back to the L1 bucket-size rule. Getting this wrong would silently /// change how buckets consolidate, and therefore how much of the window a query @@ -701,7 +701,7 @@ fn tumbling_univmon_q_windows_carry_exact_aggregates_through_rotation() { /// `EhBucketSketch::merge`. A missing arm is invisible at runtime — `EhBucket::merge` /// discards the `Result` — so the histogram keeps counting bucket sizes while /// silently dropping the merged sketch's contents, and every query afterwards -/// reads a single bucket. `ELASTIC` shipped without its arm and lost the whole +/// reads a single bucket. `ElasticSketch` shipped without its arm and lost the whole /// window; this pins every variant so the next one cannot. #[test] fn every_eh_variant_can_merge_into_its_own_kind() { @@ -767,7 +767,7 @@ fn every_eh_variant_can_merge_into_its_own_kind() { } } -/// `UNIFORM` is the experimental reservoir payload: its window answers are +/// `Uniform` is the experimental reservoir payload: its window answers are /// retained-sample bookkeeping, not an estimate, so they are checked exactly. #[cfg(feature = "experimental")] #[test] @@ -1287,7 +1287,7 @@ mod documented_matrix { } } -// ---------------------------------------------------------------- MicroCM +// ---------------------------------------------------------------- MicroCm /// MicroscopeSketch's Count-Min grid over a sliding window. /// From de9542eaced2775767b9e9d734353518bd743eae Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:38:14 +0000 Subject: [PATCH 05/11] refactor!: delete the dead APIs #209 lists Kll::wire_k, PRECOMPUTED_SAMPLE_RATE_1PERCENT with common::precompute_sample2, common::hashspec, Vector2D::get_row/get_col, HeapEntry::init_item and CountSketchWithL2Norm::fast_update_and_est_without_l2 have no remaining callers (wire_k callers use k()). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 + docs/api/api_common.md | 2 - docs/api/api_common_input.md | 1 - docs/api/api_count_sketch.md | 1 - docs/library_map.md | 5 +- docs/tests.md | 17 +- src/common/hashspec.rs | 307 ------------------------ src/common/input.rs | 8 - src/common/mod.rs | 6 - src/common/precompute_sample.rs | 5 +- src/common/precompute_sample2.rs | 23 -- src/common/structures/vector2d.rs | 12 - src/message_pack_format/portable/kll.rs | 6 +- src/sketch_framework/hydra/wire.rs | 8 +- src/sketches/countsketch_l2_norm.rs | 8 - src/sketches/kll.rs | 6 - tests/asapv1_golden.rs | 4 +- tests/e2e/quantiles.rs | 6 +- tests/sketches_go_parity_probe.rs | 6 +- 19 files changed, 29 insertions(+), 411 deletions(-) delete mode 100644 src/common/hashspec.rs delete mode 100644 src/common/precompute_sample2.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af9833e..e1ec993a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,6 +303,15 @@ signals a backwards-compatible change. as an empty window like any other, and that window takes a place in `max_windows`. +### Removed + +- **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); + `PRECOMPUTED_SAMPLE_RATE_1PERCENT` and module `common::precompute_sample2`; + module `common::hashspec` (`HashSpec`, `derive_index`, `derive_sign`, + `CANONICAL_HASH_SEED_TABLE`); `Vector2D::get_row`/`get_col` (use + `rows()`/`cols()`); `HeapEntry::init_item` (use `from_input`); + `CountSketchWithL2Norm::fast_update_and_est_without_l2`. + ### Fixed - `KLLDynamic` panicked on every quantile query once a NaN had entered the diff --git a/docs/api/api_common.md b/docs/api/api_common.md index 387e51e7..03a2caed 100644 --- a/docs/api/api_common.md +++ b/docs/api/api_common.md @@ -21,7 +21,6 @@ src/common/ ├── structure_utils.rs # Nitro and median utility ├── precompute_hash.rs # PRECOMPUTED_HASH ├── precompute_sample.rs # LN_ONE_MINUS_U_TABLE -├── precompute_sample2.rs # PRECOMPUTED_SAMPLE_RATE_1PERCENT └── structures/ ├── vector1d.rs ├── vector2d.rs @@ -69,7 +68,6 @@ use asap_sketchlib::common::{ // Precomputed assets PRECOMPUTED_HASH, LN_ONE_MINUS_U_TABLE, - PRECOMPUTED_SAMPLE_RATE_1PERCENT, }; ``` diff --git a/docs/api/api_common_input.md b/docs/api/api_common_input.md index 241a4420..9023dc09 100644 --- a/docs/api/api_common_input.md +++ b/docs/api/api_common_input.md @@ -27,7 +27,6 @@ fn input_to_owned(input: &DataInput<'_>) -> OwnedDataInput // HeapEntry fn from_input(k: DataInput, count: C) -> Self fn new(key: OwnedDataInput, count: C) -> Self -fn init_item(k: DataInput, count: C) -> Self ``` ## Insert/Update diff --git a/docs/api/api_count_sketch.md b/docs/api/api_count_sketch.md index ebe70b90..55c52d12 100644 --- a/docs/api/api_count_sketch.md +++ b/docs/api/api_count_sketch.md @@ -47,7 +47,6 @@ fn estimate_with_hash(&self, hashed_val: &H::HashType) -> f64 fn estimate(&self, val: &DataInput) -> f64 fn estimate_with_hash(&self, hashed_val: u128) -> f64 fn update_weighted_and_estimate(&mut self, val: &DataInput, weight: i64) -> f64 -fn fast_update_and_est_without_l2(&mut self, val: &DataInput, c: i64) -> f64 fn l2(&self) -> f64 fn l2_squared(&self) -> f64 ``` diff --git a/docs/library_map.md b/docs/library_map.md index 9d78eed2..3c6a6dfa 100644 --- a/docs/library_map.md +++ b/docs/library_map.md @@ -7,7 +7,7 @@ - `structures/` - High-performance data structures (`Vector1D`, `Vector2D`, `Vector3D`, `BitMatrix`, `CommonHeap`, `MatrixStorage`, `FixedMatrix`) - `heap.rs` - `TopKHeap` convenience wrapper for heavy hitter tracking - `hash.rs` - Hashing utilities (`hash_for_matrix`, `hash64_seeded`, `SEED_LIST`, `UNIVMON_BOTTOM_LAYER_SEED_INDEX`) plus `SketchHasher` for custom hasher injection and `DigestHasher` / `DigestBuildHasher` for maps keyed by an already-hashed digest - - `hashspec.rs` - `HashSpec`/`derive_index`/`derive_sign` byte parity with `sketchlib-go`; `numerical.rs` - `NumericalValue`; `structure_utils.rs` - `Nitro`, `NitroContext`, `ToF64` + - `numerical.rs` - `NumericalValue`; `structure_utils.rs` - `Nitro`, `NitroContext`, `ToF64` - **`src/sketches/`** - Sketch implementations (status source: [apis.md](./apis.md)) - `Ready` in API index: `countminsketch.rs`, `countsketch.rs`, `hll.rs`, `kll.rs`, `ddsketch.rs`, `countminsketch_heap.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `space_saving.rs`, `bloom.rs`, `elastic.rs`, `coco.rs` @@ -40,8 +40,7 @@ - The large precomputed hash/sample tables are built lazily at runtime via `std::sync::LazyLock` in - `src/common/precompute_hash.rs`, `src/common/precompute_sample.rs`, and - `src/common/precompute_sample2.rs`. + `src/common/precompute_hash.rs` and `src/common/precompute_sample.rs`. ## Proto code generation diff --git a/docs/tests.md b/docs/tests.md index 138b6c66..16ac9edb 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1677,19 +1677,6 @@ Test file: [`src/common/structures/bit_matrix.rs`](../src/common/structures/bit_ | `a_packed_64_grid_decodes_a_distinct_window_per_row` | Each row of a packed-64 grid reads its own window of the hash. | For a `5x1024` `BitMatrix`, verifies `hash_layout_for_matrix(5, 1024)` is `MatrixHashLayout::Packed64`, then `fast_insert`s 300 `U64` keys and verifies every pair of rows differs at some column; the per-row bit windows themselves are not compared. | | `dimensions_whose_word_count_overflows_are_rejected` | Dimensions whose word count overflows fail at decode. | Verifies a crafted payload declaring `usize::MAX` rows by `128` columns with an empty `words` array is refused with an error containing "dimensions overflow". | -### Common Hash Spec - -Test file: [`src/common/hashspec.rs`](../src/common/hashspec.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `seed_table_matches_sketchlib_go` | The seed table and the canonical seed index are pinned values. | Verifies `CANONICAL_HASH_SEED_TABLE` equals the twenty listed seeds (`0xcafe3553`, `0xade3415118`, through `0xdb0c2e0d`) in order, and that `DEFAULT_CANONICAL_SEED_INDEX` is `5`. | -| `hash_with_spec_matches_sketchlib_go` | The byte-key hash of one fixed key is a pinned value. | Verifies `hash_with_spec(&HashSpec::default(), b"projectasap")` is `887548862923853302`. | -| `derive_index_matches_go_bit_slicing` | `derive_index` slices the row's own window out of one hash. | For the `HashSpec::default()` hash of `b"projectasap"`, verifies `derive_index(&spec, row, h, 512)` is `(h >> (row * 9)) & 0x1ff` and `derive_index(&spec, row, h, 1024)` is `(h >> (row * 10)) & 0x3ff`, for rows `0..3` of each width. | -| `derive_sign_matches_go_high_bit` | `derive_sign` reads bit `63 - row` of the hash. | For the same hash, verifies `derive_sign(&spec, row, h)` is `1` where bit `63 - row` is set and `-1` where it is clear, for rows `0..5`. | -| `mask_bits_for_width_matches_go` | The mask width is the column count's bit length. | Verifies `mask_bits_for_width` returns `1` for `1` and for `2`, `2` for `4`, `9` for `512`, `10` for `1024`, and `12` for `4096`. | -| `default_hashspec_has_packed_derivation` | The default spec names packed derivation, the canonical index, and the full seed table. | Verifies `HashSpec::default()` carries `seed_derivation == SeedDerivation::Packed`, `canonical_seed_index == DEFAULT_CANONICAL_SEED_INDEX`, and a `seed_list` as long as `CANONICAL_HASH_SEED_TABLE`. | - ### MatrixHash (Common Structure) Test file: [`src/common/structures/matrix_storage.rs`](../src/common/structures/matrix_storage.rs) @@ -1837,8 +1824,8 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `kll_minimum_compactor_capacity_axis_keeps_the_count_exact_and_reports_its_own_geometry` | Every supported `m` leaves `k` alone, reports its own geometry on the wire, and keeps its answers inside the observed range. | For `m in {2, 4, 8, 16, 64}` builds `Kll::with_seed(200, m, kll_axis_seed(m))` over 20_000 uniform values (seed `0x4B11_0001`) and asserts `k() == 200`, `m() == m` and `wire_k() == 200`. Despite the name the count is not pinned exactly: `count()` is only asserted between `0.95x` and `1.05x` of 20_000. The quantile check is likewise weak — each `quantile(q)` over the 7-point `RANK_QS` grid is only required to lie between the true min and max, with no rank error asserted. | -| `kll_normalizes_a_minimum_compactor_capacity_outside_its_supported_range` | An out-of-range `m` is normalized and `k` is raised to `m` when it falls below it. | Asserts `Kll::with_seed(200, 0, ..)` reports `m() == 2` with `wire_k()` still `200`, and `Kll::with_seed(4, 64, ..)` reports `m() == 64` with `wire_k()` raised to `64`. Then feeds 5_000 uniform values (seed `0x4B11_0102`) into the raised sketch and asserts `count()` lands between `0.85x` and `1.15x` of 5_000; the count is banded, not exact. | +| `kll_minimum_compactor_capacity_axis_keeps_the_count_exact_and_reports_its_own_geometry` | Every supported `m` leaves `k` alone, reports its own geometry on the wire, and keeps its answers inside the observed range. | For `m in {2, 4, 8, 16, 64}` builds `Kll::with_seed(200, m, kll_axis_seed(m))` over 20_000 uniform values (seed `0x4B11_0001`) and asserts `k() == 200`, `m() == m` and `k() == 200`. Despite the name the count is not pinned exactly: `count()` is only asserted between `0.95x` and `1.05x` of 20_000. The quantile check is likewise weak — each `quantile(q)` over the 7-point `RANK_QS` grid is only required to lie between the true min and max, with no rank error asserted. | +| `kll_normalizes_a_minimum_compactor_capacity_outside_its_supported_range` | An out-of-range `m` is normalized and `k` is raised to `m` when it falls below it. | Asserts `Kll::with_seed(200, 0, ..)` reports `m() == 2` with `k()` still `200`, and `Kll::with_seed(4, 64, ..)` reports `m() == 64` with `k()` raised to `64`. Then feeds 5_000 uniform values (seed `0x4B11_0102`) into the raised sketch and asserts `count()` lands between `0.85x` and `1.15x` of 5_000; the count is banded, not exact. | | `kll_minimum_compactor_capacity_at_or_above_the_default_satisfies_the_rank_characterization` | At or above the default minimum compactor capacity, KLL still meets `eps(k)`. | For `m in {8, 16, 64}` x the six `rank_streams(case, 30_000)` shapes (18 trials), builds `Kll::with_seed(200, m, kll_axis_seed(0x0200 + case*16 + i))`, reduces each sketch to its worst rank error over `RANK_QS` via `KllRankSpec::datasketches(200)`, and accepts with `assert_independent_binomial` at a per-trial failure probability of `0.01` against `eps(200) ~ 0.0165`. | | `kll_minimum_compactor_capacity_below_the_default_stays_within_a_widened_rank_band` | Below the default minimum compactor capacity the rank error stays inside a 3x-widened band, with no violations at all. | For `m in {2, 4}` x the six `rank_streams(case + 8, 30_000)` shapes (12 trials), seeds `kll_axis_seed(0x0300 + case*16 + i)`, `k=200`: requires each sketch's worst rank error over `RANK_QS` to be at most `KLL_WIDENED_EPSILON_FACTOR * eps(200) = 3.0 * 0.0165 ~ 0.0496` and closes with `Tally::assert_none`, tolerating zero violations. | | `kll_small_k_satisfies_the_rank_characterization_at_its_own_epsilon` | Small `k` values are held to their own, looser `eps(k)` rather than to a fixed number. | For `k in {8, 16, 32}` x the six `rank_streams(case + 16, 20_000)` shapes, seeds `kll_axis_seed(0x0400 + case*16 + i)`: asserts `count()` between `0.85x` and `1.15x` of 20_000, then scores each sketch's worst rank error over `RANK_QS` against `KllRankSpec::datasketches(k).epsilon()` (`~0.359`, `~0.187`, `~0.0972`) and accepts by binomial at `p = 0.01`. | diff --git a/src/common/hashspec.rs b/src/common/hashspec.rs deleted file mode 100644 index 13c44c19..00000000 --- a/src/common/hashspec.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Cross-language hash layer mirroring `sketchlib-go::common`'s -//! `HashSpec` / `DeriveIndex` / `DeriveSign` decomposition for -//! matrix-backed sketches (CountSketch, CountMinSketch). -//! -//! `sketchlib-go` and `asap_sketchlib` share three building blocks for -//! these sketches: -//! -//! 1. A 20-entry seed table (`CANONICAL_HASH_SEED_TABLE`, identical -//! bit-for-bit to Go's `seedList`). -//! 2. A canonical seed index (`CANONICAL_HASH_SEED` = 5). -//! 3. A single XXH3-64-with-seed call per (key, seed) pair, with the -//! resulting `u64` then bit-sliced into per-row column indices and -//! ±1 signs. -//! -//! This module re-exposes that pipeline as a small, byte-key API so -//! sketches that consume keys as raw bytes (e.g. the wire-format -//! `CountSketch::update`) can match Go's emitted matrix -//! cell-for-cell. CountMinSketch's pending byte-parity fix will reuse -//! the same primitives — the seed table, derive_index, and derive_sign -//! are agnostic to whether ±1 signing is applied. -//! -//! # Example -//! -//! ``` -//! use asap_sketchlib::common::hashspec::{HashSpec, hash_with_spec, derive_index, derive_sign}; -//! -//! let spec = HashSpec::default(); // matches Go's portableHashSpec() -//! let key = b"flow-7"; -//! let h = hash_with_spec(&spec, key); // single XXH3-64 with seed_list[0] -//! let col = derive_index(&spec, 0, h, 512); // row 0, width 512 -//! let sign = derive_sign(&spec, 0, h); // -1 or +1 -//! assert!(col < 512); -//! assert!(sign == -1 || sign == 1); -//! ``` - -use twox_hash::XxHash3_64; - -/// 20-entry seed table shared by all matrix-backed sketches. -/// -/// Bit-for-bit identical to `sketchlib-go::common.seedList` — the order -/// MUST match because `derive_index` / `derive_sign` produce byte -/// parity only when both libraries pull the same `u64` from index `i`. -pub const CANONICAL_HASH_SEED_TABLE: [u64; 20] = [ - 0xcafe3553, - 0xade3415118, - 0x8cc70208, - 0x2f024b2b, - 0x451a3df5, - 0x6a09e667, - 0xbb67ae85, - 0x3c6ef372, - 0xa54ff53a, - 0x510e527f, - 0x9b05688c, - 0x1f83d9ab, - 0x5be0cd19, - 0xcbbb9d5d, - 0x629a292a, - 0x9159015a, - 0x152fecd8, - 0x67332667, - 0x8eb44a87, - 0xdb0c2e0d, -]; - -/// Default canonical-seed index used by single-hash matrix-sketch -/// operations. Matches `sketchlib-go::common.CanonicalHashSeed` and -/// `asap_sketchlib::common::hash::CANONICAL_HASH_SEED`. -pub const CANONICAL_HASH_SEED: usize = 5; - -/// Hash configuration shared by matrix-backed sketches. -/// -/// Mirrors `sketchlib-go::common.HashSpec` semantics: a producer -/// records the seed table, the canonical seed index, and the seed -/// derivation strategy so consumers can validate hash compatibility -/// before merging. -/// -/// `asap_sketchlib`'s wire-format CountSketch constructs a -/// `HashSpec::default()` and feeds it to [`derive_index`] / -/// [`derive_sign`] on the hot path. The default matches Go's -/// `portableHashSpec()`: -/// -/// - `seed_list = CANONICAL_HASH_SEED_TABLE` -/// - `canonical_seed_index = CANONICAL_HASH_SEED` (= 5) -/// - `seed_derivation = SeedDerivation::Packed` — the matrix sketches -/// used today fit in a single packed `u64` (rows × bits_per_row ≤ 64), -/// so a single hash with `seed_list[0]` covers all rows. The -/// `Additive` variant is reserved for the per-row-hash fallback used -/// by larger matrices and is not yet exercised by the wire path. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct HashSpec { - /// Full seed table, length must be > 0. - pub seed_list: Vec, - /// Index into `seed_list` for the canonical (single-hash) seed. - pub canonical_seed_index: usize, - /// Strategy for deriving per-row seeds. - pub seed_derivation: SeedDerivation, -} - -/// Strategy for deriving per-row seeds, mirroring -/// `sketchlib-go::common.SeedDerivation`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum SeedDerivation { - /// All rows share `seed_list[0]`; per-row column index and sign are - /// extracted via bit slicing on a single `u64` hash. Used when - /// `rows × bits_per_row ≤ 64`. - Packed, - /// Each row r uses `seed_list[(base + r) % len]`; one - /// XXH3-64-with-seed call per row. Reserved for matrices that - /// exceed the packed budget. - Additive, -} - -impl Default for HashSpec { - fn default() -> Self { - Self { - seed_list: CANONICAL_HASH_SEED_TABLE.to_vec(), - canonical_seed_index: CANONICAL_HASH_SEED, - seed_derivation: SeedDerivation::Packed, - } - } -} - -impl HashSpec { - /// Returns the seed used for the single-hash packed path. Matches - /// Go's `seedList[0]` (because Go's `Hash64` calls `HashIt(0, key)`, - /// not `HashIt(canonical_seed_index, key)` — the canonical-seed - /// index governs `CanonicalHash`/`hh_keys` lookups, not the - /// matrix hot path). The matrix path always uses index 0, - /// regardless of `canonical_seed_index`. - #[inline] - pub fn matrix_seed(&self) -> u64 { - // Mirror Go's `common.Hash64`: `HashIt(0, key)` reads - // `seedList[normalizedSeedIdx(0)]` = seedList[0]. - self.seed_list[0] - } - - /// Returns the seed for row `r` under the additive-offset - /// derivation (reserved; not used by the packed wire path). - #[inline] - pub fn row_seed(&self, row: usize) -> u64 { - let idx = row % self.seed_list.len(); - self.seed_list[idx] - } -} - -/// Hashes `key` under `spec`'s matrix seed (single XXH3-64-with-seed -/// call, mirroring Go's `common.Hash64`). Producers of byte-parity -/// CountSketch / CMS update paths call this once per key, then invoke -/// [`derive_index`] / [`derive_sign`] per row to populate the matrix. -#[inline] -pub fn hash_with_spec(spec: &HashSpec, key: &[u8]) -> u64 { - XxHash3_64::oneshot_with_seed(spec.matrix_seed(), key) -} - -/// Returns `mask_bits` such that `(1 << mask_bits) - 1` is the column -/// mask for `width`. Mirrors Go's `maskBitsForWidth`. -#[inline] -fn mask_bits_for_width(width: u32) -> u32 { - if width <= 1 { - return 1; - } - // ceil(log2(width)). Wire-path widths are power-of-two, so this - // collapses to `width.trailing_zeros()`. - let mut u = width - 1; - let mut bits = 0u32; - while u > 0 { - bits += 1; - u >>= 1; - } - bits -} - -/// Derives the row-local column index from a precomputed `hash`, -/// mirroring `sketchlib-go::common.DeriveIndex`: -/// -/// ```text -/// shift = row * mask_bits(width) -/// index = (hash >> shift) & (width - 1) -/// ``` -/// -/// `width` MUST be power-of-two (the wire-format matrix sketches -/// enforce this on construction); other values produce a mask that -/// does not equal `width - 1` and the caller's matrix-cell layout -/// will diverge from Go's. -#[inline] -pub fn derive_index(_spec: &HashSpec, row: usize, hash: u64, width: u32) -> usize { - let shift = (row as u32) * mask_bits_for_width(width); - let mask = (width as u64).wrapping_sub(1); - ((hash >> shift) & mask) as usize -} - -/// Derives the ±1 sign for row `row` from a precomputed `hash`, -/// mirroring `sketchlib-go::common.DeriveSign`: -/// -/// ```text -/// bit = (hash >> (63 - row)) & 1 -/// sign = bit == 1 ? +1 : -1 -/// ``` -/// -/// `row` must satisfy `row <= 63`; callers with deeper sketches must -/// switch to the per-row hash fallback (not yet exposed here). -#[inline] -pub fn derive_sign(_spec: &HashSpec, row: usize, hash: u64) -> i64 { - debug_assert!( - row <= 63, - "derive_sign: row {row} out of range for u64 hash" - ); - let bit = (hash >> (63 - row as u32)) & 1; - if bit == 0 { -1 } else { 1 } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Locks the seed table to Go's `seedList`. Any drift here will - /// silently desync the matrix-cell layout from `sketchlib-go`'s. - #[test] - fn seed_table_matches_sketchlib_go() { - // Captured from `sketchlib-go::common.seedList` (hash.go:13-34). - let go_seed_list: [u64; 20] = [ - 0xcafe3553, - 0xade3415118, - 0x8cc70208, - 0x2f024b2b, - 0x451a3df5, - 0x6a09e667, - 0xbb67ae85, - 0x3c6ef372, - 0xa54ff53a, - 0x510e527f, - 0x9b05688c, - 0x1f83d9ab, - 0x5be0cd19, - 0xcbbb9d5d, - 0x629a292a, - 0x9159015a, - 0x152fecd8, - 0x67332667, - 0x8eb44a87, - 0xdb0c2e0d, - ]; - assert_eq!(CANONICAL_HASH_SEED_TABLE, go_seed_list); - assert_eq!(CANONICAL_HASH_SEED, 5); - } - - /// Locks `hash_with_spec` to Go's `Hash64` for a representative - /// key. Captured by running `common.Hash64([]byte("projectasap"))` - /// in sketchlib-go (see `common/hash_test.go::TestXxh3RegressionVectors`). - #[test] - fn hash_with_spec_matches_sketchlib_go() { - let spec = HashSpec::default(); - let got = hash_with_spec(&spec, b"projectasap"); - // From sketchlib-go::common::hash_test.go: Hash64(key) = 887548862923853302. - assert_eq!(got, 887548862923853302); - } - - /// derive_index slices the hash exactly the way Go's DeriveIndex - /// does. Sanity-check across rows and widths. - #[test] - fn derive_index_matches_go_bit_slicing() { - let spec = HashSpec::default(); - // Same hash Go would compute for "projectasap" with seedList[0]. - let h = hash_with_spec(&spec, b"projectasap"); - // 9-bit width (512 cols) → mask 0x1ff, shift = row*9. - for row in 0..3 { - let expected = ((h >> (row as u32 * 9)) & 0x1ff) as usize; - assert_eq!(derive_index(&spec, row, h, 512), expected); - } - // 10-bit width (1024 cols) → mask 0x3ff, shift = row*10. - for row in 0..3 { - let expected = ((h >> (row as u32 * 10)) & 0x3ff) as usize; - assert_eq!(derive_index(&spec, row, h, 1024), expected); - } - } - - /// derive_sign extracts the high-bit-minus-row exactly like Go. - #[test] - fn derive_sign_matches_go_high_bit() { - let spec = HashSpec::default(); - let h = hash_with_spec(&spec, b"projectasap"); - for row in 0..5 { - let bit = (h >> (63 - row as u32)) & 1; - let expected: i64 = if bit == 0 { -1 } else { 1 }; - assert_eq!(derive_sign(&spec, row, h), expected); - } - } - - #[test] - fn mask_bits_for_width_matches_go() { - assert_eq!(mask_bits_for_width(1), 1); - assert_eq!(mask_bits_for_width(2), 1); - assert_eq!(mask_bits_for_width(4), 2); - assert_eq!(mask_bits_for_width(512), 9); - assert_eq!(mask_bits_for_width(1024), 10); - assert_eq!(mask_bits_for_width(4096), 12); - } - - #[test] - fn default_hashspec_has_packed_derivation() { - let spec = HashSpec::default(); - assert_eq!(spec.seed_derivation, SeedDerivation::Packed); - assert_eq!(spec.canonical_seed_index, CANONICAL_HASH_SEED); - assert_eq!(spec.seed_list.len(), CANONICAL_HASH_SEED_TABLE.len()); - } -} diff --git a/src/common/input.rs b/src/common/input.rs index 542e2f20..fe788606 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -528,14 +528,6 @@ impl HeapEntry { HeapEntry { key, count } } - /// Legacy constructor for compatibility. - pub fn init_item(k: DataInput, count: C) -> Self { - HeapEntry { - key: input_to_owned(&k), - count, - } - } - /// Prints the item in a human-readable format. pub fn print_entry(&self) { println!("key: {:?} with count: {}", self.key, self.count); diff --git a/src/common/mod.rs b/src/common/mod.rs index db0099a7..68f7fae0 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -19,16 +19,11 @@ /// Hashing utilities and seed definitions shared across sketches. pub mod hash; -/// Cross-language hash spec (`HashSpec`/`derive_index`/`derive_sign`) -/// shared by matrix-backed sketches that need byte parity with -/// `sketchlib-go`. -pub mod hashspec; pub mod heap; pub mod input; pub mod numerical; pub mod precompute_hash; pub mod precompute_sample; -pub mod precompute_sample2; pub mod structure_utils; /// Reusable storage backends and low-level containers. pub mod structures; @@ -47,7 +42,6 @@ pub use input::{DataInput, HeapEntry, OwnedDataInput, heap_item_to_sketch_input, pub use numerical::NumericalValue; pub use precompute_hash::PRECOMPUTED_HASH; pub use precompute_sample::LN_ONE_MINUS_U_TABLE; -pub use precompute_sample2::PRECOMPUTED_SAMPLE_RATE_1PERCENT; pub use structure_utils::{Nitro, NitroContext, compute_median_inline_f64}; pub use structures::{ BitMatrix, CommonHeap, CommonHeapOrder, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, diff --git a/src/common/precompute_sample.rs b/src/common/precompute_sample.rs index 6f5e98ca..84275020 100644 --- a/src/common/precompute_sample.rs +++ b/src/common/precompute_sample.rs @@ -33,9 +33,8 @@ pub static LN_ONE_MINUS_U_TABLE: LazyLock> = /// `ln(1 - u_i) * scale`, where `u_i ∈ (0, 1)` is drawn from a `SmallRng` /// seeded with [`SEED`]. /// -/// Shared by [`LN_ONE_MINUS_U_TABLE`] and -/// [`super::precompute_sample2::PRECOMPUTED_SAMPLE_RATE_1PERCENT`]. -pub(super) fn build_ln_one_minus_u_table(scale: f64) -> Box<[f64]> { +/// Backs [`LN_ONE_MINUS_U_TABLE`]. +fn build_ln_one_minus_u_table(scale: f64) -> Box<[f64]> { let mut generator = SmallRng::seed_from_u64(SEED); (0..LN_ONE_MINUS_U_TABLE_LEN) .map(|_| { diff --git a/src/common/precompute_sample2.rs b/src/common/precompute_sample2.rs deleted file mode 100644 index 472df258..00000000 --- a/src/common/precompute_sample2.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Lazily-built table of geometric-skip distances for a 1% sampling rate. -//! -//! Storing this as a [`LazyLock`] instead of a 64K-entry literal array keeps -//! the crate small enough to publish to crates.io while preserving the same -//! `PRECOMPUTED_SAMPLE_RATE_1PERCENT[i]` access pattern. The table is -//! generated once per process from a [`rand::rngs::SmallRng`] with the same -//! fixed seed as [`super::precompute_sample::LN_ONE_MINUS_U_TABLE`], so values -//! are stable across runs and across versions of this crate. - -use std::sync::LazyLock; - -use super::precompute_sample::build_ln_one_minus_u_table; - -/// Precomputed geometric-skip distances for sampling-rate `p = 0.01`, i.e. -/// `ln(1 - u) / ln(1 - p) = ln(1 - u) / ln(0.99)` for `u ∈ (0, 1)`. -/// -/// The table is materialised lazily on first access through [`LazyLock`]. -/// Indexing (`PRECOMPUTED_SAMPLE_RATE_1PERCENT[i]`), -/// iteration (`PRECOMPUTED_SAMPLE_RATE_1PERCENT.iter()`), and length -/// (`PRECOMPUTED_SAMPLE_RATE_1PERCENT.len()`) all work via `Deref` to the -/// underlying slice. -pub static PRECOMPUTED_SAMPLE_RATE_1PERCENT: LazyLock> = - LazyLock::new(|| build_ln_one_minus_u_table(1.0 / 0.99_f64.ln())); diff --git a/src/common/structures/vector2d.rs b/src/common/structures/vector2d.rs index c9121cd9..b81e5967 100644 --- a/src/common/structures/vector2d.rs +++ b/src/common/structures/vector2d.rs @@ -675,18 +675,6 @@ impl Vector2D { let end = start + self.cols; &mut self.data[start..end] } - - /// Returns the number of rows (legacy helper). - #[inline(always)] - pub fn get_row(&self) -> usize { - self.rows - } - - /// Returns the number of columns (legacy helper). - #[inline(always)] - pub fn get_col(&self) -> usize { - self.cols - } } impl MatrixStorage for Vector2D diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index 14ac5ae1..c61fb54f 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -513,7 +513,7 @@ mod tests { let (state, bit_cache, remaining_bits) = sk.coin_state(); let kll_state = KllState { - k: sk.wire_k(), + k: sk.k() as u32, m: sk.m(), num_levels: sk.num_levels(), levels: sk.level_bounds_top_down(), @@ -630,7 +630,7 @@ mod tests { // Raw-f64 form. let raw_state = KllState { - k: sk.wire_k(), + k: sk.k() as u32, m: sk.m(), num_levels, levels: levels.clone(), @@ -646,7 +646,7 @@ mod tests { let (offset, scale, residuals) = encode_value_offset(&raw_items).expect("integer items representable"); let fp_state = KllState { - k: sk.wire_k(), + k: sk.k() as u32, m: sk.m(), num_levels, levels, diff --git a/src/sketch_framework/hydra/wire.rs b/src/sketch_framework/hydra/wire.rs index 24621ba9..28b74607 100644 --- a/src/sketch_framework/hydra/wire.rs +++ b/src/sketch_framework/hydra/wire.rs @@ -702,19 +702,19 @@ impl Hydra { self.check_fresh_prototype( prototype.num_levels() == 1 && prototype.items_top_down().is_empty(), )?; - let (k, m) = (prototype.wire_k(), prototype.m()); + let (k, m) = (prototype.k() as u32, prototype.m()); let counters = self.cells(|counter| match counter { HydraCounter::Kll(kll) => Some(kll), _ => None, })?; let mut cells = Vec::with_capacity(counters.len()); for (i, kll) in counters.iter().enumerate() { - if (kll.wire_k(), kll.m()) != (k, m) { + if (kll.k() as u32, kll.m()) != (k, m) { return Err(encode_error(format!( "ASAPv1 Hydra envelope: cell ({}, {}) has k={}, m={} against the prototype's k={k}, m={m}", i / self.cols, i % self.cols, - kll.wire_k(), + kll.k(), kll.m() ))); } @@ -1838,7 +1838,7 @@ mod tests { }, }; assert_eq!( - kll_cell_envelope(kll.wire_k(), kll.m(), &cell).expect("KLL cell"), + kll_cell_envelope(kll.k() as u32, kll.m(), &cell).expect("KLL cell"), kll.to_asapv1().expect("KLL envelope") ); diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs index 6950050b..74f3ce0c 100644 --- a/src/sketches/countsketch_l2_norm.rs +++ b/src/sketches/countsketch_l2_norm.rs @@ -233,14 +233,6 @@ impl CountSketchWithL2Norm { self.estimate_with_hash(hashed_val) } - /// Update and estimate without L2 with hash optimization. - /// due to the limitation of seeds, use fast_insert only - pub fn fast_update_and_est_without_l2(&mut self, val: &DataInput, c: i64) -> f64 { - let hashed_val = H::hash128_seeded(self.seed_index, val); - self.update_weighted_with_hash_without_l2(hashed_val, c); - self.estimate_with_hash(hashed_val) - } - /// Returns the median estimate of the squared L2 norm across rows. pub fn l2_squared(&self) -> f64 { let mut values: Vec = self.row_l2_squared.as_slice()[..self.rows] diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 8a2abf58..cfb6801a 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -767,12 +767,6 @@ impl Kll { // -- Wire-format-aligned accessors -------------------------------------- - /// Returns the configured `k` parameter. - #[inline] - pub fn wire_k(&self) -> u32 { - self.k as u32 - } - /// Returns the configured `m` parameter. #[inline] pub fn m(&self) -> u32 { diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index a1d32764..3c46c343 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -1586,8 +1586,8 @@ fn assert_same_grid(got: &Hydra, want: &Hydra) { for (i, (g, w)) in cells.enumerate() { let same = match (g, w) { (HydraCounter::Kll(g), HydraCounter::Kll(w)) => { - (g.wire_k(), g.m(), g.level_bounds_top_down(), g.coin_state()) - == (w.wire_k(), w.m(), w.level_bounds_top_down(), w.coin_state()) + (g.k(), g.m(), g.level_bounds_top_down(), g.coin_state()) + == (w.k(), w.m(), w.level_bounds_top_down(), w.coin_state()) && g.items_top_down() == w.items_top_down() } (HydraCounter::CountMinSketch(g), HydraCounter::CountMinSketch(w)) => { diff --git a/tests/e2e/quantiles.rs b/tests/e2e/quantiles.rs index 88c08e0c..9355b65b 100644 --- a/tests/e2e/quantiles.rs +++ b/tests/e2e/quantiles.rs @@ -1833,7 +1833,7 @@ fn kll_minimum_compactor_capacity_axis_keeps_the_count_exact_and_reports_its_own let ctx = format!("m={m} k=200 n={} sketch_seed={seed:#x}", values.len()); assert_eq!(sketch.k(), 200, "{ctx}: k moved"); assert_eq!(sketch.m() as usize, m, "{ctx}: reported m"); - assert_eq!(sketch.wire_k() as usize, 200, "{ctx}: reported k"); + assert_eq!(sketch.k(), 200, "{ctx}: reported k"); assert_between( sketch.count() as f64, values.len() as f64 * 0.95, @@ -1855,11 +1855,11 @@ fn kll_minimum_compactor_capacity_axis_keeps_the_count_exact_and_reports_its_own fn kll_normalizes_a_minimum_compactor_capacity_outside_its_supported_range() { let clamped: Kll = Kll::with_seed(200, 0, kll_axis_seed(0x0100)); assert_eq!(clamped.m(), 2, "m=0 must normalize to 2"); - assert_eq!(clamped.wire_k(), 200, "m=0 must leave k alone"); + assert_eq!(clamped.k(), 200, "m=0 must leave k alone"); let raised: Kll = Kll::with_seed(4, 64, kll_axis_seed(0x0101)); assert_eq!(raised.m(), 64, "m=64 must survive"); - assert_eq!(raised.wire_k(), 64, "k below m must be raised to m"); + assert_eq!(raised.k(), 64, "k below m must be raised to m"); let mut usable: Kll = Kll::with_seed(4, 64, kll_axis_seed(0x0102)); let values = kll_axis_values(5_000, 0x4B11_0102); diff --git a/tests/sketches_go_parity_probe.rs b/tests/sketches_go_parity_probe.rs index bf468563..67ee0a1f 100644 --- a/tests/sketches_go_parity_probe.rs +++ b/tests/sketches_go_parity_probe.rs @@ -1,12 +1,10 @@ -//! Probe: do `sketches::*` defaults already produce Go-byte-parity output, -//! making a separate hashspec-direct path inside the wire-format-aligned -//! types unnecessary? +//! Probe: `sketches::*` defaults produce Go-byte-parity output. //! //! Approach: reuse the exact same Go-golden envelopes that the //! `message_pack_format::portable` parity tests use, but build the //! matrix via `sketches::CountSketch` / `sketches::CountMinSketch` with `FastPath` //! and `DefaultXxHasher`. Identical bytes confirm the shared FastPath -//! math is sufficient — no hashspec bypass needed. +//! math is sufficient. use asap_sketchlib::common::DataInput; use asap_sketchlib::common::hash::DEFAULT_CANONICAL_SEED_INDEX; From ac7a2eb8239c4f8d62642eac94df6e2b850bdca3 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:39:51 +0000 Subject: [PATCH 06/11] refactor!: merge the duplicates #209 lists UnivMonQ::update is generic over NumericalValue and replaces add; UnivMon::estimate_g_sum is the recurrence itself, without the calc_g_sum_heuristic alias. QUICKSTART_ROWS/COLS and DEFAULT_ROWS/COLS come from fixed_structure, the heap-size default of 32 from common::heap, and KllDynamic reads Kll's MAX_K. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++++ docs/api/api_univmon_q.md | 7 +++---- docs/tests.md | 2 +- examples/audit_univmon_q_order.rs | 2 +- examples/compare_univmon_q.rs | 16 ++++++++-------- examples/compare_univmon_q_bundle.rs | 6 +++--- examples/compare_univmon_rank_modes.rs | 4 ++-- examples/evaluate_univmon_q_entropy.rs | 2 +- examples/evaluate_univmon_q_skew.rs | 2 +- examples/quantile_univmon_q.rs | 2 +- src/common/heap.rs | 3 +++ src/sketch_framework/eh_univmon.rs | 2 +- src/sketch_framework/univmon.rs | 16 ++++------------ src/sketch_framework/univmon_q.rs | 16 +++++----------- src/sketch_framework/univmon_two_tier.rs | 6 +++--- src/sketches/countminsketch.rs | 8 ++------ src/sketches/countminsketch_heap.rs | 2 +- src/sketches/countsketch.rs | 3 +-- src/sketches/countsketch_heap.rs | 2 +- src/sketches/countsketch_l2_norm.rs | 3 +-- src/sketches/kll.rs | 2 +- src/sketches/kll_dynamic.rs | 3 +-- tests/e2e/frameworks.rs | 5 ----- 23 files changed, 52 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ec993a..c7452762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,6 +303,13 @@ signals a backwards-compatible change. as an empty window like any other, and that window takes a place in `max_windows`. +- **Duplicates are merged** (breaking). `UnivMonQ::add` and `update` are one + generic `update(&T)`; `UnivMon::calc_g_sum_heuristic` is + gone and `estimate_g_sum` is the one g-sum. `QUICKSTART_ROWS` / + `QUICKSTART_COLS` and the default 3 x 4096 geometry are defined once in + `common::structures::fixed_structure`, and the default heap size (32) and + KLL's largest `k` once each. + ### Removed - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); diff --git a/docs/api/api_univmon_q.md b/docs/api/api_univmon_q.md index a6bc3416..c01725aa 100644 --- a/docs/api/api_univmon_q.md +++ b/docs/api/api_univmon_q.md @@ -54,8 +54,7 @@ all merged shards. ## Update ```rust -fn update(&mut self, value: &f64) -fn add(&mut self, value: &T) +fn update(&mut self, value: &T) fn update_data_input(&mut self, value: &DataInput) -> Result<(), &'static str> fn clear(&mut self) ``` @@ -270,10 +269,10 @@ let mut left = UnivMonQ::with_source_id(config, 10)?; let mut right = UnivMonQ::with_source_id(config, 20)?; for value in 0..50_000 { - left.add(&value); + left.update(&value); } for value in 50_000..100_000 { - right.add(&value); + right.update(&value); } left.merge(&right)?; diff --git a/docs/tests.md b/docs/tests.md index 16ac9edb..23c2aee7 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1992,7 +1992,7 @@ Test file: [`tests/e2e/frameworks.rs`](../tests/e2e/frameworks.rs) | --- | --- | --- | | `univmon_weighted_metrics_and_fast_insert_parity` | UnivMon's weighted L1 is exact and its L2, entropy and cardinality hold their bands, with `fast_insert` tracking the standard path. | Drives two `UnivMon::new(32, 5, 2048, 8)` sketches over `zipf_u64(20_000, 1000, 1.2, 4003)` with weight `1 + i % 7`, one through `update` and one through `fast_insert`, then asserts both `estimate_l1()` equal the exact weighted total with `assert_eq!`; `estimate_l2()` inside `univmon_l2_band(exact, 5, 2048)` (`b = sqrt(2*3/2048)`, about `0.054`); `estimate_entropy()` in `[0.88, 1.12]` times the exact bits; `estimate_cardinality()` in `[0.94, 1.06]` times the exact distinct count; and the fast path's `estimate_l2()` in `[0.85, 1.15]` times exact. | | `univmon_pyramid_weighted_metrics` | UnivMonTwoTier's weighted L1 is exact and its L2 and cardinality hold their bands. | Drives `UnivMonTwoTier::with_defaults()` over `zipf_u64(15_000, 800, 1.3, 4004)` with weight `1 + i % 3` and asserts `estimate_l1()` equals the exact weighted total with `assert_eq!`, `estimate_l2()` is in `[0.85, 1.15]` times exact, and `estimate_cardinality()` in `[0.80, 1.20]` times the exact distinct count; entropy is not checked. | -| `univmon_generic_g_sum_reproduces_every_named_estimator` | The generic g-sum reproduces the L2, cardinality and entropy estimators and its own heuristic delegate. | Over `UnivMon::new(32, 5, 2_048, 8)` fed `zipf_u64(20_000, 1_000, 1.2, 4_201)` with weight `1 + i % 7`, `assert_eq!`s `estimate_g_sum(\|x\| x*x, false).sqrt()` against `estimate_l2()`, `estimate_g_sum(\|_\| 1.0, true)` against `estimate_cardinality()`, `estimate_g_sum(x^2, false)` against `calc_g_sum_heuristic(x^2, false)`, and `estimate_g_sum(\|_\| 0.0, false)` against `0.0`; requires `estimate_entropy()` to equal `log2(L1) - gsum(x*log2 x)/L1` within `1e-9`; and checks `estimate_g_sum(\|x\| x, false)` only against `[0.50, 1.50]` times the exact L1, so the identity g-sum's agreement with L1 is not pinned tightly. | +| `univmon_generic_g_sum_reproduces_every_named_estimator` | The generic g-sum reproduces the L2, cardinality and entropy estimators. | Over `UnivMon::new(32, 5, 2_048, 8)` fed `zipf_u64(20_000, 1_000, 1.2, 4_201)` with weight `1 + i % 7`, `assert_eq!`s `estimate_g_sum(\|x\| x*x, false).sqrt()` against `estimate_l2()`, `estimate_g_sum(\|_\| 1.0, true)` against `estimate_cardinality()`, and `estimate_g_sum(\|_\| 0.0, false)` against `0.0`; requires `estimate_entropy()` to equal `log2(L1) - gsum(x*log2 x)/L1` within `1e-9`; and checks `estimate_g_sum(\|x\| x, false)` only against `[0.50, 1.50]` times the exact L1, so the identity g-sum's agreement with L1 is not pinned tightly. | | `univmon_g_sum_is_zero_on_an_empty_sketch` | An empty UnivMon reports zero for every estimator. | On a fresh `UnivMon::new(32, 5, 256, 4)`, `assert_eq!`s `estimate_l1()`, `estimate_entropy()`, `estimate_g_sum(\|x\| x*x, false)` and `estimate_cardinality()` each against `0.0`. | | `a_univmon_pool_recycles_a_returned_sketch_and_hands_back_a_cleared_one` | A returned sketch becomes available again and comes back cleared without a new allocation. | On `UnivMonPool::new(2, 16, 3, 256, 4)`: asserts `available() == 2` and `total_allocated() == 2` when fresh, `available() == 0` after two `take()`s with `total_allocated()` still `2`, and `total_allocated() == 3` after a third `take()` past capacity; then fills the first sketch with `zipf_u64(5_000, 512, 1.1, 4_202)` inserts (checking `estimate_l1() > 0.0`), `put()`s it back and asserts `available() == 1`, and requires the next `take()` to report `estimate_l1() == 0.0` with `total_allocated()` still `3`. | diff --git a/examples/audit_univmon_q_order.rs b/examples/audit_univmon_q_order.rs index 4a77099d..981058e8 100644 --- a/examples/audit_univmon_q_order.rs +++ b/examples/audit_univmon_q_order.rs @@ -211,7 +211,7 @@ fn build_q_shards( .wrapping_add(shard as u64 + 1); let mut sketch = UnivMonQ::with_source_id(config, source_id).unwrap(); for value in &data[start..end] { - sketch.add(value); + sketch.update(value); } shards.push(sketch); } diff --git a/examples/compare_univmon_q.rs b/examples/compare_univmon_q.rs index 61cbf85e..c198ef8b 100644 --- a/examples/compare_univmon_q.rs +++ b/examples/compare_univmon_q.rs @@ -207,7 +207,7 @@ fn main() { let mut sketch = UnivMonQ::new(q_base).unwrap(); let start = Instant::now(); for &value in &values { - sketch.add(&black_box(value)); + sketch.update(&black_box(value)); } q_base_times.push(start.elapsed()); black_box(sketch.count()); @@ -215,7 +215,7 @@ fn main() { let mut sketch = UnivMonQ::new(q_ordered).unwrap(); let start = Instant::now(); for &value in &values { - sketch.add(&black_box(value)); + sketch.update(&black_box(value)); } q_ordered_times.push(start.elapsed()); black_box(sketch.count()); @@ -223,7 +223,7 @@ fn main() { let mut sketch = UnivMonQ::new(q_memory_matched).unwrap(); let start = Instant::now(); for &value in &values { - sketch.add(&black_box(value)); + sketch.update(&black_box(value)); } q_memory_matched_times.push(start.elapsed()); black_box(sketch.count()); @@ -245,9 +245,9 @@ fn main() { let input = DataInput::U64(value); regular.update_weighted(&input, 1); fast.update_weighted_terminal(&input, 1); - q_base_sketch.add(&value); - q_ordered_sketch.add(&value); - q_memory_matched_sketch.add(&value); + q_base_sketch.update(&value); + q_ordered_sketch.update(&value); + q_memory_matched_sketch.update(&value); } let regular_wire = regular.to_asapv1().unwrap().len(); @@ -668,10 +668,10 @@ fn main() { let mut left = UnivMonQ::new(q_ordered).unwrap(); let mut right = UnivMonQ::new(q_ordered).unwrap(); for &value in &values[..midpoint] { - left.add(&value); + left.update(&value); } for &value in &values[midpoint..] { - right.add(&value); + right.update(&value); } let mut q_merge_times = Vec::with_capacity(MERGE_REPEATS); for _ in 0..MERGE_REPEATS { diff --git a/examples/compare_univmon_q_bundle.rs b/examples/compare_univmon_q_bundle.rs index ab10b8f0..4f2b376b 100644 --- a/examples/compare_univmon_q_bundle.rs +++ b/examples/compare_univmon_q_bundle.rs @@ -196,7 +196,7 @@ fn timed_q(data: &[i64], config: UnivMonQConfig) -> (UnivMonQ, Duration) { let mut sketch = UnivMonQ::new(config).unwrap(); let start = Instant::now(); for value in data { - sketch.add(value); + sketch.update(value); } times.push(start.elapsed()); retained = Some(sketch); @@ -224,10 +224,10 @@ fn merged_q(data: &[i64], config: UnivMonQConfig) -> (UnivMonQ, Duration) { let mut left = UnivMonQ::new(config).unwrap(); let mut right = UnivMonQ::new(config).unwrap(); for value in &data[..midpoint] { - left.add(value); + left.update(value); } for value in &data[midpoint..] { - right.add(value); + right.update(value); } let mut times = Vec::with_capacity(MERGE_REPEATS); for _ in 0..MERGE_REPEATS { diff --git a/examples/compare_univmon_rank_modes.rs b/examples/compare_univmon_rank_modes.rs index 714f031e..3e5354af 100644 --- a/examples/compare_univmon_rank_modes.rs +++ b/examples/compare_univmon_rank_modes.rs @@ -350,7 +350,7 @@ fn build_pure(data: &[i64], config: UnivMonQConfig, fanout: usize) -> UnivMonQ { let end = (shard + 1) * data.len() / fanout; let mut sketch = UnivMonQ::new(config).unwrap(); for value in &data[start..end] { - sketch.add(value); + sketch.update(value); } shards.push(sketch); } @@ -372,7 +372,7 @@ fn build_occurrence( let mut core = UnivMonQ::new(config).unwrap(); let mut sample = OccurrenceBottomK::new(capacity, seed); for (offset, &value) in data[start..end].iter().enumerate() { - core.add(&value); + core.update(&value); sample.update(value, (start + offset) as u64); } shards.push((core, sample)); diff --git a/examples/evaluate_univmon_q_entropy.rs b/examples/evaluate_univmon_q_entropy.rs index 0f659174..ca9a2bbd 100644 --- a/examples/evaluate_univmon_q_entropy.rs +++ b/examples/evaluate_univmon_q_entropy.rs @@ -307,7 +307,7 @@ fn build_shards(data: &[usize], config: UnivMonQConfig, seed: u64) -> Vec Vec Result<(), Box> { let mut sketch = UnivMonQ::new(config)?; for value in 1..=100_000 { - sketch.add(&value); + sketch.update(&value); } let query = sketch.prepare_queries(); diff --git a/src/common/heap.rs b/src/common/heap.rs index 5306c3e7..9e86b46b 100644 --- a/src/common/heap.rs +++ b/src/common/heap.rs @@ -15,6 +15,9 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::fmt; +/// Default heap capacity of the top-k sketches and UnivMon layers. +pub(crate) const DEFAULT_HEAP_SIZE: usize = 32; + /// The count an [`HeapEntry`] carries: `i64`, or `f64` ranked by /// [`f64::total_cmp`]. pub trait HeapCount: Copy + fmt::Display + Serialize + DeserializeOwned { diff --git a/src/sketch_framework/eh_univmon.rs b/src/sketch_framework/eh_univmon.rs index d690d551..172f9bd4 100644 --- a/src/sketch_framework/eh_univmon.rs +++ b/src/sketch_framework/eh_univmon.rs @@ -9,12 +9,12 @@ use std::collections::HashMap; +use crate::common::heap::DEFAULT_HEAP_SIZE; use crate::common::input::{heap_item_to_sketch_input, input_to_owned}; use crate::sketch_framework::univmon_two_tier::UnivMonPool; use crate::{DataInput, OwnedDataInput, UnivMon}; const MASS_EPSILON: f64 = 1e-9; -const DEFAULT_HEAP_SIZE: usize = 32; const DEFAULT_ROWS: usize = 5; const DEFAULT_COLS: usize = 2048; const DEFAULT_LAYERS: usize = 8; diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index e6e8edec..9253451f 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -23,7 +23,8 @@ //! //! This implementation is part of the `asap_sketchlib` library. -use crate::common::heap::TopKHeap; +use crate::common::Vector1D; +use crate::common::heap::{DEFAULT_HEAP_SIZE, TopKHeap}; use crate::common::{ DataInput, OwnedDataInput, SEED_LIST, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, hash64_seeded_owned, heap_item_to_sketch_input, @@ -38,7 +39,6 @@ pub(crate) mod wire; const DEFAULT_ROWS: usize = 5; const DEFAULT_COLS: usize = 2048; -const DEFAULT_HEAP_SIZE: usize = 32; const DEFAULT_LAYERS: usize = 8; #[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -308,8 +308,8 @@ impl UnivMon { } } - /// Computes a g-sum estimate using the heuristic recurrence. - pub fn calc_g_sum_heuristic(&self, g: F, heavy_only: bool) -> f64 + /// Computes a g-sum estimate with the UnivMon recurrence. + pub fn estimate_g_sum(&self, g: F, heavy_only: bool) -> f64 where F: Fn(f64) -> f64, { @@ -450,14 +450,6 @@ impl UnivMon { } } - /// Computes a g-sum estimate. - pub fn estimate_g_sum(&self, g: F, heavy_only: bool) -> f64 - where - F: Fn(f64) -> f64, - { - self.calc_g_sum_heuristic(g, heavy_only) - } - /// Returns the exact L1 norm for the supported non-negative update stream. pub fn estimate_l1(&self) -> f64 { self.total_weight as f64 diff --git a/src/sketch_framework/univmon_q.rs b/src/sketch_framework/univmon_q.rs index 04006b72..cd5920fd 100644 --- a/src/sketch_framework/univmon_q.rs +++ b/src/sketch_framework/univmon_q.rs @@ -840,15 +840,9 @@ impl UnivMonQ { self.source_id } - /// Adds one numeric value. + /// Adds one numeric value, projected to `f64`. #[inline(always)] - pub fn update(&mut self, value: &f64) { - self.update_value(*value); - } - - /// Adds any Sketchlib numeric primitive after projection to `f64`. - #[inline(always)] - pub fn add(&mut self, value: &T) { + pub fn update(&mut self, value: &T) { self.update_value(value.to_f64()); } @@ -1892,13 +1886,13 @@ mod tests { .find(|value| sketch.sample_level(float_to_ordered(*value as f64)) == terminal) .unwrap(); for _ in 0..200 { - sketch.add(&target); + sketch.update(&target); } let mut distinct_tail = 0_usize; for value in 1_u64.. { if value != target && sketch.sample_level(float_to_ordered(value as f64)) == terminal { - sketch.add(&value); + sketch.update(&value); distinct_tail += 1; if distinct_tail == 50_000 { break; @@ -2248,7 +2242,7 @@ mod tests { fn shard(source_id: u64, offset: u64) -> UnivMonQ { let mut sketch = UnivMonQ::with_source_id(tiny_config(), source_id).unwrap(); for index in 0..1_000_u64 { - sketch.add(&((index.wrapping_mul(17) + offset) % 991)); + sketch.update(&((index.wrapping_mul(17) + offset) % 991)); } sketch } diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index 02a52456..5baba16e 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -6,7 +6,8 @@ //! "mouse" layers for the long tail, matching the PromSketch design. use crate::UnivMon; -use crate::common::heap::TopKHeap; +use crate::common::Vector1D; +use crate::common::heap::{DEFAULT_HEAP_SIZE, TopKHeap}; use crate::common::{ DataInput, OwnedDataInput, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, hash64_seeded_owned, heap_item_to_sketch_input, @@ -94,7 +95,6 @@ const DEFAULT_ELEPHANT_ROWS: usize = 3; const DEFAULT_ELEPHANT_COLS: usize = 2048; const DEFAULT_MOUSE_ROWS: usize = 3; const DEFAULT_MOUSE_COLS: usize = 512; -const DEFAULT_PYRAMID_HEAP: usize = 32; const DEFAULT_LAYERS: usize = 16; /// Pyramid-structured UnivMon with two-tier sketch dimensions. @@ -215,7 +215,7 @@ impl UnivMonTwoTier { /// Creates a pyramid using built-in default dimensions. pub fn with_defaults() -> Self { Self::new( - DEFAULT_PYRAMID_HEAP, + DEFAULT_HEAP_SIZE, DEFAULT_ELEPHANT_LAYERS, DEFAULT_ELEPHANT_ROWS, DEFAULT_ELEPHANT_COLS, diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 33a5dc3b..bb2bb1fb 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -29,12 +29,8 @@ mod wire; pub use delta::CountMinSketchDelta; pub(crate) use wire::{CmsWireCounter, CmsWireMode}; -const DEFAULT_ROWS: usize = 3; -const DEFAULT_COLS: usize = 4096; -/// Recommended row count for quick-start examples. -pub const QUICKSTART_ROWS: usize = 5; -/// Recommended column count for quick-start examples. -pub const QUICKSTART_COLS: usize = 2048; +use crate::common::structures::fixed_structure::{DEFAULT_COLS, DEFAULT_ROWS}; +pub use crate::common::structures::fixed_structure::{QUICKSTART_COLS, QUICKSTART_ROWS}; const LOWER_32_MASK: u64 = (1u64 << 32) - 1; /// A Count-Min Sketch for estimating item frequencies in a data stream. diff --git a/src/sketches/countminsketch_heap.rs b/src/sketches/countminsketch_heap.rs index 3ecdb884..fa042d82 100644 --- a/src/sketches/countminsketch_heap.rs +++ b/src/sketches/countminsketch_heap.rs @@ -19,7 +19,7 @@ pub use delta::CountMinSketchWithHeapDelta; /// The heap count type of a `CountMinSketchWithHeap` over storage `S`. type HeapCountOf = <::Counter as HeapCounter>::HeapCount; -const DEFAULT_HEAP_SIZE: usize = 32; +use crate::common::heap::DEFAULT_HEAP_SIZE; /// A Count-Min Sketch paired with a top-k heavy-hitter heap. /// diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 9c5cf48d..6b944d29 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -23,8 +23,7 @@ mod wire; pub use delta::CountSketchDelta; pub(crate) use wire::{CsWireCounter, CsWireMode}; -const DEFAULT_ROWS: usize = 3; -const DEFAULT_COLS: usize = 4096; +use crate::common::structures::fixed_structure::{DEFAULT_COLS, DEFAULT_ROWS}; const LOWER_32_MASK: u64 = (1u64 << 32) - 1; /// A frequency-estimation sketch using random sign projections (Count Sketch). diff --git a/src/sketches/countsketch_heap.rs b/src/sketches/countsketch_heap.rs index 07e6e7ff..7b29658e 100644 --- a/src/sketches/countsketch_heap.rs +++ b/src/sketches/countsketch_heap.rs @@ -80,7 +80,7 @@ impl CountSketchHeapCounter for f64 { } } -const DEFAULT_HEAP_SIZE: usize = 32; +use crate::common::heap::DEFAULT_HEAP_SIZE; /// A Count Sketch paired with a top-k heavy-hitter heap. /// diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs index 74f3ce0c..1e9e071a 100644 --- a/src/sketches/countsketch_l2_norm.rs +++ b/src/sketches/countsketch_l2_norm.rs @@ -14,8 +14,7 @@ use crate::{ pub(crate) mod wire; -const DEFAULT_ROWS: usize = 3; -const DEFAULT_COLS: usize = 4096; +use crate::common::structures::fixed_structure::{DEFAULT_COLS, DEFAULT_ROWS}; /// Count Sketch augmented with per-row L2 norm tracking for heavy-hitter detection. #[derive(Serialize, Deserialize, Clone, Debug)] diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index cfb6801a..eba87651 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -26,7 +26,7 @@ pub(crate) use wire::{ const MAX_LEVELS: usize = 61; const CAPACITY_CACHE_LEN: usize = 20; -const MAX_K: usize = 26_602; +pub(crate) const MAX_K: usize = 26_602; const CAPACITY_DECAY: f64 = 2.0 / 3.0; const DEFAULT_K: i32 = 200; diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 9311d656..68cd42d2 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -18,12 +18,11 @@ use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; use crate::{DataInput, Vector1D}; -use super::kll::{Coin, merge_sorted_runs, randomly_halve_up}; +use super::kll::{Coin, MAX_K, merge_sorted_runs, randomly_halve_up}; mod wire; const CAPACITY_CACHE_LEN: usize = 20; -const MAX_K: usize = 26_602; const CAPACITY_DECAY: f64 = 2.0 / 3.0; const DEFAULT_K: i32 = 200; diff --git a/tests/e2e/frameworks.rs b/tests/e2e/frameworks.rs index 61be34c3..4f877a2d 100644 --- a/tests/e2e/frameworks.rs +++ b/tests/e2e/frameworks.rs @@ -1779,11 +1779,6 @@ fn univmon_generic_g_sum_reproduces_every_named_estimator() { um.estimate_l1().log2() - x_log_x / um.estimate_l1() + 1e-9, "the entropy estimator is the x*log2(x) g-sum rearranged", ); - assert_eq!( - um.estimate_g_sum(|x| x * x, false), - um.calc_g_sum_heuristic(|x| x * x, false), - "the public g-sum must be the heuristic it delegates to" - ); assert_eq!( um.estimate_g_sum(|_| 0.0, false), 0.0, From 93087ad761d0a510170ff11569720bd3e99650e0 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:43:21 +0000 Subject: [PATCH 07/11] refactor!: the structural refactors #209 lists OwnedDataInput::as_data_input and DataInput::to_owned_input replace the free conversion functions; UnivMonPool moves into univmon; the private find_bottom_layer_num wrappers are inlined as UnivMon::bottom_layer_for_hash; heap_count_from_estimate is pub(crate); KLL names its default m DEFAULT_M; Vector2D::filled replaces the unfilled + fill(0) pairs; nitro_skip takes &self. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ docs/api/api_common.md | 1 - docs/api/api_common_input.md | 11 +- docs/api/api_common_structures.md | 1 + docs/api/api_cs_heap.md | 2 +- docs/api/api_kll.md | 2 + docs/api/api_univmon.md | 14 +- docs/api/api_univmon_optimized.md | 14 +- docs/features.md | 4 +- docs/tests.md | 6 +- examples/hhheap_probe.rs | 10 +- src/common/heap.rs | 10 +- src/common/input.rs | 92 ++++++----- src/common/mod.rs | 2 +- src/common/structures/vector2d.rs | 12 +- src/sketch_framework/eh_univmon.rs | 11 +- src/sketch_framework/mod.rs | 4 +- src/sketch_framework/nitro.rs | 16 +- src/sketch_framework/octo.rs | 16 +- src/sketch_framework/univmon.rs | 148 ++++++++++++++--- src/sketch_framework/univmon_two_tier.rs | 151 ++---------------- src/sketch_framework/univmon_two_tier/wire.rs | 2 +- src/sketches/count_min_fold.rs | 10 +- src/sketches/count_sketch_fold.rs | 13 +- src/sketches/countminsketch.rs | 13 +- src/sketches/countminsketch_heap.rs | 22 +-- src/sketches/countminsketch_space_saving.rs | 8 +- src/sketches/countsketch.rs | 13 +- src/sketches/countsketch_heap.rs | 28 ++-- src/sketches/countsketch_l2_norm.rs | 8 +- src/sketches/kll.rs | 8 +- src/sketches/kll_dynamic.rs | 8 +- src/sketches/mod.rs | 4 +- src/sketches/space_saving.rs | 6 +- tests/e2e/matrix_instances.rs | 16 +- tests/e2e/octo.rs | 7 +- tests/pbt/octo_delta.rs | 12 +- tests/pbt/topk_wrappers.rs | 10 +- 38 files changed, 358 insertions(+), 368 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7452762..73fee356 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -310,6 +310,17 @@ signals a backwards-compatible change. `common::structures::fixed_structure`, and the default heap size (32) and KLL's largest `k` once each. +- **Key conversions are methods** (breaking). `heap_item_to_sketch_input(&k)` + -> `k.as_data_input()` on `OwnedDataInput`, `input_to_owned(&k)` -> + `k.to_owned_input()` on `DataInput`. +- **`UnivMonPool` lives in `sketch_framework::univmon`** beside `UnivMon`; + the crate-root path is unchanged. +- **`heap_count_from_estimate` is crate-private** (breaking); + `CountSketchHeapCounter::heap_count` is the public form. +- KLL's default `m` is the constant `kll::DEFAULT_M` (8), which `with_k` and + `with_k_and_seed` use. `Vector2D::filled(rows, cols, value)` builds a filled + matrix in one call. `Vector2D::nitro_skip` takes `&self`. + ### Removed - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); diff --git a/docs/api/api_common.md b/docs/api/api_common.md index 03a2caed..01f56627 100644 --- a/docs/api/api_common.md +++ b/docs/api/api_common.md @@ -37,7 +37,6 @@ src/common/ use asap_sketchlib::common::{ // Input layer DataInput, OwnedDataInput, HeapEntry, - input_to_owned, heap_item_to_sketch_input, // Hash layer SketchHasher, DefaultXxHasher, diff --git a/docs/api/api_common_input.md b/docs/api/api_common_input.md index 9023dc09..0d569b29 100644 --- a/docs/api/api_common_input.md +++ b/docs/api/api_common_input.md @@ -21,8 +21,11 @@ Canonical input and ownership model shared across sketches/frameworks. ## Constructors / Conversions ```rust -fn heap_item_to_sketch_input(item: &OwnedDataInput) -> DataInput<'_> -fn input_to_owned(input: &DataInput<'_>) -> OwnedDataInput +// OwnedDataInput +fn as_data_input(&self) -> DataInput<'_> + +// DataInput +fn to_owned_input(&self) -> OwnedDataInput // HeapEntry fn from_input(k: DataInput, count: C) -> Self @@ -103,14 +106,14 @@ follow the bytes: `-0.0 == 0.0` and `NaN == NaN`. ## Examples ```rust -use asap_sketchlib::{DataInput, input_to_owned}; +use asap_sketchlib::DataInput; use asap_sketchlib::common::input::HydraQuery; let q = HydraQuery::Frequency(DataInput::Str("key")); let _ = q; let key = DataInput::String("flow".to_string()); -let _owned = input_to_owned(&key); +let _owned = key.to_owned_input(); ``` ## See Also diff --git a/docs/api/api_common_structures.md b/docs/api/api_common_structures.md index f59988a3..52a9c65f 100644 --- a/docs/api/api_common_structures.md +++ b/docs/api/api_common_structures.md @@ -20,6 +20,7 @@ Shared matrix, vector, and bit storage plus utility structures used by sketch im ```rust // Vector2D fn unfilled(rows: usize, cols: usize) -> Self +fn filled(rows: usize, cols: usize, value: T) -> Self fn from_fn(rows: usize, cols: usize, f: F) -> Self // Vector3D diff --git a/docs/api/api_cs_heap.md b/docs/api/api_cs_heap.md index ca145b1e..08e0ad0e 100644 --- a/docs/api/api_cs_heap.md +++ b/docs/api/api_cs_heap.md @@ -131,7 +131,7 @@ assert!(sk.estimate(&DataInput::Str("flow")) >= 1.0); - Estimate semantics follow Count Sketch and may be non-integer. - Merge requires matching dimensions and compatible type parameters. - **Over integer counters, heap counts are a saturating, truncating view of the - estimate.** The insert path stores `heap_count_from_estimate(estimate)`, where the + estimate.** The insert path stores `CountSketchHeapCounter::heap_count(estimate)`, where the estimate is the row median as `f64` and the heap holds `i64`. The conversion saturates at `i64::MAX` / `i64::MIN` (and maps `NaN` to `0`) rather than wrapping, because a wrapped negative count would corrupt the heap's ordering. diff --git a/docs/api/api_kll.md b/docs/api/api_kll.md index ebbb30d9..e998595f 100644 --- a/docs/api/api_kll.md +++ b/docs/api/api_kll.md @@ -21,6 +21,8 @@ fn with_k_and_seed(k: i32, seed: u64) -> Self fn with_seed(k: usize, m: usize, seed: u64) -> Self ``` +`with_k` and `with_k_and_seed` set `m` to `DEFAULT_M` (8). + The unseeded constructors draw the compaction coin from the wall clock, so two runs over the same input produce different sketches. The `*_with_seed` forms fix the coin and store the seed, so `clear()` re-seeds from it too — use them diff --git a/docs/api/api_univmon.md b/docs/api/api_univmon.md index 6a841ec5..1aa9943b 100644 --- a/docs/api/api_univmon.md +++ b/docs/api/api_univmon.md @@ -4,19 +4,31 @@ Status: `Ready` ## Purpose -Universal stream-monitoring sketch for L1/L2/cardinality/entropy. +Universal stream-monitoring sketch for L1/L2/cardinality/entropy, and a +free-list pool of scratch `UnivMon`s. ## Type/Struct - `UnivMon` +- `UnivMonPool` ## Constructors ```rust fn default() -> Self fn new(heap_size: usize, rows: usize, cols: usize, layers: usize) -> Self + +// UnivMonPool +fn new(initial_size: usize, heap_size: usize, rows: usize, cols: usize, layers: usize) -> Self +fn take(&mut self) -> UnivMon +fn put(&mut self, sketch: UnivMon) +fn available(&self) -> usize +fn total_allocated(&self) -> usize ``` +`UnivMonPool` is a free-list of scratch `UnivMon`s rather than a sketch, and +has no wire kind. + `new` panics on a shape whose hashes would not be independent: - `layers > UNIVMON_MAX_LAYERS` (19). Layer `i` hashes at seed index `i`, and diff --git a/docs/api/api_univmon_optimized.md b/docs/api/api_univmon_optimized.md index 7dde1f72..4f5d7ec6 100644 --- a/docs/api/api_univmon_optimized.md +++ b/docs/api/api_univmon_optimized.md @@ -4,20 +4,15 @@ Status: `Ready` ## Purpose -Two-tier UnivMon pyramid, and a free-list pool of scratch `UnivMon`s. +Two-tier UnivMon pyramid. ## Type/Struct -- `UnivMonPool` - `UnivMonTwoTier` ## Constructors ```rust -// UnivMonPool -fn new(initial_size: usize, heap_size: usize, rows: usize, cols: usize, layers: usize) -> Self - -// UnivMonTwoTier fn new( heap_size: usize, elephant_layers: usize, @@ -46,10 +41,6 @@ fn estimate_l2(&self) -> f64 fn estimate_entropy(&self) -> f64 fn estimate_cardinality(&self) -> f64 fn estimate_g_sum(&self, g: F, heavy_only: bool) -> f64 - -// UnivMonPool -fn available(&self) -> usize -fn total_allocated(&self) -> usize ``` For the supported non-negative update stream, `estimate_l1()` returns the exact @@ -79,9 +70,6 @@ and the mouse dimensions after, so every per-layer geometry is derived and none is stored. `elephant_rows` and `mouse_rows` carry `UnivMon`'s per-layer row bound, `1 <= rows <= 20` (`MATRIX_MAX_ROWS`, the seed list length). -`UnivMonPool` is a free-list of scratch `UnivMon`s rather than a sketch, and -has no wire kind. - ## Examples ```rust diff --git a/docs/features.md b/docs/features.md index dc555805..2f5f5d63 100644 --- a/docs/features.md +++ b/docs/features.md @@ -35,9 +35,9 @@ This document provides a high-level overview of implemented and planned features **Hydra** - Subpopulation queries over multi-dimensional keys ([apis.md](apis.md)) -**UnivMon** - Universal monitoring (L1, L2, entropy, cardinality from single structure) ([apis.md](apis.md)) +**UnivMon** - Universal monitoring (L1, L2, entropy, cardinality from single structure), with `UnivMonPool` to recycle scratch sketches ([apis.md](apis.md)) -**UnivMonTwoTier** - Optimized two-tier UnivMon with `UnivMonPool` for insert and memory management ([apis.md](apis.md)) +**UnivMonTwoTier** - Two-tier UnivMon: larger elephant layers on top, smaller mouse layers below ([apis.md](apis.md)) **UnivMonQ (Experimental)** - Terminal-stratum UnivMon with exact insertion-only L1, F0/F2/compatible g-sum queries, an adaptively assisted occurrence sample for entropy, ranks, and quantiles, and reusable prepared query views. Its API, estimators, and guarantees may change. diff --git a/docs/tests.md b/docs/tests.md index 23c2aee7..ef1ca40c 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1145,7 +1145,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `compact_worker_uses_a_quarter_of_the_counter_memory` | The worker holds one byte per counter against a full sketch's four. | For a `3x4096` `CountMinSketchWorker`, verifies `counter_bytes() * 4` equals `rows * cols * size_of::()`. | | `compact_worker_honours_a_custom_threshold` | A custom threshold decides how often a counter promotes. | Inserts `U64(7)` 100 times into a `1x16` `CountMinSketchWorker` at a threshold of `10` and verifies exactly `10` promotions. | | `compact_count_worker_promotes_on_magnitude_and_clears` | The signed worker thresholds on `\|counter\|` and clears what it sends. | Inserts `U64(99)` 200 times into a `3x64` `CountSketchWorker` at `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD`, and verifies at least one delta was emitted, that every delta's `value.unsigned_abs()` is `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD`, and that every residual counter's magnitude is under `31`. | -| `keyed_deltas_carry_the_key_that_triggered_them` | A keyed delta names the key whose insert produced it. | Drives `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` inserts of `Str("flow-a")` through `update_emit_keyed_delta_with_row_hashes` on a `2x8` `CountMinSketchWorker` and verifies the emitted deltas are non-empty and every one carries `input_to_owned` of that key; the delta count and the delta values are not checked. | +| `keyed_deltas_carry_the_key_that_triggered_them` | A keyed delta names the key whose insert produced it. | Drives `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` inserts of `Str("flow-a")` through `update_emit_keyed_delta_with_row_hashes` on a `2x8` `CountMinSketchWorker` and verifies the emitted deltas are non-empty and every one carries `DataInput::to_owned_input` of that key; the delta count and the delta values are not checked. | | `dd_worker_promotes_and_clears_a_bucket` | A DDSketch bucket promotes at the threshold and is cleared. | Adds `42.0` `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` times to a `DdSketchWorker` at alpha `0.01` under a threshold of `OCTO_COUNT_MIN_DEFAULT_THRESHOLD`, and verifies exactly one delta carrying a value of `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` and an empty residual. | | `dd_worker_drops_what_the_parent_would_drop` | Values `DdSketch::add` would itself drop never promote. | Adds each of `0.0`, `-1.0`, `f64::NAN` and `f64::INFINITY` 1,000 times at a threshold of `1` and verifies zero promotions. | | `dd_promotion_bounds_the_quantile_rank_error_by_what_it_holds_back` | The held-back mass bounds the parent's quantile rank error. | Feeds 200,000 values `1.0 + (i * 7) % 999` through a `DdSketchWorker` at alpha `0.01` and a threshold of `OCTO_DDSKETCH_DEFAULT_THRESHOLD` into a parent `DdSketch`, and verifies `held_back()` plus the parent's `get_count()` equals `200_000`, that the held-back share is under `0.05`, and that the parent's `0.1`, `0.5`, `0.9` and `0.99` quantiles each fall between the exact value at rank `q - slack` scaled by `1 - alpha` and the exact value at rank `q + slack` scaled by `1 + alpha`. | @@ -1280,7 +1280,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `cm_top_k_point_estimates_trail_the_single_thread_reference_by_at_most_k_tau` | The Count-Min top-k aggregator trails a single thread by at most `k*tau` and keeps Count-Min's own bound. | Drives 200,000 zipf(1.1) keys, domain `8_192`, seed `0x0C70_1101`, through `CountMinSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both `HashByKey` and `RoundRobin`, and per route tallies over every truth key that the estimate is at or below a `CountMinSketchWithHeap` reference's and no more than `k*tau = 124` under it, that it is at or above `truth - 124`, that it is under `CountMinSpec::new(5, 2048).simultaneous_bound(..., SIMULTANEOUS_LEVEL = 1e-3)` with zero violations tolerated, and that violations of the marginal `e*(N-f)/w` bound occur at a rate no worse than `e^-5`. | | `count_top_k_point_estimates_satisfy_the_l2_bound_within_the_promotion_residual` | The Count top-k aggregator holds the L2 bound widened by exactly `k*tau`. | Drives the same 200,000-key stream through `CountSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both routes, and per route tallies over every truth key that the absolute gap to a `CountSketchWithHeap` reference is at most `k*tau = 124`, that the absolute error against truth is within `CountSketchSpec::new(5, 2048).scale_at(simultaneous_kappa(distinct, 1e-3), \|\|f_-i\|\|_2) + 124` with zero violations, and that violations of `sqrt(3/w)*\|\|f_-i\|\|_2 + 124` occur at a rate no worse than the spec's marginal failure probability. | | `top_k_plans_recover_the_heavy_hitters_their_promotion_floor_guarantees` | Both top-k plans recover every heavy hitter above the promotion floor, with a consistent heap. | Over the same 200,000-key stream and both routes, drives `CountMinSketchWithHeapOctoPlan` and `CountSketchWithHeapOctoPlan` at `5x2048` on `4` workers with a top-`32` aggregator and verifies per plan and route that the heap holds at most `32` entries with no duplicate key, that at least `16` of the true top `32` clear the `k*tau = 124` floor (a premise check), that *every* true top-32 key above that floor is present, that no heap key's true count is below the true 32nd count minus `124`, and that each heap entry's stored count equals the aggregator sketch's own estimate for that key. | -| `a_key_below_the_promotion_threshold_never_reaches_the_top_k_aggregator` | A flow below tau is absent from the top-k aggregator entirely, and the first crossing ships exactly tau. | On `CountMinSketchWithHeapOctoPlan::new(5, 2048)` with one worker and a top-`32` aggregator, sends `U64(0xC01D)` `TAU - 1 = 30` times and verifies zero deltas shipped, that the aggregator estimates `0` for it, and that the heap is empty; then sends one more arrival and verifies at least one delta was promoted, that every delta's `key` is `input_to_owned` of that key, that the aggregator then estimates exactly `TAU = 31`, and that the heap holds exactly `1` entry. | +| `a_key_below_the_promotion_threshold_never_reaches_the_top_k_aggregator` | A flow below tau is absent from the top-k aggregator entirely, and the first crossing ships exactly tau. | On `CountMinSketchWithHeapOctoPlan::new(5, 2048)` with one worker and a top-`32` aggregator, sends `U64(0xC01D)` `TAU - 1 = 30` times and verifies zero deltas shipped, that the aggregator estimates `0` for it, and that the heap is empty; then sends one more arrival and verifies at least one delta was promoted, that every delta's `key` is `DataInput::to_owned_input` of that key, that the aggregator then estimates exactly `TAU = 31`, and that the heap holds exactly `1` entry. | | `hll_octo_plan_is_register_exact_and_in_band_under_both_partitions` | `HllOctoPlan` is register-exact and in cardinality band under either partition. | Drives 60,000 zipf(1.1) keys, domain `1_024`, seed `0x0C70_ACC0`, through `HllOctoPlan` on `4` workers under `HashByKey` and `RoundRobin`, flushes each worker, and per route verifies the parent's `registers_as_slice()` equals a single-threaded `HyperLogLog`'s exactly and that `estimate()` passes `CardinalityConfidenceSpec::hll(14, 4.0)` against the exact distinct count. | | `dd_octo_plan_is_bucket_exact_after_flush_and_holds_alpha_under_both_partitions` | `DdOctoPlan` is bucket-exact after a flush and holds alpha under either partition. | Turns the same 60,000-key stream into values `1.0 + key`, drives them through `DdOctoPlan::new(0.01)` on `4` workers under both routes, flushes, and per route verifies `get_count()` equals a single-pass `DdSketch`'s, that every bucket over the union of both stores' index ranges matches the reference (offset differences allowed for), and that the quantiles `0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0` all satisfy `RelativeQuantileSpec::core(0.01)` against the exact order statistics, with zero violations tolerated. | | `univmon_octo_plan_reports_the_delivered_l1_and_holds_its_f2_bound_under_both_partitions` | The UnivMon parent's L1 is exactly the delivered weight, and its F2 holds the AMS bound. | Drives the same 60,000-key stream through `UnivMonOctoPlan::new(5, 4096, 8)` with heap `64` on `4` workers under both routes, recording the largest `total_weight` each worker stamped on a delta, and per route verifies the delivered sum never exceeds `N`, that `parent.estimate_l1()` equals that sum *exactly* (`UnivMonOctoWorker` has no flush, so mass after a worker's last delta stays in flight and the printed context names how much), and that `estimate_l2().powi(2)` passes `SecondMomentSpec::new(5, 4096)`'s `sqrt(2*kappa/w)` relative band against the exact F2. | @@ -1571,7 +1571,7 @@ Test file: [`src/common/input.rs`](../src/common/input.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `a_byte_array_owns_and_borrows_back_unchanged` | A borrowed byte array owns as a byte array, not as a string. | Verifies `input_to_owned(&DataInput::Bytes([0xff, 0x00, 0xfe]))` is `OwnedDataInput::Bytes` of those bytes, that `heap_item_to_sketch_input` borrows the same bytes back, and that the owned key compares equal to the original input. | +| `a_byte_array_owns_and_borrows_back_unchanged` | A borrowed byte array owns as a byte array, not as a string. | Verifies `DataInput::Bytes([0xff, 0x00, 0xfe]).to_owned_input()` is `OwnedDataInput::Bytes` of those bytes, that `OwnedDataInput::as_data_input` borrows the same bytes back, and that the owned key compares equal to the original input. | | `a_byte_key_is_not_the_string_of_the_same_bytes` | `Bytes` and `Str` are different keys even where the bytes are valid UTF-8. | Verifies the owned forms of `Bytes(b"abc")` and `Str("abc")` are unequal, that the byte key matches only `DataInput::Bytes`, and that the string key matches `Str` and `String` but not `Bytes`. | ### Tuple Keys diff --git a/examples/hhheap_probe.rs b/examples/hhheap_probe.rs index ca311b1b..39710b0e 100644 --- a/examples/hhheap_probe.rs +++ b/examples/hhheap_probe.rs @@ -10,7 +10,7 @@ use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, DataInput, FastPath, HeapEntry, OwnedDataInput, - TopKHeap, Vector2D, hash64_seeded, input_to_owned, + TopKHeap, Vector2D, hash64_seeded, }; use std::collections::{HashMap, HashSet}; use std::hash::{BuildHasherDefault, Hasher}; @@ -81,7 +81,7 @@ impl IndexedHeap { let retained_every_key = self.data.len() < self.k; if self.data.len() < self.k { let idx = self.data.len(); - self.data.push(HeapEntry::new(input_to_owned(key), count)); + self.data.push(HeapEntry::new(key.to_owned_input(), count)); self.slots.push(slot); self.register(slot, idx); self.bubble_up(idx); @@ -94,7 +94,7 @@ impl IndexedHeap { let old_slot = self.slots[0]; self.unregister(old_slot, 0); - self.data[0] = HeapEntry::new(input_to_owned(key), count); + self.data[0] = HeapEntry::new(key.to_owned_input(), count); self.slots[0] = slot; self.register(slot, 0); self.bubble_down(0); @@ -218,14 +218,14 @@ impl ScanHeap { let retained_every_key = self.data.len() < self.k; if self.data.len() < self.k { - self.data.push(HeapEntry::new(input_to_owned(key), count)); + self.data.push(HeapEntry::new(key.to_owned_input(), count)); self.bubble_up(self.data.len() - 1); return retained_every_key; } if count <= self.data[0].count { return retained_every_key; } - self.data[0] = HeapEntry::new(input_to_owned(key), count); + self.data[0] = HeapEntry::new(key.to_owned_input(), count); self.bubble_down(0); retained_every_key } diff --git a/src/common/heap.rs b/src/common/heap.rs index 9e86b46b..a4e06516 100644 --- a/src/common/heap.rs +++ b/src/common/heap.rs @@ -8,7 +8,7 @@ use crate::common::input::HeapEntry; use crate::common::structures::heap::PREALLOCATED_SLOTS; use crate::common::{CommonHeap, DigestBuildHasher, RetainLargest}; -use crate::{DataInput, OwnedDataInput, hash64_seeded, hash64_seeded_owned, input_to_owned}; +use crate::{DataInput, OwnedDataInput, hash64_seeded, hash64_seeded_owned}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use smallvec::SmallVec; use std::cmp::Ordering; @@ -152,7 +152,7 @@ impl TopKHeap { if !self.should_accept_new(count) { return retained_every_key; } - self.seat(input_to_owned(key), slot, count); + self.seat(key.to_owned_input(), slot, count); retained_every_key } @@ -1138,7 +1138,7 @@ mod differential { return retained_every_key; } - let owned = input_to_owned(key); + let owned = key.to_owned_input(); self.heap.push(HeapEntry::new(owned, count)); self.refresh_positions(); retained_every_key @@ -1193,7 +1193,7 @@ mod differential { } if self.residents.len() < self.k { self.residents - .push(HeapEntry::new(input_to_owned(key), count)); + .push(HeapEntry::new(key.to_owned_input(), count)); return retained_every_key; } @@ -1205,7 +1205,7 @@ mod differential { .map(|(idx, _)| idx) .expect("a full heap of capacity k > 0 has residents"); if count > self.residents[smallest].count { - self.residents[smallest] = HeapEntry::new(input_to_owned(key), count); + self.residents[smallest] = HeapEntry::new(key.to_owned_input(), count); } retained_every_key } diff --git a/src/common/input.rs b/src/common/input.rs index fe788606..b84954f6 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -89,48 +89,52 @@ pub enum OwnedDataInput { Bytes(Vec), } -/// Converts a heap-owned key into a borrowed sketch input. -pub fn heap_item_to_sketch_input(item: &OwnedDataInput) -> DataInput<'_> { - match item { - OwnedDataInput::I8(v) => DataInput::I8(*v), - OwnedDataInput::I16(v) => DataInput::I16(*v), - OwnedDataInput::I32(v) => DataInput::I32(*v), - OwnedDataInput::I64(v) => DataInput::I64(*v), - OwnedDataInput::I128(v) => DataInput::I128(*v), - OwnedDataInput::Isize(v) => DataInput::Isize(*v), - OwnedDataInput::U8(v) => DataInput::U8(*v), - OwnedDataInput::U16(v) => DataInput::U16(*v), - OwnedDataInput::U32(v) => DataInput::U32(*v), - OwnedDataInput::U64(v) => DataInput::U64(*v), - OwnedDataInput::U128(v) => DataInput::U128(*v), - OwnedDataInput::Usize(v) => DataInput::Usize(*v), - OwnedDataInput::F32(v) => DataInput::F32(*v), - OwnedDataInput::F64(v) => DataInput::F64(*v), - OwnedDataInput::String(s) => DataInput::Str(s), - OwnedDataInput::Bytes(b) => DataInput::Bytes(b.as_slice()), +impl OwnedDataInput { + /// Borrows this owned key as a sketch input. + pub fn as_data_input(&self) -> DataInput<'_> { + match self { + OwnedDataInput::I8(v) => DataInput::I8(*v), + OwnedDataInput::I16(v) => DataInput::I16(*v), + OwnedDataInput::I32(v) => DataInput::I32(*v), + OwnedDataInput::I64(v) => DataInput::I64(*v), + OwnedDataInput::I128(v) => DataInput::I128(*v), + OwnedDataInput::Isize(v) => DataInput::Isize(*v), + OwnedDataInput::U8(v) => DataInput::U8(*v), + OwnedDataInput::U16(v) => DataInput::U16(*v), + OwnedDataInput::U32(v) => DataInput::U32(*v), + OwnedDataInput::U64(v) => DataInput::U64(*v), + OwnedDataInput::U128(v) => DataInput::U128(*v), + OwnedDataInput::Usize(v) => DataInput::Usize(*v), + OwnedDataInput::F32(v) => DataInput::F32(*v), + OwnedDataInput::F64(v) => DataInput::F64(*v), + OwnedDataInput::String(s) => DataInput::Str(s), + OwnedDataInput::Bytes(b) => DataInput::Bytes(b.as_slice()), + } } } -/// Converts a sketch input into an owned heap item. -pub fn input_to_owned<'a>(input: &DataInput<'a>) -> OwnedDataInput { - match input { - DataInput::I8(i) => OwnedDataInput::I8(*i), - DataInput::I16(i) => OwnedDataInput::I16(*i), - DataInput::I32(i) => OwnedDataInput::I32(*i), - DataInput::I64(i) => OwnedDataInput::I64(*i), - DataInput::I128(i) => OwnedDataInput::I128(*i), - DataInput::Isize(i) => OwnedDataInput::Isize(*i), - DataInput::U8(u) => OwnedDataInput::U8(*u), - DataInput::U16(u) => OwnedDataInput::U16(*u), - DataInput::U32(u) => OwnedDataInput::U32(*u), - DataInput::U64(u) => OwnedDataInput::U64(*u), - DataInput::U128(u) => OwnedDataInput::U128(*u), - DataInput::Usize(u) => OwnedDataInput::Usize(*u), - DataInput::F32(f) => OwnedDataInput::F32(*f), - DataInput::F64(f) => OwnedDataInput::F64(*f), - DataInput::Str(s) => OwnedDataInput::String((*s).to_owned()), - DataInput::String(s) => OwnedDataInput::String((*s).to_owned()), - DataInput::Bytes(b) => OwnedDataInput::Bytes((*b).to_vec()), +impl DataInput<'_> { + /// Copies this input into an owned key. + pub fn to_owned_input(&self) -> OwnedDataInput { + match self { + DataInput::I8(i) => OwnedDataInput::I8(*i), + DataInput::I16(i) => OwnedDataInput::I16(*i), + DataInput::I32(i) => OwnedDataInput::I32(*i), + DataInput::I64(i) => OwnedDataInput::I64(*i), + DataInput::I128(i) => OwnedDataInput::I128(*i), + DataInput::Isize(i) => OwnedDataInput::Isize(*i), + DataInput::U8(u) => OwnedDataInput::U8(*u), + DataInput::U16(u) => OwnedDataInput::U16(*u), + DataInput::U32(u) => OwnedDataInput::U32(*u), + DataInput::U64(u) => OwnedDataInput::U64(*u), + DataInput::U128(u) => OwnedDataInput::U128(*u), + DataInput::Usize(u) => OwnedDataInput::Usize(*u), + DataInput::F32(f) => OwnedDataInput::F32(*f), + DataInput::F64(f) => OwnedDataInput::F64(*f), + DataInput::Str(s) => OwnedDataInput::String((*s).to_owned()), + DataInput::String(s) => OwnedDataInput::String((*s).to_owned()), + DataInput::Bytes(b) => OwnedDataInput::Bytes((*b).to_vec()), + } } } @@ -518,7 +522,7 @@ impl HeapEntry { /// Creates an entry by copying a borrowed input. pub fn from_input(k: DataInput, count: C) -> Self { HeapEntry { - key: input_to_owned(&k), + key: k.to_owned_input(), count, } } @@ -567,9 +571,9 @@ mod tests { /// bytes, whatever they are. #[test] fn a_byte_array_owns_and_borrows_back_unchanged() { - let owned = input_to_owned(&DataInput::Bytes(RAW)); + let owned = DataInput::Bytes(RAW).to_owned_input(); assert_eq!(owned, OwnedDataInput::Bytes(RAW.to_vec())); - assert_eq!(heap_item_to_sketch_input(&owned), DataInput::Bytes(RAW)); + assert_eq!(owned.as_data_input(), DataInput::Bytes(RAW)); assert_eq!(owned, DataInput::Bytes(RAW)); } @@ -577,8 +581,8 @@ mod tests { /// UTF-8, so a sketch holding one never answers for the other. #[test] fn a_byte_key_is_not_the_string_of_the_same_bytes() { - let bytes = input_to_owned(&DataInput::Bytes(b"abc")); - let string = input_to_owned(&DataInput::Str("abc")); + let bytes = DataInput::Bytes(b"abc").to_owned_input(); + let string = DataInput::Str("abc").to_owned_input(); assert_ne!(bytes, string); assert_eq!(bytes, DataInput::Bytes(b"abc")); diff --git a/src/common/mod.rs b/src/common/mod.rs index 68f7fae0..727d02cc 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -38,7 +38,7 @@ pub use hash::{ hash64_seeded_owned, hash128_seeded, hash128_seeded_owned, }; pub use heap::{HeapCount, HeapCounter, TopKHeap}; -pub use input::{DataInput, HeapEntry, OwnedDataInput, heap_item_to_sketch_input, input_to_owned}; +pub use input::{DataInput, HeapEntry, OwnedDataInput}; pub use numerical::NumericalValue; pub use precompute_hash::PRECOMPUTED_HASH; pub use precompute_sample::LN_ONE_MINUS_U_TABLE; diff --git a/src/common/structures/vector2d.rs b/src/common/structures/vector2d.rs index b81e5967..a6ec9a22 100644 --- a/src/common/structures/vector2d.rs +++ b/src/common/structures/vector2d.rs @@ -66,6 +66,16 @@ impl Vector2D { } } + /// Creates a `rows` x `cols` matrix whose every cell holds `value`. + pub fn filled(rows: usize, cols: usize, value: T) -> Self + where + T: Clone, + { + let mut matrix = Self::unfilled(rows, cols); + matrix.fill(value); + matrix + } + /// Builds a matrix by invoking a generator for every `(row, col)` position. /// Useful for types that require per-cell construction logic (e.g., heaps or buckets) /// instead of cloning a single value across all cells. @@ -318,7 +328,7 @@ impl Vector2D { #[inline(always)] /// Returns the current Nitro skip counter. - pub fn nitro_skip(&mut self) -> usize { + pub fn nitro_skip(&self) -> usize { self.nitro.to_skip } diff --git a/src/sketch_framework/eh_univmon.rs b/src/sketch_framework/eh_univmon.rs index 172f9bd4..d8a82624 100644 --- a/src/sketch_framework/eh_univmon.rs +++ b/src/sketch_framework/eh_univmon.rs @@ -4,14 +4,13 @@ //! buckets use full UnivMon sketches. When a map bucket grows too large, //! it is promoted to a UnivMon sketch. //! -//! Uses `UnivMonPool` (from `univmon_two_tier`) to recycle `UnivMon` +//! Uses `UnivMonPool` (from `univmon`) to recycle `UnivMon` //! instances across promotion, merge, and expiration cycles. use std::collections::HashMap; use crate::common::heap::DEFAULT_HEAP_SIZE; -use crate::common::input::{heap_item_to_sketch_input, input_to_owned}; -use crate::sketch_framework::univmon_two_tier::UnivMonPool; +use crate::sketch_framework::univmon::UnivMonPool; use crate::{DataInput, OwnedDataInput, UnivMon}; const MASS_EPSILON: f64 = 1e-9; @@ -202,7 +201,7 @@ impl ExponentialHistogramUnivMon { } // 3. Create new map bucket - let owned_key = input_to_owned(key); + let owned_key = key.to_owned_input(); let mut counts = HashMap::new(); counts.insert(owned_key, weight); self.exact_buckets.push(EhUnivMonExactBucket { @@ -257,7 +256,7 @@ impl ExponentialHistogramUnivMon { // if the pool has a recycled sketch available. let mut sketch = self.pool.take(); for (key, value) in &oldest.counts { - let input = heap_item_to_sketch_input(key); + let input = key.as_data_input(); sketch.update_weighted(&input, *value); } @@ -421,7 +420,7 @@ impl ExponentialHistogramUnivMon { // Insert map entries into merged sketch for (key, value) in &map_merged { - let input = heap_item_to_sketch_input(key); + let input = key.as_data_input(); merged.update_weighted(&input, *value); } diff --git a/src/sketch_framework/mod.rs b/src/sketch_framework/mod.rs index 794581b4..d05e6320 100644 --- a/src/sketch_framework/mod.rs +++ b/src/sketch_framework/mod.rs @@ -37,7 +37,7 @@ pub mod hydra; pub use hydra::Hydra; pub mod univmon; -pub use univmon::{UnivMon, UnivMonDeltaFidelity}; +pub use univmon::{UnivMon, UnivMonDeltaFidelity, UnivMonPool}; pub mod univmon_q; pub use univmon_q::{ @@ -46,7 +46,7 @@ pub use univmon_q::{ }; pub mod univmon_two_tier; -pub use univmon_two_tier::{UnivMonPool, UnivMonTwoTier}; +pub use univmon_two_tier::UnivMonTwoTier; pub mod nitro; pub use nitro::{ diff --git a/src/sketch_framework/nitro.rs b/src/sketch_framework/nitro.rs index 9969a56f..55dea103 100644 --- a/src/sketch_framework/nitro.rs +++ b/src/sketch_framework/nitro.rs @@ -151,7 +151,7 @@ fn new_small_rng() -> SmallRng { impl Default for NitroBatch> { fn default() -> Self { - let mut n = NitroBatch { + NitroBatch { sampling_rate: 0.0, to_skip: 0, inv_ln_one_minus_p: 0.0, @@ -159,27 +159,21 @@ impl Default for NitroBatch> { generator: new_small_rng(), idx: 0, mask: SKIP_TABLE_LEN - 1, - sk: Vector2D::unfilled(5, 2048), - }; - n.sk.fill(0); - n + sk: Vector2D::filled(5, 2048, 0), + } } } impl NitroBatch> { /// Creates a Nitro sketch with the given sampling rate. pub fn new(rate: f64) -> Self { - let mut sk = Vector2D::unfilled(5, 2048); - sk.fill(0); - Self::with_target(rate, sk) + Self::with_target(rate, Vector2D::filled(5, 2048, 0)) } /// [`NitroBatch::new`] with an explicit sampling-RNG seed. See /// [`NitroBatch::with_target_and_seed`]. pub fn with_seed(rate: f64, seed: u64) -> Self { - let mut sk = Vector2D::unfilled(5, 2048); - sk.fill(0); - Self::with_target_and_seed(rate, sk, seed) + Self::with_target_and_seed(rate, Vector2D::filled(5, 2048, 0), seed) } } diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index d0aceeaf..3d6a67ef 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -51,7 +51,7 @@ use crate::{ DdSketch, HllRegisterDelta, HyperLogLog, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, OCTO_HLL_DEFAULT_THRESHOLD, OwnedDataInput, RegularPath, UNIVMON_BOTTOM_LAYER_SEED_INDEX, UnivMon, UnivMonCellDelta, Vector2D, hash64_seeded, - hash128_seeded, heap_item_to_sketch_input, input_to_owned, + hash128_seeded, }; use crate::{ CocoDelta, DEFAULT_CANONICAL_SEED_INDEX, ElasticDelta, OCTO_COCO_DEFAULT_THRESHOLD, @@ -1548,7 +1548,7 @@ impl OctoAggregator for CountMinSketchWithHeapOctoAggregator { #[inline(always)] fn apply(&mut self, delta: KeyedCountMinSketchCellDelta) { self.sketch.count_min_sketch_mut().apply_delta(delta.delta); - let key = heap_item_to_sketch_input(&delta.key); + let key = delta.key.as_data_input(); let estimate = self.sketch.count_min_sketch().estimate(&key); self.sketch.heap_mut().update(&key, estimate as i64); } @@ -1700,7 +1700,7 @@ impl OctoAggregator for CountSketchWithHeapOctoAggregator { #[inline(always)] fn apply(&mut self, delta: KeyedCountSketchCellDelta) { self.sketch.count_sketch_mut().apply_delta(delta.delta); - let key = heap_item_to_sketch_input(&delta.key); + let key = delta.key.as_data_input(); let estimate = self.sketch.count_sketch().estimate(&key); self.sketch.heap_mut().update(&key, estimate as i64); } @@ -2640,7 +2640,7 @@ impl OctoPlan for CountMinSketchWithHeapOctoPlan { fn prepare(&self, input: &DataInput<'_>) -> KeyedHashes { KeyedHashes { hashes: CountMinSketchWorker::hashes(self.rows, input), - key: input_to_owned(input), + key: input.to_owned_input(), } } } @@ -2655,7 +2655,7 @@ impl OctoPlan for CountSketchWithHeapOctoPlan { fn prepare(&self, input: &DataInput<'_>) -> KeyedHashes { KeyedHashes { hashes: CountMinSketchWorker::hashes(self.rows, input), - key: input_to_owned(input), + key: input.to_owned_input(), } } } @@ -2937,7 +2937,7 @@ impl OctoPlan for UnivMonOctoPlan { KeyedLayerHashes { bottom_layer: bottom, layer_hashes: (0..=bottom).map(|l| hash128_seeded(l, input)).collect(), - key: input_to_owned(input), + key: input.to_owned_input(), } } } @@ -3047,14 +3047,14 @@ mod worker_tests { for _ in 0..OCTO_COUNT_MIN_DEFAULT_THRESHOLD { worker.update_emit_keyed_delta_with_row_hashes( &CountMinSketchWorker::hashes(2, &key), - &input_to_owned(&key), + &key.to_owned_input(), OCTO_COUNT_MIN_DEFAULT_THRESHOLD, &mut |d| keyed.push(d), ); } assert!(!keyed.is_empty()); for d in &keyed { - assert_eq!(d.key, input_to_owned(&key)); + assert_eq!(d.key, key.to_owned_input()); } } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 9253451f..37de88e2 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -27,9 +27,8 @@ use crate::common::Vector1D; use crate::common::heap::{DEFAULT_HEAP_SIZE, TopKHeap}; use crate::common::{ DataInput, OwnedDataInput, SEED_LIST, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, - hash64_seeded_owned, heap_item_to_sketch_input, + hash64_seeded_owned, }; -use crate::common::Vector1D; use crate::octo_delta::UnivMonCellDelta; use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use serde::{Deserialize, Serialize}; @@ -171,11 +170,6 @@ impl UnivMon { .expect("UnivMon total weight overflowed usize"); } - #[inline(always)] - fn find_bottom_layer_num(&self, hash: u64, layer: usize) -> usize { - UnivMon::bottom_layer_for_hash(hash, layer) - } - /// Deepest pyramid layer an insert of `key` reaches; it touches `0..=this`. /// /// A pure function of the key's hash, so an OctoSketch worker picks exactly @@ -246,7 +240,7 @@ impl UnivMon { ); self.layer_sketches[layer].apply_delta(delta.cell); - let key = heap_item_to_sketch_input(&delta.key); + let key = delta.key.as_data_input(); let count = self.layer_sketches[layer].estimate(&key); let heap_kept_everything = self.layer_heaps[layer].update(&key, count as i64); @@ -280,7 +274,7 @@ impl UnivMon { pub fn update_weighted(&mut self, key: &DataInput, weight: i64) { self.begin_update(weight, UnivMonUpdateMode::Standard); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - let bottom_layer_num = self.find_bottom_layer_num(h, self.layers); + let bottom_layer_num = UnivMon::bottom_layer_for_hash(h, self.layers); self.process_univmon(key, weight, bottom_layer_num) } @@ -292,7 +286,7 @@ impl UnivMon { pub fn update_weighted_terminal(&mut self, key: &DataInput, weight: i64) { self.begin_update(weight, UnivMonUpdateMode::Terminal); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - let bottom_layer_num = self.find_bottom_layer_num(h, self.layers); + let bottom_layer_num = UnivMon::bottom_layer_for_hash(h, self.layers); let count = self.layer_sketches[bottom_layer_num].update_weighted_and_estimate(key, weight); if !self.layer_heaps[bottom_layer_num].update(key, count as i64) { self.candidate_complete[bottom_layer_num] = false; @@ -332,7 +326,7 @@ impl UnivMon { tmp = 0.0; for item in self.layer_heaps[self.layers - 1].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[self.layers - 1].estimate(&input) as i64; if count > threshold { tmp += g(count as f64); @@ -350,7 +344,7 @@ impl UnivMon { }; for item in self.layer_heaps[i].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[i].estimate(&input) as i64; if count > threshold { let hash = (hash64_seeded_owned(UNIVMON_BOTTOM_LAYER_SEED_INDEX, &item.key) @@ -415,7 +409,7 @@ impl UnivMon { for level in (0..self.layers).rev() { suffix_complete &= self.candidate_complete[level]; for item in self.layer_heaps[level].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[level].estimate(&input) as i64; cumulative .entry(item.key.clone()) @@ -532,7 +526,7 @@ impl UnivMon { self.layer_sketches[i].merge(&other.layer_sketches[i]); self.layer_heaps[i].clear(); for key in candidate_keys { - let input = heap_item_to_sketch_input(&key); + let input = key.as_data_input(); let count = self.layer_sketches[i].estimate(&input) as i64; self.layer_heaps[i].update(&input, count); } @@ -546,6 +540,73 @@ impl UnivMon { } } +/// Object pool for `UnivMon` sketches. +/// +/// Maintains a free-list of pre-allocated sketches. Callers take ownership +/// via `take()` and return sketches via `put()`, which resets and recycles +/// them. This avoids repeated heap allocation/deallocation for large sketch +/// matrices during promotion, merge, and expiration cycles. +pub struct UnivMonPool { + free_list: Vec, + total_allocated: usize, + heap_size: usize, + rows: usize, + cols: usize, + layers: usize, +} + +impl UnivMonPool { + /// Creates a new pool with `initial_size` pre-allocated sketches. + pub fn new( + initial_size: usize, + heap_size: usize, + rows: usize, + cols: usize, + layers: usize, + ) -> Self { + let free_list: Vec = (0..initial_size) + .map(|_| UnivMon::new(heap_size, rows, cols, layers)) + .collect(); + UnivMonPool { + free_list, + total_allocated: initial_size, + heap_size, + rows, + cols, + layers, + } + } + + /// Takes ownership of a clean sketch from the pool. + /// + /// Pops a recycled sketch from the free-list if available, otherwise + /// allocates a fresh one. + pub fn take(&mut self) -> UnivMon { + if let Some(sketch) = self.free_list.pop() { + sketch + } else { + self.total_allocated += 1; + UnivMon::new(self.heap_size, self.rows, self.cols, self.layers) + } + } + + /// Returns a sketch to the pool for reuse. Resets all internal state. + pub fn put(&mut self, mut sketch: UnivMon) { + sketch.clear(); + self.free_list.push(sketch); + } + + /// Number of sketches currently available in the pool. + pub fn available(&self) -> usize { + self.free_list.len() + } + + /// Total number of sketches ever allocated by this pool. + pub fn total_allocated(&self) -> usize { + self.total_allocated + } +} + #[cfg(test)] mod tests { use super::*; @@ -553,6 +614,57 @@ mod tests { use rand::{Rng, SeedableRng, rngs::StdRng}; use std::collections::HashMap; + #[test] + fn pool_basic_take_put() { + let mut pool = UnivMonPool::new(2, 16, 2, 5, 2); + assert_eq!(pool.available(), 2); + assert_eq!(pool.total_allocated(), 2); + + let s0 = pool.take(); + assert_eq!(pool.available(), 1); + + let s1 = pool.take(); + assert_eq!(pool.available(), 0); + + // Pool is empty — next take allocates a new one + let s2 = pool.take(); + assert_eq!(pool.available(), 0); + assert_eq!(pool.total_allocated(), 3); + + // Return one + pool.put(s1); + assert_eq!(pool.available(), 1); + + // Should reuse the returned sketch + let s3 = pool.take(); + assert_eq!(pool.available(), 0); + assert_eq!(pool.total_allocated(), 3); // no new allocation + + // Return all + pool.put(s0); + pool.put(s2); + pool.put(s3); + assert_eq!(pool.available(), 3); + } + + #[test] + fn pool_free_resets_sketch() { + let mut pool = UnivMonPool::new(1, 16, 2, 5, 2); + + // Take a sketch, insert some data + let mut sketch = pool.take(); + sketch.update_weighted(&DataInput::I64(42), 100); + assert!(sketch.total_weight > 0); + + // Return it — should reset + pool.put(sketch); + + // Take it back — should be clean + let sketch2 = pool.take(); + assert_eq!(sketch2.total_weight, 0); + assert!((sketch2.layer_sketches[0].l2()).abs() < 1e-9); + } + /// Every layer hashes at its own seed, inside the seed list (no wrap onto /// layer 0) and never at the seed that picks the layers. #[test] @@ -797,11 +909,11 @@ mod tests { .map(DataInput::U64) .find(|key| { let hash = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - sketch.find_bottom_layer_num(hash, sketch.layers) >= 2 + UnivMon::bottom_layer_for_hash(hash, sketch.layers) >= 2 }) .expect("find a key sampled into multiple layers"); let hash = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, &key); - let bottom = sketch.find_bottom_layer_num(hash, sketch.layers); + let bottom = UnivMon::bottom_layer_for_hash(hash, sketch.layers); sketch.update_weighted(&key, 3); for level in 0..=bottom { assert_eq!(sketch.layer_sketches[level].l2(), 3.0); @@ -815,11 +927,11 @@ mod tests { .map(DataInput::U64) .find(|key| { let hash = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - sketch.find_bottom_layer_num(hash, sketch.layers) >= 2 + UnivMon::bottom_layer_for_hash(hash, sketch.layers) >= 2 }) .expect("find a key sampled into multiple layers"); let hash = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, &key); - let bottom = sketch.find_bottom_layer_num(hash, sketch.layers); + let bottom = UnivMon::bottom_layer_for_hash(hash, sketch.layers); sketch.update_weighted_terminal(&key, 3); for level in 0..sketch.layers { diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index 5baba16e..03eb8e61 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -1,91 +1,19 @@ -//! Optimized UnivMon variants and pooling. -//! -//! - `UnivMonPool`: free-list pool for `UnivMon` sketch reuse. -//! - `UnivMonTwoTier`: pyramid-structured UnivMon with two-tier sketch -//! dimensions — larger "elephant" layers for heavy hitters and smaller -//! "mouse" layers for the long tail, matching the PromSketch design. +//! `UnivMonTwoTier`: pyramid-structured UnivMon with two-tier sketch +//! dimensions — larger "elephant" layers for heavy hitters and smaller +//! "mouse" layers for the long tail, matching the PromSketch design. use crate::UnivMon; use crate::common::Vector1D; use crate::common::heap::{DEFAULT_HEAP_SIZE, TopKHeap}; use crate::common::{ DataInput, OwnedDataInput, UNIVMON_BOTTOM_LAYER_SEED_INDEX, hash64_seeded, hash64_seeded_owned, - heap_item_to_sketch_input, }; -use crate::common::Vector1D; use crate::sketch_framework::univmon::{UNIVMON_MAX_LAYERS, UnivMonUpdateMode}; use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; use std::collections::{HashMap, HashSet}; mod wire; -/// Object pool for `UnivMon` sketches. -/// -/// Maintains a free-list of pre-allocated sketches. Callers take ownership -/// via `take()` and return sketches via `put()`, which resets and recycles -/// them. This avoids repeated heap allocation/deallocation for large sketch -/// matrices during promotion, merge, and expiration cycles. -pub struct UnivMonPool { - free_list: Vec, - total_allocated: usize, - heap_size: usize, - rows: usize, - cols: usize, - layers: usize, -} - -impl UnivMonPool { - /// Creates a new pool with `initial_size` pre-allocated sketches. - pub fn new( - initial_size: usize, - heap_size: usize, - rows: usize, - cols: usize, - layers: usize, - ) -> Self { - let free_list: Vec = (0..initial_size) - .map(|_| UnivMon::new(heap_size, rows, cols, layers)) - .collect(); - UnivMonPool { - free_list, - total_allocated: initial_size, - heap_size, - rows, - cols, - layers, - } - } - - /// Takes ownership of a clean sketch from the pool. - /// - /// Pops a recycled sketch from the free-list if available, otherwise - /// allocates a fresh one. - pub fn take(&mut self) -> UnivMon { - if let Some(sketch) = self.free_list.pop() { - sketch - } else { - self.total_allocated += 1; - UnivMon::new(self.heap_size, self.rows, self.cols, self.layers) - } - } - - /// Returns a sketch to the pool for reuse. Resets all internal state. - pub fn put(&mut self, mut sketch: UnivMon) { - sketch.clear(); - self.free_list.push(sketch); - } - - /// Number of sketches currently available in the pool. - pub fn available(&self) -> usize { - self.free_list.len() - } - - /// Total number of sketches ever allocated by this pool. - pub fn total_allocated(&self) -> usize { - self.total_allocated - } -} - // --------------------------------------------------------------------------- // UnivMonTwoTier // --------------------------------------------------------------------------- @@ -225,21 +153,11 @@ impl UnivMonTwoTier { ) } - #[inline(always)] - fn find_bottom_layer_num(&self, hash: u64) -> usize { - for l in 1..self.layers { - if ((hash >> l) & 1) == 0 { - return l - 1; - } - } - self.layers - 1 - } - /// Standard insert: updates sketch + heap at every layer 0..=bottom. pub fn update_weighted(&mut self, key: &DataInput, weight: i64) { self.begin_update(weight, UnivMonUpdateMode::Standard); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - let bottom = self.find_bottom_layer_num(h); + let bottom = UnivMon::bottom_layer_for_hash(h, self.layers); for i in 0..=bottom { let count = self.layer_sketches[i].update_weighted_and_estimate(key, weight); if !self.layer_heaps[i].update(key, count as i64) { @@ -255,7 +173,7 @@ impl UnivMonTwoTier { pub fn update_weighted_terminal(&mut self, key: &DataInput, weight: i64) { self.begin_update(weight, UnivMonUpdateMode::Terminal); let h = hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key); - let bottom = self.find_bottom_layer_num(h); + let bottom = UnivMon::bottom_layer_for_hash(h, self.layers); let count = self.layer_sketches[bottom].update_weighted_and_estimate(key, weight); if !self.layer_heaps[bottom].update(key, count as i64) { self.candidate_complete[bottom] = false; @@ -287,7 +205,7 @@ impl UnivMonTwoTier { let mut tmp = 0.0; for item in self.layer_heaps[self.layers - 1].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[self.layers - 1].estimate(&input) as i64; if count > threshold_top { tmp += g(count as f64); @@ -305,7 +223,7 @@ impl UnivMonTwoTier { }; for item in self.layer_heaps[i].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[i].estimate(&input) as i64; if count > threshold { let hash = (hash64_seeded_owned(UNIVMON_BOTTOM_LAYER_SEED_INDEX, &item.key) @@ -367,7 +285,7 @@ impl UnivMonTwoTier { for level in (0..self.layers).rev() { suffix_complete &= self.candidate_complete[level]; for item in self.layer_heaps[level].heap() { - let input = heap_item_to_sketch_input(&item.key); + let input = item.key.as_data_input(); let count = self.layer_sketches[level].estimate(&input) as i64; cumulative .entry(item.key.clone()) @@ -486,7 +404,7 @@ impl UnivMonTwoTier { self.layer_sketches[i].merge(&other.layer_sketches[i]); self.layer_heaps[i].clear(); for key in candidate_keys { - let input = heap_item_to_sketch_input(&key); + let input = key.as_data_input(); let count = self.layer_sketches[i].estimate(&input) as i64; self.layer_heaps[i].update(&input, count); } @@ -505,57 +423,6 @@ mod tests { use super::*; use crate::DataInput; - #[test] - fn pool_basic_take_put() { - let mut pool = UnivMonPool::new(2, 16, 2, 5, 2); - assert_eq!(pool.available(), 2); - assert_eq!(pool.total_allocated(), 2); - - let s0 = pool.take(); - assert_eq!(pool.available(), 1); - - let s1 = pool.take(); - assert_eq!(pool.available(), 0); - - // Pool is empty — next take allocates a new one - let s2 = pool.take(); - assert_eq!(pool.available(), 0); - assert_eq!(pool.total_allocated(), 3); - - // Return one - pool.put(s1); - assert_eq!(pool.available(), 1); - - // Should reuse the returned sketch - let s3 = pool.take(); - assert_eq!(pool.available(), 0); - assert_eq!(pool.total_allocated(), 3); // no new allocation - - // Return all - pool.put(s0); - pool.put(s2); - pool.put(s3); - assert_eq!(pool.available(), 3); - } - - #[test] - fn pool_free_resets_sketch() { - let mut pool = UnivMonPool::new(1, 16, 2, 5, 2); - - // Take a sketch, insert some data - let mut sketch = pool.take(); - sketch.update_weighted(&DataInput::I64(42), 100); - assert!(sketch.total_weight > 0); - - // Return it — should reset - pool.put(sketch); - - // Take it back — should be clean - let sketch2 = pool.take(); - assert_eq!(sketch2.total_weight, 0); - assert!((sketch2.layer_sketches[0].l2()).abs() < 1e-9); - } - // ======================================================================= // UnivMonTwoTier tests // ======================================================================= diff --git a/src/sketch_framework/univmon_two_tier/wire.rs b/src/sketch_framework/univmon_two_tier/wire.rs index 147688e0..986a5b42 100644 --- a/src/sketch_framework/univmon_two_tier/wire.rs +++ b/src/sketch_framework/univmon_two_tier/wire.rs @@ -15,7 +15,7 @@ //! and the mouse dimensions after, so every per-layer geometry is derived and //! none is stored. //! -//! [`UnivMonPool`](super::UnivMonPool) is a free-list of scratch +//! [`UnivMonPool`](crate::UnivMonPool) is a free-list of scratch //! `UnivMon`s rather than a sketch, and has no wire kind. use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; diff --git a/src/sketches/count_min_fold.rs b/src/sketches/count_min_fold.rs index 90fe66d5..fe627a1d 100644 --- a/src/sketches/count_min_fold.rs +++ b/src/sketches/count_min_fold.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; -use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap, heap_item_to_sketch_input}; +use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap}; use std::marker::PhantomData; const LOWER_32_MASK: u64 = (1u64 << 32) - 1; @@ -453,7 +453,7 @@ impl CountMinSketchFold { // Reconcile top-k heaps from both sources. for source in [a, b] { for item in source.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -500,7 +500,7 @@ impl CountMinSketchFold { // Reconcile heap. for item in self.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -544,7 +544,7 @@ impl CountMinSketchFold { // Reconcile heaps from all sources. for sk in sketches { for item in sk.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -591,7 +591,7 @@ impl CountMinSketchFold { /// Re-query all heap items from `other` against `self` and update our heap. fn reconcile_heap_from(&mut self, other: &CountMinSketchFold) { for item in other.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = self.estimate(&key_ref); self.heap.update(&key_ref, est); } diff --git a/src/sketches/count_sketch_fold.rs b/src/sketches/count_sketch_fold.rs index 5d28f8bc..57a5e723 100644 --- a/src/sketches/count_sketch_fold.rs +++ b/src/sketches/count_sketch_fold.rs @@ -17,10 +17,7 @@ use serde::{Deserialize, Serialize}; use crate::count_min_fold::FoldCell; -use crate::{ - DataInput, DefaultXxHasher, SketchHasher, TopKHeap, compute_median_inline_f64, - heap_item_to_sketch_input, -}; +use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap, compute_median_inline_f64}; use std::marker::PhantomData; const LOWER_32_MASK: u64 = (1u64 << 32) - 1; @@ -290,7 +287,7 @@ impl CountSketchFold { // Reconcile top-k heaps from both sources. for source in [a, b] { for item in source.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -337,7 +334,7 @@ impl CountSketchFold { // Reconcile heap. for item in self.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -381,7 +378,7 @@ impl CountSketchFold { // Reconcile heaps from all sources. for sk in sketches { for item in sk.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = result.estimate(&key_ref); result.heap.update(&key_ref, est); } @@ -425,7 +422,7 @@ impl CountSketchFold { /// Re-query all heap items from `other` against `self` and update our heap. fn reconcile_heap_from(&mut self, other: &CountSketchFold) { for item in other.heap.heap() { - let key_ref = heap_item_to_sketch_input(&item.key); + let key_ref = item.key.as_data_input(); let est = self.estimate(&key_ref); self.heap.update(&key_ref, est); } diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index bb2bb1fb..bc0b7183 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -12,7 +12,6 @@ use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::structure_utils::AdmittedRows; -use crate::input_to_owned; use crate::message_pack_format::admission_rate::{admit_all, deserialize_rate}; use crate::octo_delta::{ CountMinSketchCellDelta, KeyedCountMinSketchCellDelta, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, @@ -198,16 +197,14 @@ where { /// Creates a sketch with the requested number of rows and columns. pub fn with_dimensions(rows: usize, cols: usize) -> Self { - let mut sk = CountMinSketch { - counts: Vector2D::unfilled(rows, cols), + CountMinSketch { + counts: Vector2D::filled(rows, cols, T::default()), rows, cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), - }; - sk.counts.fill(T::default()); - sk + } } } @@ -443,7 +440,7 @@ impl, H: SketchHasher> CountMinSketch(heap: &TopKHeap, key: &DataInput) -> Option { heap.heap() .iter() - .find(|item| heap_item_to_sketch_input(&item.key) == *key) + .find(|item| item.key.as_data_input() == *key) .map(|item| item.count) } @@ -396,7 +396,7 @@ mod tests { fn top_k_heap_keys(heap: &TopKHeap) -> HashSet { heap.heap() .iter() - .map(|item| match heap_item_to_sketch_input(&item.key) { + .map(|item| match item.key.as_data_input() { DataInput::U64(v) => v, other => panic!("expected U64 key in zipf tests, got {other:?}"), }) @@ -461,7 +461,7 @@ mod tests { .heap() .iter() .find(|item| { - let k = heap_item_to_sketch_input(&item.key); + let k = item.key.as_data_input(); k == key }) .expect("key should be in heap"); @@ -722,7 +722,7 @@ mod tests { ]); for item in sh.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sh.estimate(&key)); } @@ -732,7 +732,7 @@ mod tests { sh.merge(&other); for item in sh.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sh.estimate(&key)); } } @@ -846,7 +846,7 @@ mod tests { .heap() .heap() .iter() - .filter(|item| heap_item_to_sketch_input(&item.key) == DataInput::Str("dup")) + .filter(|item| item.key.as_data_input() == DataInput::Str("dup")) .count(); assert_eq!(dup_entries, 1); } @@ -862,7 +862,7 @@ mod tests { assert!(sketch.heap().len() <= top_k); for item in sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sketch.estimate(&key)); } @@ -886,7 +886,7 @@ mod tests { assert!(sketch.heap().len() <= top_k); for item in sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sketch.estimate(&key)); } diff --git a/src/sketches/countminsketch_space_saving.rs b/src/sketches/countminsketch_space_saving.rs index 38f9bc93..e6a8d5fb 100644 --- a/src/sketches/countminsketch_space_saving.rs +++ b/src/sketches/countminsketch_space_saving.rs @@ -7,7 +7,7 @@ use std::cmp::Ordering; use crate::sketches::space_saving::parts::key_cmp; use crate::{ CountMinSketch, DataInput, DefaultXxHasher, MatrixStorage, OwnedDataInput, RegularPath, - SketchHasher, SpaceSaving, Vector2D, heap_item_to_sketch_input, + SketchHasher, SpaceSaving, Vector2D, }; pub(crate) mod summary_wire; @@ -79,7 +79,7 @@ pub(crate) fn rank_candidates( .entries() .into_iter() .map(|(key, _, _)| { - let count = estimate(&heap_item_to_sketch_input(&key)); + let count = estimate(&key.as_data_input()); (key, count) }) .collect(); @@ -246,7 +246,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{CountMinSketchWithHeap, FastPath, heap_item_to_sketch_input}; + use crate::{CountMinSketchWithHeap, FastPath}; fn u64_key(item: &OwnedDataInput) -> u64 { match item { @@ -408,7 +408,7 @@ mod tests { sk.update_weighted(&DataInput::U64(2), 1.5); assert_eq!(sk.space_saving().estimate(&DataInput::U64(1)), 1); let top = sk.top_k(2); - assert_eq!(heap_item_to_sketch_input(&top[0].0), DataInput::U64(2)); + assert_eq!(top[0].0.as_data_input(), DataInput::U64(2)); assert_eq!(top[0].1, 1.5); assert_eq!(top[1].1, 0.25); } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 6b944d29..2cbf536f 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -212,16 +212,14 @@ where { /// Creates a sketch with the requested number of rows and columns. pub fn with_dimensions(rows: usize, cols: usize) -> Self { - let mut sk = CountSketch { - counts: Vector2D::unfilled(rows, cols), + CountSketch { + counts: Vector2D::filled(rows, cols, T::from(0)), rows, cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), - }; - sk.counts.fill(T::from(0)); - sk + } } } @@ -563,7 +561,6 @@ impl NitroTarget for CountSketch, FastPath, H> { } } -use crate::input_to_owned; use crate::octo_delta::{ CountSketchCellDelta, KeyedCountSketchCellDelta, OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, OCTO_MAX_PROMOTION_THRESHOLD, @@ -621,7 +618,7 @@ impl, H: SketchHasher> CountSketch i64 { +pub(crate) fn heap_count_from_estimate(estimate: f64) -> i64 { // `as` on a float is a saturating cast in Rust (since 1.45); this is the // documented behaviour, not a fallback. estimate as i64 @@ -49,8 +49,8 @@ pub fn heap_count_from_estimate(estimate: f64) -> i64 { type HeapCountOf = <::Counter as HeapCounter>::HeapCount; /// A Count Sketch counter type a `CountSketchWithHeap` keeps a heap beside: the heap count -/// of a median estimate is [`heap_count_from_estimate`]'s for `i32`, `i64` and `i128`, and -/// the unrounded median for `f64`. +/// of a median estimate is the median cast to `i64` (saturating, truncating toward zero) +/// for `i32`, `i64` and `i128`, and the unrounded median for `f64`. pub trait CountSketchHeapCounter: CountSketchCounter + HeapCounter { /// The heap count of a median `estimate`. fn heap_count(estimate: f64) -> Self::HeapCount; @@ -324,7 +324,7 @@ where } self.heap.clear(); for key in candidate_keys { - let key_ref = heap_item_to_sketch_input(&key); + let key_ref = key.as_data_input(); let est = self.count_sketch.estimate(&key_ref); self.heap.update(&key_ref, S::Counter::heap_count(est)); } @@ -379,7 +379,7 @@ where } self.heap.clear(); for key in candidate_keys { - let key_ref = heap_item_to_sketch_input(&key); + let key_ref = key.as_data_input(); let est = self.count_sketch.estimate(&key_ref); self.heap.update(&key_ref, S::Counter::heap_count(est)); } @@ -396,7 +396,7 @@ mod tests { fn heap_count_for_key(heap: &TopKHeap, key: &DataInput) -> Option { heap.heap() .iter() - .find(|item| heap_item_to_sketch_input(&item.key) == *key) + .find(|item| item.key.as_data_input() == *key) .map(|item| item.count) } @@ -454,7 +454,7 @@ mod tests { fn top_k_heap_keys(heap: &TopKHeap) -> HashSet { heap.heap() .iter() - .map(|item| match heap_item_to_sketch_input(&item.key) { + .map(|item| match item.key.as_data_input() { DataInput::U64(v) => v, other => panic!("expected U64 key in zipf tests, got {other:?}"), }) @@ -519,7 +519,7 @@ mod tests { .heap() .iter() .find(|item| { - let k = heap_item_to_sketch_input(&item.key); + let k = item.key.as_data_input(); k == key }) .expect("key should be in heap"); @@ -802,7 +802,7 @@ mod tests { ]); for item in sh.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sh.estimate(&key) as i64); } @@ -812,7 +812,7 @@ mod tests { sh.merge(&other); for item in sh.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sh.estimate(&key) as i64); } } @@ -926,7 +926,7 @@ mod tests { .heap() .heap() .iter() - .filter(|item| heap_item_to_sketch_input(&item.key) == DataInput::Str("dup")) + .filter(|item| item.key.as_data_input() == DataInput::Str("dup")) .count(); assert_eq!(dup_entries, 1); } @@ -942,7 +942,7 @@ mod tests { assert!(sketch.heap().len() <= top_k); for item in sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sketch.estimate(&key) as i64); } @@ -966,7 +966,7 @@ mod tests { assert!(sketch.heap().len() <= top_k); for item in sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); assert_eq!(item.count, sketch.estimate(&key) as i64); } diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs index 1e9e071a..0d4ee7bb 100644 --- a/src/sketches/countsketch_l2_norm.rs +++ b/src/sketches/countsketch_l2_norm.rs @@ -89,16 +89,14 @@ impl CountSketchWithL2Norm { if let Err(problem) = CountSketchWithL2Norm::check_hash_bits(rows, cols) { panic!("{problem}"); } - let mut sk = CountSketchWithL2Norm { - counts: Vector2D::unfilled(rows, cols), + CountSketchWithL2Norm { + counts: Vector2D::filled(rows, cols, 0), row_l2_squared: Vector1D::filled(rows, 0), rows, cols, seed_index, _hasher: PhantomData, - }; - sk.counts.fill(0); - sk + } } /// Number of rows in the sketch. diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index eba87651..56916afc 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -27,6 +27,8 @@ const MAX_LEVELS: usize = 61; const CAPACITY_CACHE_LEN: usize = 20; pub(crate) const MAX_K: usize = 26_602; +/// Minimum level capacity `m` that `with_k` and `with_k_and_seed` use. +pub const DEFAULT_M: usize = 8; const CAPACITY_DECAY: f64 = 2.0 / 3.0; const DEFAULT_K: i32 = 200; @@ -287,14 +289,14 @@ impl Kll { Self::init_internal(k, m, Coin::from_seed(seed), Some(seed)) } - /// Creates a new KLL sketch with the given `k` and a default minimum level capacity of 8. + /// Creates a new KLL sketch with the given `k` and `m` = [`DEFAULT_M`]. pub fn with_k(k: i32) -> Self { - Self::new(k as usize, 8) + Self::new(k as usize, DEFAULT_M) } /// `with_k` with an explicit RNG seed. See `with_seed`. pub fn with_k_and_seed(k: i32, seed: u64) -> Self { - Self::with_seed(k as usize, 8, seed) + Self::with_seed(k as usize, DEFAULT_M, seed) } fn init_internal(k: usize, m: usize, coin: Coin, seed: Option) -> Self { diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 68cd42d2..61990951 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -18,7 +18,7 @@ use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; use crate::{DataInput, Vector1D}; -use super::kll::{Coin, MAX_K, merge_sorted_runs, randomly_halve_up}; +use super::kll::{Coin, DEFAULT_M, MAX_K, merge_sorted_runs, randomly_halve_up}; mod wire; @@ -113,14 +113,14 @@ impl KllDynamic { s } - /// Creates a KLL dynamic sketch with default `m` and the provided `k`. + /// Creates a KLL dynamic sketch with the given `k` and `m` = [`DEFAULT_M`]. pub fn with_k(k: i32) -> Self { - Self::new(k as usize, 8) + Self::new(k as usize, DEFAULT_M) } /// `with_k` with an explicit RNG seed. See [`KllDynamic::with_seed`]. pub fn with_k_and_seed(k: i32, seed: u64) -> Self { - Self::with_seed(k as usize, 8, seed) + Self::with_seed(k as usize, DEFAULT_M, seed) } fn push_value(&mut self, value: T) { diff --git a/src/sketches/mod.rs b/src/sketches/mod.rs index 1c1b8d14..ddb6a47b 100644 --- a/src/sketches/mod.rs +++ b/src/sketches/mod.rs @@ -110,9 +110,7 @@ pub mod countminsketch_heap; pub use countminsketch_heap::{CountMinSketchWithHeap, CountMinSketchWithHeapDelta}; pub mod countsketch_heap; -pub use countsketch_heap::{ - CountSketchHeapCounter, CountSketchWithHeap, CountSketchWithHeapDelta, heap_count_from_estimate, -}; +pub use countsketch_heap::{CountSketchHeapCounter, CountSketchWithHeap, CountSketchWithHeapDelta}; pub mod countsketch_l2_norm; pub use countsketch_l2_norm::CountSketchWithL2Norm; diff --git a/src/sketches/space_saving.rs b/src/sketches/space_saving.rs index f40a9143..c24a816c 100644 --- a/src/sketches/space_saving.rs +++ b/src/sketches/space_saving.rs @@ -27,7 +27,7 @@ //! ICDT 2005. use crate::common::DigestBuildHasher; -use crate::{DataInput, DefaultXxHasher, OwnedDataInput, SketchHasher, input_to_owned}; +use crate::{DataInput, DefaultXxHasher, OwnedDataInput, SketchHasher}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use smallvec::SmallVec; use std::collections::HashMap; @@ -188,7 +188,7 @@ impl SpaceSaving { if self.monitored.len() < self.capacity { let seated = self.discarded_max.saturating_add(weight); let discarded_max = self.discarded_max; - self.seat(digest, input_to_owned(value), seated, discarded_max); + self.seat(digest, value.to_owned_input(), seated, discarded_max); return; } @@ -201,7 +201,7 @@ impl SpaceSaving { ); self.discarded_max = lowest; self.unindex(self.monitored[victim].digest, victim); - self.monitored[victim].key = input_to_owned(value); + self.monitored[victim].key = value.to_owned_input(); self.monitored[victim].digest = digest; self.monitored[victim].error = lowest; self.index.entry(digest).or_default().push(victim); diff --git a/tests/e2e/matrix_instances.rs b/tests/e2e/matrix_instances.rs index 85343943..66f07fdc 100644 --- a/tests/e2e/matrix_instances.rs +++ b/tests/e2e/matrix_instances.rs @@ -39,7 +39,7 @@ use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, DataInput, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, FastPath, FixedMatrix, OwnedDataInput, QuickMatrixI64, QuickMatrixI128, RegularPath, Vector2D, hash_for_matrix, - heap_count_from_estimate, + sketches::CountSketchHeapCounter, }; /// Medium stream: large enough that collisions are statistically meaningful at @@ -533,7 +533,7 @@ macro_rules! csheap_instance { single.heap().heap(), $truth, |k| single.estimate(&DataInput::U64(k as u64)), - heap_count_from_estimate, + ::heap_count, ); }}; } @@ -543,7 +543,7 @@ macro_rules! csheap_instance { /// /// `count_of` maps the sketch's estimate to the integer a heap entry holds. It /// is the identity for `CountMinSketchWithHeap`, whose estimate is already the counter type, -/// and `heap_count_from_estimate` for `CountSketchWithHeap`, whose estimate is an `f64` median and is +/// and `CountSketchHeapCounter::heap_count` for `CountSketchWithHeap`, whose estimate is an `f64` median and is /// documented to reach the heap by a saturating, truncating conversion. fn assert_heap_matches_sketch( label: &str, @@ -673,12 +673,12 @@ heap_matrix_test!( /// `CountSketchWithHeap` over an `i128` counter really does accept counts past `i64::MAX`, /// and this pins what happens to the heap entry when it does. /// -/// The insert path is `heap.update(key, heap_count_from_estimate(estimate))`, where the +/// The insert path is `heap.update(key, CountSketchHeapCounter::heap_count(estimate))`, where the /// estimate is the row **median** as `f64`. Two lossy steps are unavoidable and /// both are documented API rather than accidents: /// /// 1. `i128 -> f64` inside `CountSketch::estimate`, exact only below `2^53`; -/// 2. `f64 -> i64` in `heap_count_from_estimate`, which **saturates** at `i64::MAX`. +/// 2. `f64 -> i64` in `CountSketchHeapCounter::heap_count`, which **saturates** at `i64::MAX`. /// /// Saturating is the chosen semantics. Wrapping would turn a huge positive /// count into a negative one and corrupt the heap's ordering, which is the one @@ -726,9 +726,9 @@ fn csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { "{label}: the sketch itself must still hold the i128 mass, got {est}" ); assert_eq!( - heap_count_from_estimate(est), + ::heap_count(est), i64::MAX, - "{label}: heap_count_from_estimate must saturate at i64::MAX" + "{label}: CountSketchHeapCounter::heap_count must saturate at i64::MAX" ); assert_eq!( heap_count(huge.heap().heap(), &key), @@ -784,7 +784,7 @@ fn csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { fn heap_count(items: &[asap_sketchlib::HeapEntry], key: &DataInput) -> Option { items .iter() - .find(|it| asap_sketchlib::heap_item_to_sketch_input(&it.key) == *key) + .find(|it| it.key.as_data_input() == *key) .map(|it| it.count) } diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index 3079ee1c..cf26c390 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -40,8 +40,7 @@ use asap_sketchlib::{ OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, OCTO_HLL_DEFAULT_THRESHOLD, OctoAggregator, OctoPlan, OctoThreshold, OctoWorker, RegularPath, UnivMon, UnivMonCellDelta, UnivMonDeltaFidelity, UnivMonOctoAggregator, UnivMonOctoPlan, - UnivMonOctoWorker, Vector2D, hash64_seeded, hash128_seeded, input_to_owned, - univmon_layer_threshold, + UnivMonOctoWorker, Vector2D, hash64_seeded, hash128_seeded, univmon_layer_threshold, }; use common::{FreqTruth, zipf_u64}; use rand::SeedableRng; @@ -2765,7 +2764,7 @@ fn univmon_through_deltas( hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, &input), layers, ); - let owned = input_to_owned(&input); + let owned = input.to_owned_input(); for (layer, worker) in workers.iter_mut().enumerate().take(bottom + 1) { let hashed = hash128_seeded(layer, &input); let mut promoted = Vec::new(); @@ -4749,7 +4748,7 @@ mod top_k_plans { for d in promoted { assert_eq!( d.key, - asap_sketchlib::input_to_owned(&cold), + cold.to_owned_input(), "every top-k delta must carry the flow key that produced it" ); parent.apply(d); diff --git a/tests/pbt/octo_delta.rs b/tests/pbt/octo_delta.rs index fee04de4..b435f8ab 100644 --- a/tests/pbt/octo_delta.rs +++ b/tests/pbt/octo_delta.rs @@ -30,8 +30,8 @@ use asap_sketchlib::{ OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD, OCTO_DDSKETCH_DEFAULT_THRESHOLD, OCTO_ELASTIC_DEFAULT_THRESHOLD, OCTO_HLL_DEFAULT_THRESHOLD, OCTO_MAX_PROMOTION_THRESHOLD, OCTO_UNIVMON_DEFAULT_THRESHOLD, OctoAggregator, OctoPlan, OctoThreshold, OctoWorker, - RegularPath, UnivMonCellDelta, UnivMonOctoPlan, Vector2D, heap_item_to_sketch_input, - input_to_owned, max_hll_threshold, univmon_layer_threshold, + RegularPath, UnivMonCellDelta, UnivMonOctoPlan, Vector2D, max_hll_threshold, + univmon_layer_threshold, }; use proptest::prelude::*; @@ -891,7 +891,7 @@ proptest! { d.total_weight, *i as u64 + 1, "insert {} reported a running weight of {}", i, d.total_weight ); - prop_assert_eq!(&d.key, &input_to_owned(&DataInput::U64(keys[*i]))); + prop_assert_eq!(&d.key, &DataInput::U64(keys[*i]).to_owned_input()); } } @@ -933,7 +933,7 @@ proptest! { parent.sketch.heap().len(), promoted.len(), top_k ); for item in parent.sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); let estimate = i64::from(parent.sketch.count_min_sketch().estimate(&key)); prop_assert!( matches!(key, DataInput::U64(k) if seen.contains(&k)), @@ -961,7 +961,7 @@ proptest! { for k in &keys { let input = DataInput::U64(*k); let hashes = CountMinSketchWorker::hashes(rows, &input); - let key = input_to_owned(&input); + let key = input.to_owned_input(); worker.update_emit_keyed_delta_with_row_hashes(&hashes, &key, tau, &mut |d| { deltas.push(d.clone()); promoted.insert(*k); @@ -988,7 +988,7 @@ proptest! { parent.sketch.heap().len(), promoted.len(), top_k ); for item in parent.sketch.heap().heap() { - let key = heap_item_to_sketch_input(&item.key); + let key = item.key.as_data_input(); prop_assert!( matches!(key, DataInput::U64(k) if seen.contains(&k)), "the heap holds {:?}, which never arrived", item.key diff --git a/tests/pbt/topk_wrappers.rs b/tests/pbt/topk_wrappers.rs index 56266a48..9d306ecc 100644 --- a/tests/pbt/topk_wrappers.rs +++ b/tests/pbt/topk_wrappers.rs @@ -9,7 +9,7 @@ use crate::support::{PROBE_KEYS, edge_dimension, edge_rows, grid, keys}; use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, CountSketch, CountSketchWithHeap, CountSketchWithL2Norm, DataInput, DefaultXxHasher, FastPath, RegularPath, TopKHeap, Vector2D, - heap_count_from_estimate, heap_item_to_sketch_input, + sketches::CountSketchHeapCounter, }; use proptest::prelude::*; use std::collections::{HashMap, HashSet}; @@ -271,7 +271,7 @@ proptest! { for k in &stream { let probe = DataInput::U64(*k); wrapped.update(&probe); - read_at_last_insert.insert(*k, heap_count_from_estimate(wrapped.estimate(&probe))); + read_at_last_insert.insert(*k, ::heap_count(wrapped.estimate(&probe))); } for (key, expected) in read_at_last_insert { @@ -410,7 +410,7 @@ proptest! { cs.merge(&cs_other); for item in cms.heap().heap() { - let probe = heap_item_to_sketch_input(&item.key); + let probe = item.key.as_data_input(); prop_assert_eq!( item.count, i64::from(cms.estimate(&probe)), @@ -418,10 +418,10 @@ proptest! { ); } for item in cs.heap().heap() { - let probe = heap_item_to_sketch_input(&item.key); + let probe = item.key.as_data_input(); prop_assert_eq!( item.count, - heap_count_from_estimate(cs.estimate(&probe)), + ::heap_count(cs.estimate(&probe)), "CS key {:?}", item.key ); } From e74df078757bf9d730cb0e18614adecfbf13aa45 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 05:44:27 +0000 Subject: [PATCH 08/11] docs: the doc fixes #209 lists FastPath/RegularPath, MatrixFastHash and FastPathHasher describe the hashing they select; UnivMon and UnivMonTwoTier total_weight is the total inserted weight; UnivMon entropy is in bits and UnivMon-Q entropy in nats; spec rule 3 of 3.15/3.16 bounds layers by 19, as UNIVMON_MAX_LAYERS does. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ docs/api/api_univmon.md | 3 ++- docs/api/api_univmon_optimized.md | 2 +- docs/api/api_univmon_q.md | 2 ++ docs/asapv1_wire_format.md | 6 +++--- src/common/structures/matrix_storage.rs | 6 ++++-- src/sketch_framework/eh_univmon.rs | 1 + src/sketch_framework/univmon.rs | 4 ++-- src/sketch_framework/univmon_two_tier.rs | 4 ++-- src/sketches/mode.rs | 9 +++++++-- 10 files changed, 30 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73fee356..e728db8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -321,6 +321,12 @@ signals a backwards-compatible change. `with_k_and_seed` use. `Vector2D::filled(rows, cols, value)` builds a filled matrix in one call. `Vector2D::nitro_skip` takes `&self`. +- Docs state what the code does: `FastPath` / `RegularPath`, `MatrixFastHash` + and `FastPathHasher` describe the hashing they select; UnivMon's + `total_weight` is the total inserted weight; UnivMon's `estimate_entropy` is + in bits and UnivMon-Q's in nats. Spec §3.15 / §3.16 rule 3 bounds `layers` + by 19 (`UNIVMON_MAX_LAYERS`), the seed-table ceiling the code enforces. + ### Removed - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); diff --git a/docs/api/api_univmon.md b/docs/api/api_univmon.md index 1aa9943b..5b29cbc4 100644 --- a/docs/api/api_univmon.md +++ b/docs/api/api_univmon.md @@ -63,7 +63,8 @@ fn estimate_g_sum(&self, g: F, heavy_only: bool) -> f64 ``` Because UnivMon accepts only non-negative updates, `estimate_l1()` returns the -exact tracked total weight. The generic recurrence remains available through +exact tracked total weight. `estimate_entropy()` is Shannon entropy in bits +(log2); UnivMon-Q's is in nats. The generic recurrence remains available through `estimate_g_sum` for compatible frequency functions. ## Merge diff --git a/docs/api/api_univmon_optimized.md b/docs/api/api_univmon_optimized.md index 4f5d7ec6..d2aeafe4 100644 --- a/docs/api/api_univmon_optimized.md +++ b/docs/api/api_univmon_optimized.md @@ -45,7 +45,7 @@ fn estimate_g_sum(&self, g: F, heavy_only: bool) -> f64 For the supported non-negative update stream, `estimate_l1()` returns the exact tracked total weight rather than estimating the linear sum through the -hierarchy. +hierarchy. `estimate_entropy()` is Shannon entropy in bits (log2). ## Merge diff --git a/docs/api/api_univmon_q.md b/docs/api/api_univmon_q.md index c01725aa..b09d0c7e 100644 --- a/docs/api/api_univmon_q.md +++ b/docs/api/api_univmon_q.md @@ -108,6 +108,8 @@ original UnivMon recurrence and the coordinated occurrence-sample identity `H = E[ln(N/f_X)]`. The recurrence is used for diffuse streams and complete candidate recovery; concentrated streams with incomplete recovery use the occurrence path. The two component estimators remain exposed for experiments. +All three entropy estimates are in nats (ln); UnivMon's `estimate_entropy` is in +bits. `prepare_queries` reconstructs candidate frequencies, logical sampled levels, F2 thresholds, and the ordered CDF once. Its returned immutable view exposes diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index ce84928e..7a707dd8 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -1055,7 +1055,7 @@ UnivMon carries the hash-spec group — it hashes, and a consumer must reproduce 1. `kind_id` is `0x10 0x00`; any other id is rejected. 2. The hash-spec group matches `DefaultXxHasher`'s `HashProfile`. `layers`, `rows`, `cols`, `heap_size` and `key_type` are properties of the stored sketch, so they are echoed back rather than pinned. -3. `layers`, `rows`, `cols` and `heap_size` are **all non-zero**, and `layers <= 64` (`UNIVMON_MAX_LAYERS`). All of it is checked **before** the per-layer geometry vector is built, so it is the first thing standing between a declared shape and an allocation. The ceiling is a hash-width bound, and `bottom_layer_for_hash` is where it comes from: the layer finder tests `(hash >> l) & 1` for `l` in `1 .. layers`, and the query recurrences shift the same way, so a 64-layer pyramid shifts a 64-bit key hash by at most 63 and a deeper one would shift past the hash's width. The ceiling is checked on **both** doors; the four non-zero checks are the decode door's, since only a decoder is handed a shape it did not build. +3. `layers`, `rows`, `cols` and `heap_size` are **all non-zero**, and `layers <= 19` (`UNIVMON_MAX_LAYERS`). All of it is checked **before** the per-layer geometry vector is built, so it is the first thing standing between a declared shape and an allocation. The ceiling is a seed bound: layer `i` hashes at seed index `i`, and the layer finder hashes at `UNIVMON_BOTTOM_LAYER_SEED_INDEX` (19), so a 20th layer would share the finder's seed and later ones would wrap onto layer 0's. The ceiling is checked on **both** doors; the four non-zero checks are the decode door's, since only a decoder is handed a shape it did not build. 4. `key_type` is one of §3.5's fourteen names; the `keys` array is read **as** that type, with §3.5 rule 3's relabel limits; and `keys` and `heap_counts` have equal length. A pyramid whose heaps hold no key requires `key_type` `"u64"`. 5. `rows * layers == len(l2)` exactly, with checked arithmetic, and checked **before** the layer geometry is built. Together with rule 3's ceiling this is what makes a declared `layers` far larger than the payload carries cost nothing: `rows >= 1`, so the equality caps `layers` at `len(l2)`, and the ceiling caps it at 64 regardless. 6. `len(heap_lens) == len(candidate_complete) == layers`; every layer's `rows` and `cols` are non-zero and `rows <= 20` (`MATRIX_MAX_ROWS`, §3.2 — a layer is a CountSketchWithL2Norm matrix; `cols.ilog2()` panics on zero) and its geometry fits the 128-bit key hash, `(ceil_log2(cols) + 1) * rows <= 128` (§3.19); the layers' total cell count and total accumulator count are computed with checked arithmetic and must equal `len(counts)` and `len(l2)`; and `len(keys) == len(heap_counts) == sum(heap_lens)`. @@ -1092,11 +1092,11 @@ The encode side enforces the same shape rules — a `layers` past 64, a layer th 1. `kind_id` is `0x11 0x00`; any other id is rejected. 2. The hash-spec group matches `DefaultXxHasher`'s `HashProfile`; the seven layout fields and `key_type` are echoed back rather than pinned. -3. `layers`, `heap_size` and **all four** tier dimensions — `elephant_rows`, `elephant_cols`, `mouse_rows`, `mouse_cols` — are non-zero, and `layers <= 64` (`UNIVMON_MAX_LAYERS`, §3.15's rule 3: the same hash-width ceiling, split across the doors the same way — the ceiling on both, the non-zero checks on the decode door). All of it is checked before the per-layer geometry is built. **A tier no layer materializes must still declare non-zero dimensions.** An all-elephant pyramid carries a `mouse_rows` / `mouse_cols` pair on the wire all the same (see "One state, more than one encoding"), and a pyramid decoded with a zero pair panics the first time it is merged against a normally built one, whose constructor asserts all four are positive. `elephant_layers` carries no bound of its own; it only ever reaches the layout through `min(elephant_layers, layers)`. +3. `layers`, `heap_size` and **all four** tier dimensions — `elephant_rows`, `elephant_cols`, `mouse_rows`, `mouse_cols` — are non-zero, and `layers <= 19` (`UNIVMON_MAX_LAYERS`, §3.15's rule 3: the same seed ceiling, split across the doors the same way — the ceiling on both, the non-zero checks on the decode door). All of it is checked before the per-layer geometry is built. **A tier no layer materializes must still declare non-zero dimensions.** An all-elephant pyramid carries a `mouse_rows` / `mouse_cols` pair on the wire all the same (see "One state, more than one encoding"), and a pyramid decoded with a zero pair panics the first time it is merged against a normally built one, whose constructor asserts all four are positive. `elephant_layers` carries no bound of its own; it only ever reaches the layout through `min(elephant_layers, layers)`. 4. The declared layout's accumulator count — `min(elephant_layers, layers) * elephant_rows + max(0, layers - elephant_layers) * mouse_rows` — equals `len(l2)` exactly, computed with checked arithmetic and checked **before** the per-layer geometry is built. With rule 3's ceiling above it, that is what makes a crafted `layers` or tier width cost nothing. 5. §3.15's rules 4, 6, 7, 8, 9, 10 and 11 apply verbatim over the derived per-layer geometry — including the per-layer `l2[i] >= 0` check and the per-layer 128-bit hash-budget check (§3.19). -The encode side enforces §3.15's shape rules and rule 7 over the two-tier layout, the `layers <= 64` ceiling included. +The encode side enforces §3.15's shape rules and rule 7 over the two-tier layout, the `layers <= 19` ceiling included. **`UnivMonPool` is not a wire kind.** It is a `Vec` free list of pre-allocated scratch `UnivMon`s — `0x10 0x00`'s type, not this section's — beside a `total_allocated` counter: an allocator, not a sketch, with nothing an observer could query. `0x11 0x00` is `UnivMonTwoTier`'s alone. diff --git a/src/common/structures/matrix_storage.rs b/src/common/structures/matrix_storage.rs index b6c15df8..50ea73f2 100644 --- a/src/common/structures/matrix_storage.rs +++ b/src/common/structures/matrix_storage.rs @@ -57,7 +57,8 @@ impl MatrixHash { } } -/// Trait for hash values that support fast row/column decoding. +/// A key's [`FastPath`](crate::FastPath) matrix hash, from which each row reads +/// its column and Count Sketch sign. pub trait MatrixFastHash: Clone { /// Verifies that the hash type can encode the given dimensions. fn assert_compatible(rows: usize, cols: usize); @@ -236,7 +237,8 @@ pub trait MatrixStorage { fn query_one_counter(&self, row: usize, col: usize) -> Self::Counter; } -/// Trait for storages that can derive fast-path hashes for their dimensions. +/// A matrix storage that builds, with hasher `H`, the [`FastPath`](crate::FastPath) +/// matrix hash its own `rows` x `cols` geometry needs. pub trait FastPathHasher: MatrixStorage where H: crate::SketchHasher, diff --git a/src/sketch_framework/eh_univmon.rs b/src/sketch_framework/eh_univmon.rs index d8a82624..925b0054 100644 --- a/src/sketch_framework/eh_univmon.rs +++ b/src/sketch_framework/eh_univmon.rs @@ -88,6 +88,7 @@ impl EhUnivMonQueryResult { } } + /// Shannon entropy of the interval, in bits (log2). pub fn estimate_entropy(&self) -> f64 { match self { Self::Sketch(um) => um.estimate_entropy(), diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 37de88e2..3653ef45 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -63,7 +63,7 @@ pub struct UnivMon { pub cols: usize, /// Heap capacity per layer. pub heap_size: usize, - /// Bucket size used for hashing decisions. + /// Total inserted weight (the exact L1 norm). pub total_weight: usize, #[serde(default)] update_mode: UnivMonUpdateMode, @@ -455,7 +455,7 @@ impl UnivMon { tmp.sqrt() } - /// Returns the estimated entropy. + /// Returns the estimated Shannon entropy, in bits (log2). pub fn estimate_entropy(&self) -> f64 { if self.total_weight == 0 { return 0.0; diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index 03eb8e61..37fe6f47 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -52,7 +52,7 @@ pub struct UnivMonTwoTier { pub mouse_cols: usize, /// Heap capacity per layer. pub heap_size: usize, - /// Bucket size used for hashing decisions. + /// Total inserted weight (the exact L1 norm). pub total_weight: usize, update_mode: UnivMonUpdateMode, candidate_complete: Vec, @@ -330,7 +330,7 @@ impl UnivMonTwoTier { self.estimate_g_sum(|x| x * x, false).sqrt() } - /// Returns the estimated entropy. + /// Returns the estimated Shannon entropy, in bits (log2). pub fn estimate_entropy(&self) -> f64 { if self.total_weight == 0 { return 0.0; diff --git a/src/sketches/mode.rs b/src/sketches/mode.rs index af1f3e7b..22078b3b 100644 --- a/src/sketches/mode.rs +++ b/src/sketches/mode.rs @@ -1,9 +1,14 @@ use serde::{Deserialize, Serialize}; -/// Standard row-by-row hashing mode for matrix-backed sketches. +/// Hashing mode of a matrix-backed sketch in which row `r` hashes the key on +/// its own, at seed-table index `r`. ASAPv1 `mode` `"regular"`. #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct RegularPath; -/// Packed fast-path hashing mode for matrix-backed sketches. +/// Hashing mode of a matrix-backed sketch in which the key is hashed once, at +/// seed-table index 0, and every row reads its column (and Count Sketch sign) +/// from that one matrix hash. The hash is one packed 64- or 128-bit value when +/// the rows' bits fit, else one hash per row (`MatrixHashLayout`). ASAPv1 +/// `mode` `"fast"`. #[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] pub struct FastPath; From 05e65b089f03526f30a7f2ffc4549fe305cf94c3 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:02:24 +0000 Subject: [PATCH 09/11] refactor: file names follow the #209 names docs/api pages take their type's name in snake_case (api_count_min_sketch, api_count_min_sketch_with_heap, api_eh_bucket_sketch, api_exponential_histogram_univmon, api_univmon_two_tier, ...); hhheap_acceleration.md and the hhheap_probe example become topk_heap_*; message_pack_format::native::countsketch_topk becomes countsketch_l2_norm. Co-Authored-By: Claude Opus 5.5 --- docs/advanced_use_cases.md | 2 +- docs/api/{api_coco.md => api_coco_sketch.md} | 2 +- docs/api/api_common_heap.md | 2 +- ...pi_countmin.md => api_count_min_sketch.md} | 0 ...p.md => api_count_min_sketch_with_heap.md} | 0 ...api_count_min_sketch_with_space_saving.md} | 2 +- docs/api/api_count_sketch.md | 2 +- ..._heap.md => api_count_sketch_with_heap.md} | 0 ... => api_count_sketch_with_space_saving.md} | 4 ++-- ...hsketchlist.md => api_eh_bucket_sketch.md} | 0 .../{api_elastic.md => api_elastic_sketch.md} | 2 +- ...d => api_exponential_histogram_univmon.md} | 0 ...n_optimized.md => api_univmon_two_tier.md} | 0 docs/apis.md | 20 +++++++++---------- docs/examples.md | 2 +- docs/index.md | 2 +- ...eleration.md => topk_heap_acceleration.md} | 2 +- .../{hhheap_probe.rs => topk_heap_probe.rs} | 2 +- ...tsketch_topk.rs => countsketch_l2_norm.rs} | 0 src/message_pack_format/native/mod.rs | 2 +- tests/bug_verification.rs | 2 +- tests/e2e/heavy_hitters.rs | 2 +- 22 files changed, 25 insertions(+), 25 deletions(-) rename docs/api/{api_coco.md => api_coco_sketch.md} (99%) rename docs/api/{api_countmin.md => api_count_min_sketch.md} (100%) rename docs/api/{api_cms_heap.md => api_count_min_sketch_with_heap.md} (100%) rename docs/api/{api_cms_space_saving.md => api_count_min_sketch_with_space_saving.md} (95%) rename docs/api/{api_cs_heap.md => api_count_sketch_with_heap.md} (100%) rename docs/api/{api_cs_space_saving.md => api_count_sketch_with_space_saving.md} (92%) rename docs/api/{api_ehsketchlist.md => api_eh_bucket_sketch.md} (100%) rename docs/api/{api_elastic.md => api_elastic_sketch.md} (99%) rename docs/api/{api_ehunivoptimized.md => api_exponential_histogram_univmon.md} (100%) rename docs/api/{api_univmon_optimized.md => api_univmon_two_tier.md} (100%) rename docs/{hhheap_acceleration.md => topk_heap_acceleration.md} (98%) rename examples/{hhheap_probe.rs => topk_heap_probe.rs} (99%) rename src/message_pack_format/native/{countsketch_topk.rs => countsketch_l2_norm.rs} (100%) diff --git a/docs/advanced_use_cases.md b/docs/advanced_use_cases.md index 10e2c1f1..da196f33 100644 --- a/docs/advanced_use_cases.md +++ b/docs/advanced_use_cases.md @@ -86,4 +86,4 @@ sketch answers, while `MicroCm` answers frequency only. `MicroCm` is | Sliding window, frequency only, tight memory | `MicroCm` (Experimental) | | Sliding window + universal monitoring | `ExponentialHistogramUnivMon` (Unstable) | -API references: [`docs/api/api_exponential_histogram.md`](./api/api_exponential_histogram.md), [`docs/api/api_ehsketchlist.md`](./api/api_ehsketchlist.md), [`docs/api/api_microscope.md`](./api/api_microscope.md) +API references: [`docs/api/api_exponential_histogram.md`](./api/api_exponential_histogram.md), [`docs/api/api_eh_bucket_sketch.md`](./api/api_eh_bucket_sketch.md), [`docs/api/api_microscope.md`](./api/api_microscope.md) diff --git a/docs/api/api_coco.md b/docs/api/api_coco_sketch.md similarity index 99% rename from docs/api/api_coco.md rename to docs/api/api_coco_sketch.md index 4015fbe0..c31b51d5 100644 --- a/docs/api/api_coco.md +++ b/docs/api/api_coco_sketch.md @@ -1,4 +1,4 @@ -# API: Coco +# API: CocoSketch Status: `Ready` diff --git a/docs/api/api_common_heap.md b/docs/api/api_common_heap.md index b796e715..58f3eefd 100644 --- a/docs/api/api_common_heap.md +++ b/docs/api/api_common_heap.md @@ -18,7 +18,7 @@ A capacity-bounded min-heap of `HeapEntry` beside an index from key digest to heap position, so a keyed update is a lookup and a sift rather than a scan. The index is patched through each sift: `update` costs one hash, one map probe and `O(log k)` index writes, and its rate does not fall as `k` grows. See -[TopKHeap acceleration](../hhheap_acceleration.md) for the measurements. +[TopKHeap acceleration](../topk_heap_acceleration.md) for the measurements. ## Constructors diff --git a/docs/api/api_countmin.md b/docs/api/api_count_min_sketch.md similarity index 100% rename from docs/api/api_countmin.md rename to docs/api/api_count_min_sketch.md diff --git a/docs/api/api_cms_heap.md b/docs/api/api_count_min_sketch_with_heap.md similarity index 100% rename from docs/api/api_cms_heap.md rename to docs/api/api_count_min_sketch_with_heap.md diff --git a/docs/api/api_cms_space_saving.md b/docs/api/api_count_min_sketch_with_space_saving.md similarity index 95% rename from docs/api/api_cms_space_saving.md rename to docs/api/api_count_min_sketch_with_space_saving.md index c5545110..703c3b4e 100644 --- a/docs/api/api_cms_space_saving.md +++ b/docs/api/api_count_min_sketch_with_space_saving.md @@ -9,7 +9,7 @@ Every update reaches both; a top-k query ranks the summary's monitored keys by their Count-Min estimate. The summary picks the candidates and the matrix gives each its frequency. The summary's `capacity` is independent of `k`. -[`CountMinSketchWithHeap`](./api_cms_heap.md) keeps an estimate heap instead: a key's count is +[`CountMinSketchWithHeap`](./api_count_min_sketch_with_heap.md) keeps an estimate heap instead: a key's count is its matrix estimate and a newcomer enters only by beating the heap minimum. ## Type/Struct diff --git a/docs/api/api_count_sketch.md b/docs/api/api_count_sketch.md index 55c52d12..c7e992d6 100644 --- a/docs/api/api_count_sketch.md +++ b/docs/api/api_count_sketch.md @@ -1,4 +1,4 @@ -# API: Count +# API: CountSketch Status: `Ready` diff --git a/docs/api/api_cs_heap.md b/docs/api/api_count_sketch_with_heap.md similarity index 100% rename from docs/api/api_cs_heap.md rename to docs/api/api_count_sketch_with_heap.md diff --git a/docs/api/api_cs_space_saving.md b/docs/api/api_count_sketch_with_space_saving.md similarity index 92% rename from docs/api/api_cs_space_saving.md rename to docs/api/api_count_sketch_with_space_saving.md index 5e91c5d2..17864cd3 100644 --- a/docs/api/api_cs_space_saving.md +++ b/docs/api/api_count_sketch_with_space_saving.md @@ -10,7 +10,7 @@ their Count Sketch estimate (row median). The summary picks the candidates and the matrix gives each its frequency. The summary's `capacity` is independent of `k`. -[`CountSketchWithHeap`](./api_cs_heap.md) keeps an estimate heap instead. +[`CountSketchWithHeap`](./api_count_sketch_with_heap.md) keeps an estimate heap instead. ## Type/Struct @@ -69,7 +69,7 @@ fn from_asapv1(bytes: &[u8]) -> Result The **ASAPv1** envelope, kind `0x1f 0x00` (spec §3.25), for `CountSketchWithSpaceSaving, Mode, H>` with `T` `i32`, `i64` or `f64`, `Mode` `FastPath` or `RegularPath`, and `H: HashProfile`. Same layout as -[CountMinSketchWithSpaceSaving](./api_cms_space_saving.md). There is no delta kind. +[CountMinSketchWithSpaceSaving](./api_count_min_sketch_with_space_saving.md). There is no delta kind. ## Examples diff --git a/docs/api/api_ehsketchlist.md b/docs/api/api_eh_bucket_sketch.md similarity index 100% rename from docs/api/api_ehsketchlist.md rename to docs/api/api_eh_bucket_sketch.md diff --git a/docs/api/api_elastic.md b/docs/api/api_elastic_sketch.md similarity index 99% rename from docs/api/api_elastic.md rename to docs/api/api_elastic_sketch.md index a41d8791..5fab26f8 100644 --- a/docs/api/api_elastic.md +++ b/docs/api/api_elastic_sketch.md @@ -1,4 +1,4 @@ -# API: Elastic +# API: ElasticSketch Status: `Ready` diff --git a/docs/api/api_ehunivoptimized.md b/docs/api/api_exponential_histogram_univmon.md similarity index 100% rename from docs/api/api_ehunivoptimized.md rename to docs/api/api_exponential_histogram_univmon.md diff --git a/docs/api/api_univmon_optimized.md b/docs/api/api_univmon_two_tier.md similarity index 100% rename from docs/api/api_univmon_optimized.md rename to docs/api/api_univmon_two_tier.md diff --git a/docs/apis.md b/docs/apis.md index 658fc2a2..5d6016f9 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -4,7 +4,7 @@ This is the canonical API entry point for `asap_sketchlib`. ## Core Sketch APIs -- [Count-Min Sketch](./api/api_countmin.md) - `Ready` +- [Count-Min Sketch](./api/api_count_min_sketch.md) - `Ready` - Reference: Cormode & Muthukrishnan, "An Improved Data Stream Summary: The Count-Min Sketch and its Applications," J. Algorithms 2005. [https://www.cs.rutgers.edu/~muthu/cm-jal.pdf](https://www.cs.rutgers.edu/~muthu/cm-jal.pdf) - [Count Sketch](./api/api_count_sketch.md) - `Ready` - Reference: Charikar, Chen & Farach-Colton, "Finding Frequent Items in Data Streams," ICALP 2002. [https://dl.acm.org/doi/10.1007/3-540-45465-9_59](https://dl.acm.org/doi/10.1007/3-540-45465-9_59) @@ -16,18 +16,18 @@ This is the canonical API entry point for `asap_sketchlib`. - Reference: Karnin, Lang & Liberty, "Optimal Quantile Approximation in Streams," FOCS 2016. [https://arxiv.org/abs/1603.05346](https://arxiv.org/abs/1603.05346) - [DDSketch](./api/api_ddsketch.md) - `Ready` - Reference: Masson, Rim & Lee, "DDSketch: A Fast and Fully-Mergeable Quantile Sketch with Relative-Error Guarantees," VLDB 2019. [https://arxiv.org/abs/1908.10693](https://arxiv.org/abs/1908.10693) -- [CountMinSketchWithHeap](./api/api_cms_heap.md) - `Ready` -- [CountSketchWithHeap](./api/api_cs_heap.md) - `Ready` -- [CountMinSketchWithSpaceSaving](./api/api_cms_space_saving.md) - `Unstable` -- [CountSketchWithSpaceSaving](./api/api_cs_space_saving.md) - `Unstable` +- [CountMinSketchWithHeap](./api/api_count_min_sketch_with_heap.md) - `Ready` +- [CountSketchWithHeap](./api/api_count_sketch_with_heap.md) - `Ready` +- [CountMinSketchWithSpaceSaving](./api/api_count_min_sketch_with_space_saving.md) - `Unstable` +- [CountSketchWithSpaceSaving](./api/api_count_sketch_with_space_saving.md) - `Unstable` - [SpaceSaving](./api/api_space_saving.md) - `Ready` - Reference: Metwally, Agrawal & El Abbadi, "Efficient Computation of Frequent and Top-k Elements in Data Streams," ICDT 2005. [https://doi.org/10.1007/978-3-540-30570-5_27](https://doi.org/10.1007/978-3-540-30570-5_27) - [Bloom](./api/api_bloom.md) - `Ready` - Reference: Bloom, "Space/Time Trade-offs in Hash Coding with Allowable Errors," CACM 1970. [https://dl.acm.org/doi/10.1145/362686.362692](https://dl.acm.org/doi/10.1145/362686.362692) - Per-slice partitioning: Kirsch & Mitzenmacher, "Less Hashing, Same Performance," ESA 2006. [https://doi.org/10.1007/11841036_42](https://doi.org/10.1007/11841036_42) -- [Elastic](./api/api_elastic.md) - `Ready` +- [Elastic](./api/api_elastic_sketch.md) - `Ready` - Reference: Yang et al., "Elastic Sketch: Adaptive and Fast Network-wide Measurements," SIGCOMM 2018. [https://dl.acm.org/doi/10.1145/3230543.3230544](https://dl.acm.org/doi/10.1145/3230543.3230544) -- [Coco](./api/api_coco.md) - `Ready` +- [Coco](./api/api_coco_sketch.md) - `Ready` - Reference: "CocoSketch: High-Performance Sketch-based Measurement over Arbitrary Partial Key Query," SIGCOMM 2021. [https://dl.acm.org/doi/10.1145/3452296.3472892](https://dl.acm.org/doi/10.1145/3452296.3472892) - [UniformSampling](./api/api_uniform_sampling.md) - `Unstable` - [KMV](./api/api_kmv.md) - `Unstable` @@ -45,7 +45,7 @@ This is the canonical API entry point for `asap_sketchlib`. - [HashSketchEnsemble](./api/api_hashlayer.md) - `Ready` - [UnivMon](./api/api_univmon.md) - `Ready` - Reference: Liu et al., "One Sketch To Rule Them All: Rethinking Network Flow Monitoring with UnivMon," SIGCOMM 2016. [https://dl.acm.org/doi/10.1145/2934872.2934906](https://dl.acm.org/doi/10.1145/2934872.2934906) -- [Two-tier UnivMon](./api/api_univmon_optimized.md) - `Ready` +- [Two-tier UnivMon](./api/api_univmon_two_tier.md) - `Ready` - [UnivMon-Q](./api/api_univmon_q.md) - `Experimental` - Universal F0/F2/compatible g-sum/entropy/heavy-hitter measurements plus additive-rank quantiles in one mergeable sketch. Its API, estimators, and guarantees may change. - Evaluation: [large synthetic comparison with UnivMon](./univmon_q_evaluation.md). @@ -55,8 +55,8 @@ This is the canonical API entry point for `asap_sketchlib`. - [ExponentialHistogram](./api/api_exponential_histogram.md) - `Ready` - Reference: Datar, Gionis, Indyk & Motwani, "Maintaining Stream Statistics over Sliding Windows," SIAM J. Computing 2002. [https://dl.acm.org/doi/10.1137/S0097539701398363](https://dl.acm.org/doi/10.1137/S0097539701398363) - Related implementation context: Zhu et al., "Approximation-First Timeseries Query At Scale," PVLDB 18(8). [https://dl.acm.org/doi/pdf/10.14778/3742728.3742732](https://dl.acm.org/doi/pdf/10.14778/3742728.3742732) -- [EhBucketSketch](./api/api_ehsketchlist.md) - `Ready` -- [ExponentialHistogramUnivMon](./api/api_ehunivoptimized.md) - `Unstable` +- [EhBucketSketch](./api/api_eh_bucket_sketch.md) - `Ready` +- [ExponentialHistogramUnivMon](./api/api_exponential_histogram_univmon.md) - `Unstable` - [OctoSketch](./api/api_octo.md) - `Ready` - Reference: Zhang, Chen & Liu, "OctoSketch: Enabling Real-Time, Continuous Network Monitoring over Multiple Cores," NSDI 2024. [https://www.usenix.org/conference/nsdi24/presentation/zhang-yinda](https://www.usenix.org/conference/nsdi24/presentation/zhang-yinda) diff --git a/docs/examples.md b/docs/examples.md index f908bac2..9e506586 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -82,7 +82,7 @@ for &user_id in &user_ids { let estimated_count = cms.estimate(&DataInput::U64(101)); ``` -API reference: [`docs/api/api_countmin.md`](./api/api_countmin.md) +API reference: [`docs/api/api_count_min_sketch.md`](./api/api_count_min_sketch.md) --- diff --git a/docs/index.md b/docs/index.md index d0e66088..c935f3fc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ This page is the docs home for `asap_sketchlib`. - [ASAPv1 Wire Format](./asapv1_wire_format.md) - authoritative byte-level spec for the self-describing sketch envelope mirrored by `sketchlib-go` - [Message Pack Format](./message_pack_format.md) - how the serialization code is organized (shared envelope + per-sketch `wire.rs`; `portable`/`native` retiring) - [Advanced Use Cases](./advanced_use_cases.md) -- [TopKHeap Acceleration](./hhheap_acceleration.md) - what the top-k heap costs per update, how its digest-keyed position index is maintained through the sift, and the throughput it measures at +- [TopKHeap Acceleration](./topk_heap_acceleration.md) - what the top-k heap costs per update, how its digest-keyed position index is maintained through the sift, and the throughput it measures at - [Test Coverage Map](./tests.md) - [E2E Testing Harness](./e2e_testing_harness.md) - how the end-to-end suites are built and what a new sketch must satisfy - [Feature Status](./features.md) diff --git a/docs/hhheap_acceleration.md b/docs/topk_heap_acceleration.md similarity index 98% rename from docs/hhheap_acceleration.md rename to docs/topk_heap_acceleration.md index 91abd990..4e3f1825 100644 --- a/docs/hhheap_acceleration.md +++ b/docs/topk_heap_acceleration.md @@ -5,7 +5,7 @@ them, so its cost per update is theirs. This page describes what it does per update and what that measures at. -Reproduce with `cargo run --release --example hhheap_probe`. Numbers below are +Reproduce with `cargo run --release --example topk_heap_probe`. Numbers below are from an Apple M-series laptop, 200k updates drawn from a Zipf(1.1) stream over 20k distinct keys. diff --git a/examples/hhheap_probe.rs b/examples/topk_heap_probe.rs similarity index 99% rename from examples/hhheap_probe.rs rename to examples/topk_heap_probe.rs index 39710b0e..8cf68e8a 100644 --- a/examples/hhheap_probe.rs +++ b/examples/topk_heap_probe.rs @@ -6,7 +6,7 @@ //! per second at several capacities. The scan column is what an unindexed heap //! costs; the indexed column is the floor `TopKHeap` should stay near. //! -//! Run with `cargo run --release --example hhheap_probe`. +//! Run with `cargo run --release --example topk_heap_probe`. use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, DataInput, FastPath, HeapEntry, OwnedDataInput, diff --git a/src/message_pack_format/native/countsketch_topk.rs b/src/message_pack_format/native/countsketch_l2_norm.rs similarity index 100% rename from src/message_pack_format/native/countsketch_topk.rs rename to src/message_pack_format/native/countsketch_l2_norm.rs diff --git a/src/message_pack_format/native/mod.rs b/src/message_pack_format/native/mod.rs index 94cc6cc4..5fea5948 100644 --- a/src/message_pack_format/native/mod.rs +++ b/src/message_pack_format/native/mod.rs @@ -9,7 +9,7 @@ pub mod countminsketch; pub mod countsketch; -pub mod countsketch_topk; +pub mod countsketch_l2_norm; pub mod ddsketch; pub mod hll; pub mod kll; diff --git a/tests/bug_verification.rs b/tests/bug_verification.rs index 06a07dc8..a8bfbbf3 100644 --- a/tests/bug_verification.rs +++ b/tests/bug_verification.rs @@ -58,7 +58,7 @@ fn portable_ddsketch_respects_alpha_at_bucket_edges() { // --------------------------------------------------------------------------- // Bug 4: CountSketchWithL2Norm's hot-path L2 accumulation ran in plain i64 -// (countsketch_topk.rs:1028: old + new^2 - old^2) and wrapped silently once +// (old + new^2 - old^2) and wrapped silently once // sum(count^2) exceeded i64::MAX (panicking in debug builds). The hot path // now saturates in i128, matching the merge path. // diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index e13b6a38..9f6f17f6 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -844,7 +844,7 @@ mod keyed_bucket { assert_unbiased_mean(&format!("Coco merged ({ctx})"), &merged, exact, 0.02); } - /// `docs/api/api_elastic.md`: "The estimator is one-sided: it never returns + /// `docs/api/api_elastic_sketch.md`: "The estimator is one-sided: it never returns /// less than the true count." A resident, unflagged flow reports its own votes /// exactly; every other flow reads the light layer, so the excess above the /// truth is that layer's Count-Min error and the whole tolerance is absolute. From 7298a7b1969b00b0cdef6a3e2fc91fa12983535a Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:02:24 +0000 Subject: [PATCH 10/11] test: test names follow the #209 renames Test functions that spelled a renamed write verb or type (insert, bulk_insert, insert_many, CMSHeap, CSHeap, CountL2HH, FoldCMS/FoldCS, EHSketchList, EHUnivOptimized, UnivMonPyramid, CmDelta, HllDelta, the *_PROMASK constants, ...) use the new names; docs/tests.md follows. Test names that name a golden file keep it. Co-Authored-By: Claude Opus 5.5 --- docs/e2e_testing_harness.md | 2 +- docs/tests.md | 428 +++++++++--------- src/common/structures/vector3d.rs | 4 +- src/sketch_framework/eh.rs | 2 +- src/sketch_framework/eh_bucket_sketch.rs | 6 +- src/sketch_framework/eh_bucket_sketch/wire.rs | 28 +- src/sketch_framework/hashlayer.rs | 6 +- src/sketch_framework/octo.rs | 12 +- src/sketch_framework/tumbling.rs | 40 +- src/sketch_framework/univmon.rs | 8 +- src/sketch_framework/univmon_two_tier.rs | 16 +- src/sketch_framework/univmon_two_tier/wire.rs | 34 +- src/sketches/bloom.rs | 2 +- src/sketches/coco.rs | 4 +- src/sketches/count_min_fold.rs | 24 +- src/sketches/count_sketch_fold.rs | 26 +- src/sketches/countminsketch.rs | 4 +- src/sketches/countminsketch_heap.rs | 12 +- src/sketches/countminsketch_heap/wire.rs | 32 +- src/sketches/countminsketch_hll.rs | 2 +- src/sketches/countsketch.rs | 8 +- src/sketches/countsketch_heap.rs | 12 +- src/sketches/countsketch_heap/wire.rs | 28 +- src/sketches/countsketch_l2_norm.rs | 10 +- src/sketches/countsketch_l2_norm/wire.rs | 28 +- src/sketches/elastic.rs | 10 +- src/sketches/hll.rs | 2 +- src/sketches/kll.rs | 2 +- src/sketches/kll_dynamic.rs | 2 +- src/sketches/microscope.rs | 4 +- tests/bug_verification.rs | 4 +- tests/conformance_kit.rs | 2 +- tests/e2e/experimental.rs | 14 +- tests/e2e/frameworks.rs | 8 +- tests/e2e/frequency.rs | 4 +- tests/e2e/heavy_hitters.rs | 4 +- tests/e2e/matrix_instances.rs | 8 +- tests/e2e/membership.rs | 4 +- tests/e2e/nitro.rs | 8 +- tests/e2e/octo.rs | 34 +- tests/e2e/quantiles.rs | 8 +- tests/e2e/windows.rs | 8 +- tests/e2e/wire.rs | 2 +- tests/msgpack_compat.rs | 2 +- tests/pbt/coco.rs | 2 +- tests/pbt/count_min.rs | 2 +- tests/pbt/count_sketch.rs | 4 +- tests/pbt/elastic.rs | 2 +- tests/pbt/ensemble.rs | 2 +- tests/pbt/exponential_histogram.rs | 8 +- tests/pbt/octo_delta.rs | 8 +- tests/pbt/space_saving.rs | 2 +- tests/pbt/topk_wrappers.rs | 18 +- tests/sketches_go_parity_probe.rs | 2 +- 54 files changed, 480 insertions(+), 478 deletions(-) diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index 3bd350b9..b42e1e27 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -308,7 +308,7 @@ or, when the behaviour is a documented limitation rather than a bug, pin it with *both* an upper and a lower guard so that fixing it later fails the test and forces the documentation to be updated instead of going stale. Two tests in this suite do exactly that: `hll_classic_switchover_band_stays_within_the_documented_empirical_band` -and `eh_univ_optimized_sketch_tier_cardinality_is_documented_as_unrecoverable`. +and `exponential_histogram_univmon_sketch_tier_cardinality_is_documented_as_unrecoverable`. **Do not use a fudge factor.** `1.5 * bound` and `alpha * 1.05` are not bounds; the second one accepts results that break the advertised guarantee by 5%. If a diff --git a/docs/tests.md b/docs/tests.md index ef1ca40c..244245c2 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -36,9 +36,9 @@ Delta tests: [`src/sketches/countminsketch/delta.rs`](../src/sketches/countminsk | test_name | test_description | what_is_tested | | --- | --- | --- | | `dimension_test` | Default/custom dimensions initialize zeroed counters. | Verifies default dimensions (`rows=3`, `cols=4096`), custom dimensions (`3x17`), and zero-initialized counters after construction. | -| `fast_insert_same_estimate` | Fast and regular insert paths produce identical estimates. | Inserts five string keys once into both `RegularPath` and `FastPath` sketches (`3x64`) and asserts equal estimates for every key. | +| `fast_path_update_same_estimate` | Fast and regular insert paths produce identical estimates. | Inserts five string keys once into both `RegularPath` and `FastPath` sketches (`3x64`) and asserts equal estimates for every key. | | `merge_adds_counters_element_wise` | Merge sums counters element-wise for matching dimensions. | Merges two `2x32` sketches after inserting the same key (`1` on left, `2` on right) and checks merged per-row target counters equal `3`. | -| `countmin_insert_emit_delta_emits_at_threshold_and_resets_period` | Worker-path delta emission fires at promotion threshold and resets. | Inserts key into `3x64` CMS via `update_emit_delta`; verifies no delta emitted before `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` inserts, then exactly `3` deltas (one per row) with `value == OCTO_COUNT_MIN_DEFAULT_THRESHOLD` at threshold, no extra deltas in next sub-threshold window, and another batch of `3` at the next threshold. | +| `countmin_update_emit_delta_emits_at_threshold_and_resets_period` | Worker-path delta emission fires at promotion threshold and resets. | Inserts key into `3x64` CMS via `update_emit_delta`; verifies no delta emitted before `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` inserts, then exactly `3` deltas (one per row) with `value == OCTO_COUNT_MIN_DEFAULT_THRESHOLD` at threshold, no extra deltas in next sub-threshold window, and another batch of `3` at the next threshold. | | `countmin_apply_delta_increments_parent_counter` | Apply delta increments parent counter. | Constructs a `CountMinSketchCellDelta{row=1, col=5, value=OCTO_COUNT_MIN_DEFAULT_THRESHOLD}`, applies it to a `3x64` parent CMS, and verifies the target counter at `(1,5)` equals `OCTO_COUNT_MIN_DEFAULT_THRESHOLD`. | | `cm_regular_path_correctness` | Regular-path hashing, counters, and estimates are exact on a deterministic stream. | Recomputes expected counter indices for `I32(0..9)` using per-row hashing, asserts exact full-matrix equality after one pass, doubled counters after second pass, and estimate `== 2` for each inserted key. | | `cm_fast_path_correctness` | Fast-path counter placement matches bit-sliced hash mapping. | Recomputes expected fast-path indices for `I32(0..9)` from one hash plus row bit-slices/mask bits and asserts exact full-matrix equality. | @@ -69,13 +69,13 @@ Delta tests: [`src/sketches/countsketch/delta.rs`](../src/sketches/countsketch/d | --- | --- | --- | | `default_initializes_expected_dimensions` | Default dimensions initialize zeroed counters. | Verifies default `CountSketch` dimensions (`rows=3`, `cols=4096`) and that all counters are zero after construction. | | `with_dimensions_uses_custom_sizes` | Custom dimensions initialize zeroed rows. | Verifies `with_dimensions(3, 17)` applies requested shape and each row slice is zero-initialized. | -| `insert_updates_signed_counters_per_row` | Regular insert applies per-row signed updates. | After one insert of key `"alpha"` into a `3x64` sketch, checks each row’s hashed counter equals that row’s expected sign (`+1` or `-1`). | -| `fast_insert_produces_consistent_estimates` | Fast-path single inserts return unit estimates. | Inserts five string keys once into a fast-path sketch (`4x128`) and asserts estimate `== 1.0` for each key. | -| `insert_produces_consistent_estimates` | Regular-path single inserts return unit estimates. | Inserts five string keys once into a regular-path sketch (`3x64`) and asserts estimate `== 1.0` for each key. | +| `update_writes_signed_counters_per_row` | Regular insert applies per-row signed updates. | After one insert of key `"alpha"` into a `3x64` sketch, checks each row’s hashed counter equals that row’s expected sign (`+1` or `-1`). | +| `fast_path_update_produces_consistent_estimates` | Fast-path single inserts return unit estimates. | Inserts five string keys once into a fast-path sketch (`4x128`) and asserts estimate `== 1.0` for each key. | +| `update_produces_consistent_estimates` | Regular-path single inserts return unit estimates. | Inserts five string keys once into a regular-path sketch (`3x64`) and asserts estimate `== 1.0` for each key. | | `estimate_recovers_frequency_for_repeated_key` | Regular path recovers repeated-key frequency. | Inserts key `"theta"` 37 times into a regular-path sketch (`3x64`) and asserts estimate `== 37.0`. | | `fast_path_recovers_repeated_insertions` | Fast path recovers repeated insertions across keys. | Inserts five keys for 5 rounds into a fast-path sketch (`4x256`) and asserts estimate `== 5.0` for each key. | | `merge_adds_counters_element_wise` | Merge sums signed counters for matching dimensions. | Merges two regular-path `2x32` sketches after inserting the same key (`1` on left, `2` on right) and checks per-row target counters equal `sign(row,key) * 3`. | -| `count_child_insert_emits_at_threshold` | Worker-path delta emission fires after sufficient inserts. | Inserts key into `3x64` `CountSketch` via `update_emit_delta` for `200` iterations and verifies at least `3` deltas (one per row) are emitted. | +| `count_sketch_child_update_emits_at_threshold` | Worker-path delta emission fires after sufficient inserts. | Inserts key into `3x64` `CountSketch` via `update_emit_delta` for `200` iterations and verifies at least `3` deltas (one per row) are emitted. | | `zipf_stream_stays_within_twenty_percent_for_most_keys` | Zipf stream keeps relative error under 20% for most keys. | On Zipf stream (`rows=5`, `cols=8192`, `domain=8192`, `exponent=1.1`, `N=200_000`), computes per-key relative error and requires at least 70% of keys with error `< 0.20`. | | `cs_regular_path_correctness` | Regular-path counter/sign mapping and estimates are exact on deterministic inserts. | Recomputes expected signed counter updates for `I32(0..9)` using regular hashing/sign logic, asserts exact matrix match after one pass, doubled counters after second pass, and estimate `== 2.0` for each inserted key. | | `cs_fast_path_correctness` | Fast-path row-hash/sign mapping matches expected counters. | Recomputes expected fast-path updates for `I32(0..9)` using matrix hash row slices and row signs, then asserts exact full-matrix equality. | @@ -110,7 +110,7 @@ Delta tests: [`src/sketches/hll/delta.rs`](../src/sketches/hll/delta.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `hll_child_insert_emits_on_improvement` | Child insert emits delta only on register improvement. | Inserts a key via `update_emit_delta` into `HyperLogLog`; verifies exactly `1` delta emitted on the first insert and `0` additional deltas on a duplicate insert. | +| `hll_child_update_emits_on_improvement` | Child insert emits delta only on register improvement. | Inserts a key via `update_emit_delta` into `HyperLogLog`; verifies exactly `1` delta emitted on the first insert and `0` additional deltas on a duplicate insert. | | `hyperloglog_p12_accuracy_within_two_percent` | P12 Classic HyperLogLog stays within P12 error tolerance across scale checkpoints. | Applies the same checkpointed unique-stream accuracy test to `HyperLogLogP12`, requiring relative error `<= P12_ERROR_TOLERANCE` at each target cardinality. | | `hll_ertl_p12_accuracy_within_two_percent` | P12 Ertl HyperLogLog stays within P12 error tolerance across scale checkpoints. | Applies the same checkpointed accuracy test to `HyperLogLogP12`, requiring relative error `<= P12_ERROR_TOLERANCE`. | | `hllds_p12_accuracy_within_two_percent` | P12 HIP HyperLogLog stays within P12 error tolerance across scale checkpoints. | Applies the same checkpointed accuracy test to `HyperLogLogHipP12`, requiring relative error `<= P12_ERROR_TOLERANCE`. | @@ -172,7 +172,7 @@ Wire tests: [`src/sketches/kll/wire.rs`](../src/sketches/kll/wire.rs) | `clear_preserves_seed_determinism` | `clear` re-seeds from the stored seed, so determinism survives a window rotation. | Burns 1,500 unrelated samples (seed `99`) through a `with_k_and_seed(200, 1234)` sketch, clears it, then feeds it and a fresh seed-1234 twin the same 3,000 samples (seed `13`) and verifies their serialized bytes are equal. | | `merge_into_empty_target_preserves_weight_issue_68_repro` | Merging into an empty target preserves the total weight exactly. | Builds a `k=200` source over `1..=1000`, checks its `count` falls in `980..=1020`, merges it into an empty `k=200` target, and verifies the target's `count` equals the source's exactly and its median lands in `475.0..=525.0`. | | `merge_two_nonempty_sketches_preserves_weight_and_quantiles` | Merging two non-empty sketches is a genuinely weighted merge, not just the empty-target special case. | Builds one `k=200` sketch over 50,000 uniform samples in `[0, 1_000_000)` (seed `0xA11CE`) and another over 50,000 Zipf samples (8,192 distinct, exponent `1.1`, seed `0xB0B`), verifies each pre-merge `count` is within 3% of `50_000`, that the merged `count` is within 3% of their sum, and that `assert_quantiles_within_error` holds at `{0, 0.1, 0.25, 0.5, 0.75, 0.9, 1}` within a rank tolerance of `0.03` against the sorted union. | -| `bulk_update_equivalent_to_loop_and_empty_is_noop` | `update_batch` is the loop it replaces, and an empty batch changes nothing. | Verifies `update_batch(&[])` on a seed-42 `with_seed(200, 8, 42)` sketch holding `[1.0, 2.0, 3.0]` leaves `count`, `quantile(0.5)`, and the serialized bytes untouched; that 20,000 uniform samples in `[0, 1000)` (seed `0xBEEF_1234`) through `update_batch` match the one-at-a-time loop on `count`, bytes, and quantiles `{0.1, 0.5, 0.9}` for two seed-99 sketches; that 100 `DataInput::F64` values through `update_batch_data_input` match the loop's bytes at seed 77; and that `update_batch_data_input` over `[F64(1.0), String("x"), F64(2.0)]` errors with `count` left at `1`. | +| `update_batch_equivalent_to_loop_and_empty_is_noop` | `update_batch` is the loop it replaces, and an empty batch changes nothing. | Verifies `update_batch(&[])` on a seed-42 `with_seed(200, 8, 42)` sketch holding `[1.0, 2.0, 3.0]` leaves `count`, `quantile(0.5)`, and the serialized bytes untouched; that 20,000 uniform samples in `[0, 1000)` (seed `0xBEEF_1234`) through `update_batch` match the one-at-a-time loop on `count`, bytes, and quantiles `{0.1, 0.5, 0.9}` for two seed-99 sketches; that 100 `DataInput::F64` values through `update_batch_data_input` match the loop's bytes at seed 77; and that `update_batch_data_input` over `[F64(1.0), String("x"), F64(2.0)]` errors with `count` left at `1`. | ### KllDynamic @@ -196,7 +196,7 @@ Wire tests: [`src/sketches/kll_dynamic/wire.rs`](../src/sketches/kll_dynamic/wir | `merge_into_empty_target_preserves_weight_issue_68_repro` | Merging into an empty target preserves the total weight exactly. | Builds a `k=200` source over `1..=1000`, checks its `count` falls in `980..=1020`, merges it into an empty `k=200` target, and verifies the target's `count` equals the source's exactly and its median lands in `475.0..=525.0`. | | `merge_two_nonempty_sketches_preserves_weight_and_quantiles` | Merging two non-empty sketches is a genuinely weighted merge, not just the empty-target special case. | Builds one `k=200` sketch over 50,000 uniform samples in `[0, 1_000_000)` (seed `0xA11CE`) and another over 50,000 Zipf samples (8,192 distinct, exponent `1.1`, seed `0xB0B`), verifies each pre-merge `count` is within 3% of `50_000`, that the merged `count` is within 3% of their sum, and that `assert_quantiles_within_error` holds at `{0, 0.1, 0.25, 0.5, 0.75, 0.9, 1}` within a rank tolerance of `0.03` against the sorted union. | | `merge_handles_operand_level_that_is_not_a_single_sorted_run` | A merge sorts an operand level that is a concatenation of separately-sorted chunks. | Hand-builds a `KllDynamic::::new(50, 4)` operand whose level 1 is `[30.0, 40.0, 10.0, 20.0]` - two ascending chunks out of order - with `levels` `[0, 4, 5]` and `num_levels` `2`, checks `capacity_for_level(1)` is at least `4` so the level is never itself compacted, merges it into an empty `new(50, 4)` target, and verifies every level above 0 is a single ascending run afterwards. | -| `bulk_update_equivalent_to_loop` | `update_batch` tracks the loop it replaces. | Feeds 10,000 uniform samples in `[0, 1000)` (seed `0xCAFE_1234`) to two `with_k(200)` sketches, one value at a time and one through `update_batch`, and verifies their counts agree within 5% relative; then that 100 `DataInput::F64` values through `update_batch_data_input` give the same `count` as the loop. Only counts are compared - quantiles and bytes are not, since the wall-clock `Coin` makes them non-deterministic. | +| `update_batch_equivalent_to_loop` | `update_batch` tracks the loop it replaces. | Feeds 10,000 uniform samples in `[0, 1000)` (seed `0xCAFE_1234`) to two `with_k(200)` sketches, one value at a time and one through `update_batch`, and verifies their counts agree within 5% relative; then that 100 `DataInput::F64` values through `update_batch_data_input` give the same `count` as the loop. Only counts are compared - quantiles and bytes are not, since the wall-clock `Coin` makes them non-deterministic. | ### DDSketch @@ -255,45 +255,45 @@ Delta tests: [`src/sketches/countminsketch_heap/delta.rs`](../src/sketches/count | test_name | test_description | what_is_tested | | --- | --- | --- | -| `insert_and_estimate` | Repeated inserts increment Count-Min estimate for one key. | Inserts `"hello"` 5 times into `CountMinSketchWithHeap::new(3,64,10)` and verifies `estimate("hello") == 5`. | +| `update_and_estimate` | Repeated inserts increment Count-Min estimate for one key. | Inserts `"hello"` 5 times into `CountMinSketchWithHeap::new(3,64,10)` and verifies `estimate("hello") == 5`. | | `heap_tracks_top_k` | Heap keeps highest-frequency keys within top-k capacity. | Inserts keys `1..5` with frequencies `10,20,30,40,50` into `heap_size=3` sketch and verifies heap counts are exactly `[30,40,50]`. | | `merge_reconciles_heaps` | Merge combines counters and refreshes heap counts from merged sketch. | Merges two sketches containing `"merge_key"` counts `10` and `20`, then verifies merged estimate and heap count are both `30`. | | `f64_heap_counts_are_the_unrounded_estimate` | An `f64` sketch's heap holds the unrounded estimate. | On both paths, verifies `update_weighted("flow", 2.25)` leaves heap count `2.25` and a merge with a copy leaves `4.5`. | -| `insert_many_updates_estimate_and_heap` | `update_weighted` updates estimate and heap entry consistently. | Calls `update_weighted("many", 11)` and verifies `estimate == 11` plus heap entry count `11`. | -| `bulk_insert_updates_multiple_keys` | `update_batch` updates multiple keys and heavy-hitter counts correctly. | Inserts stream `[7,8,7,9,7]` and verifies estimates `7->3`, `8->1`, `9->1`, with heap count for key `7` equal to `3`. | +| `update_weighted_updates_estimate_and_heap` | `update_weighted` updates estimate and heap entry consistently. | Calls `update_weighted("many", 11)` and verifies `estimate == 11` plus heap entry count `11`. | +| `update_batch_updates_multiple_keys` | `update_batch` updates multiple keys and heavy-hitter counts correctly. | Inserts stream `[7,8,7,9,7]` and verifies estimates `7->3`, `8->1`, `9->1`, with heap count for key `7` equal to `3`. | | `clear_heap_keeps_cms_counters` | Clearing heap does not clear CMS counters. | After `update_weighted("persist",5)`, calls `clear_heap()`, verifies estimate remains `5`, then one more insert rebuilds heap entry to `6`. | | `from_storage_uses_storage_dimensions` | `from_storage` preserves backend dimensions and requested heap capacity. | Builds from `Vector2D::unfilled(4,128)` with `heap_size=9` and verifies `rows=4`, `cols=128`, `heap.capacity=9`. | | `merge_refreshes_existing_self_heap_entries` | Merge refreshes pre-existing self heap keys to merged estimates. | After merging sketches with `a-key` counts `10` and `5`, verifies merged `a-key` estimate `15` and heap entry count `15`. | -| `fast_path_insert_and_estimate` | Fast path repeated inserts keep estimate exact for single key. | Inserts `"fast"` 7 times into fast-path sketch and verifies estimate `7`. | -| `fast_path_insert_many_and_bulk_insert` | Fast path batched APIs keep heap and estimate in sync. | Applies `update_weighted("fast-many",6)` plus bulk inserts adding 2 more hits, then verifies estimate and heap count are `8`. | +| `fast_path_update_and_estimate` | Fast path repeated inserts keep estimate exact for single key. | Inserts `"fast"` 7 times into fast-path sketch and verifies estimate `7`. | +| `fast_path_update_weighted_and_update_batch` | Fast path batched APIs keep heap and estimate in sync. | Applies `update_weighted("fast-many",6)` plus bulk inserts adding 2 more hits, then verifies estimate and heap count are `8`. | | `fast_path_heap_tracks_top_k` | Fast path heap still preserves top-k ordering under weighted updates. | Inserts keys `1..5` with counts `10,20,30,40,50` via `update_weighted` and verifies heap counts `[30,40,50]`. | | `fast_path_merge_refreshes_existing_self_heap_entries` | Fast path merge refreshes self heap entries using merged totals. | Merges sketches where `"a-fast"` contributes `10` and `5` across sides, then verifies estimate and heap count are `15`. | | `default_construction` | Default CountMinSketchWithHeap constructor uses expected dimensions and heap capacity. | Verifies `CountMinSketchWithHeap::, RegularPath>::default()` has `rows=3`, `cols=4096`, and `heap.capacity=DEFAULT_HEAP_SIZE`. | | `default_construction_fixed_backends_parity` | Default constructors across storage backends keep intended size/capacity contracts. | Verifies defaults for Fixed/Quick backends are `5x2048`, DefaultMatrix backends are `3x4096`, and all regular/fast variants use `DEFAULT_HEAP_SIZE`. | | `merge_requires_matching_dimensions_panics` | Merge panics on incompatible sketch dimensions. | Verifies merging `CountMinSketchWithHeap::new(3,256,4)` with `CountMinSketchWithHeap::new(4,256,4)` panics with dimension-mismatch message. | | `heap_entries_match_cms_estimates_after_mutations` | Every heap entry count matches current CMS estimate after updates and merge. | Checks heap-entry equality to `estimate(key)` both before and after merging another mutated sketch. | -| `bulk_insert_equivalent_to_repeated_insert` | Bulk insert is equivalent to repeated single inserts. | Compares bulk vs repeated insertion on same stream and verifies identical per-key estimates and heap counts for keys `1..5`. | +| `update_batch_equivalent_to_repeated_update` | Bulk insert is equivalent to repeated single inserts. | Compares bulk vs repeated insertion on same stream and verifies identical per-key estimates and heap counts for keys `1..5`. | | `regular_vs_fast_equivalence_on_same_stream` | Regular and fast wrappers agree on identical deterministic stream. | Feeds same 10-item string stream to both paths and verifies per-key estimates and heap counts match for `{alpha,beta,gamma,delta,epsilon}`. | | `merge_with_empty_other_and_empty_self` | Merge behavior is stable when one side is empty. | Verifies merging non-empty with empty leaves counts unchanged and merging empty-self with non-empty copies counts/heap visibility correctly. | | `duplicate_candidate_keys_during_merge_do_not_corrupt_heap` | Duplicate merge candidates do not duplicate heap entries. | Merges sketches both containing `"dup"`; verifies merged count `19`, heap size within capacity, and exactly one heap entry for `"dup"`. | | `zipf_stream_top_k_recall_regular_fast_budget` | Regular path heap achieves high top-k recall on Zipf stream. | On Zipf stream (`rows=3`, `cols=4096`, `heap_size=16`, `domain=1024`, `exponent=1.1`, `N=20_000`), verifies heap size bound, entry-count consistency, and recall hits `>= 15` vs truth top-16. | | `zipf_stream_top_k_recall_fast_path_fast_budget` | Fast path heap achieves high top-k recall on Zipf stream. | Runs same Zipf setup in fast mode and verifies heap size bound, entry-count consistency, and recall hits `>= 15`. | | `zipf_stream_regular_fast_heap_overlap` | Regular and fast heaps substantially overlap on Zipf heavy hitters. | On shared Zipf stream (`heap_size=16`), verifies key overlap ratio between regular and fast top-k heaps is at least `0.8`. | -| `cms_heap_round_trip_serialization` | The envelope frames a sketch under kind_id `0x03 0x00`, and the bytes are stable across a round trip. | For a `3x8` regular-path `i64` sketch with a heap of `4` holding three weighted `u64` keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x03 0x00`, that the metadata carries `3x8`, `counter_type` `i64`, `mode` `regular`, `heap_size` `4`, and `key_type` `u64`, that dimensions, counters, heap length and capacity, and per-key estimates and heap counts all match, and that re-serializing reproduces the bytes. | -| `cms_heap_byte_keys_round_trip` | A raw byte-array heap key reaches the wire as msgpack `bin`. | For a `2x4` `i64` sketch whose heap holds `[0xff, 0x00, 0xfe]` at `11`, verifies the metadata names `key_type` `bytes`, the decoded heap still finds the key by its own `DataInput::Bytes` at that count, and the bytes re-serialize identically. | -| `cms_heap_f64_counters_round_trip` | The `f64` base counter reaches the wire beside a string-keyed heap. | For a `2x4` `Vector2D` sketch holding one `Str` heap key, verifies the metadata names `counter_type` `f64` and `key_type` `string`, that the counters and the heap key survive the decode, and that the bytes re-serialize identically. | -| `cms_heap_counter_type_is_pinned_by_the_target` | The base counter type separates two otherwise equal sketches. | For `2x4` sketches holding numerically equal `i64` and `f64` cells, verifies the bytes differ, that `i64` bytes fail to decode as `f64`, and that `f64` bytes fail to decode as `i64`. | -| `cms_heap_mode_in_metadata_round_trips` | The metadata `mode` pins which column derivation built the sketch. | Round-trips a `4x16` fast-path sketch, verifies the metadata names `mode` `fast` and the counters are preserved, then that the same bytes fail to decode as a `RegularPath` sketch. | -| `cms_heap_rejects_foreign_kind_ids` | The neighbouring sketches' kind_ids are refused. | Verifies `CountSketchWithHeap`, `CountMinSketch`, and `CountSketch` envelopes at `3x8` each fail to decode as a `CountMinSketchWithHeap`. | -| `cms_heap_rejects_zero_dimension_payload` | A zero dimension is a decode error, not a `Vector2D::from_fn` panic. | Verifies a crafted `4x0` envelope fails with a "must be non-zero" complaint. | -| `cms_heap_rejects_dimension_length_mismatch` | The length check fires from the declared dimensions, before any allocation is sized from them. | Verifies a crafted envelope declaring `MATRIX_MAX_ROWS x 2^24` while carrying three counters fails with a "!= rows*cols" complaint. | -| `cms_heap_rejects_serializing_zero_rows` | The encode side refuses a zero dimension. | Verifies a `0x4` sketch fails to serialize with a "non-zero" complaint. | -| `cms_heap_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies `Vector2D::::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | -| `cms_heap_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the thirteen `HeapMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HeapMetadata`. | -| `cms_heap_metadata_rejects_a_missing_heap_size_key` | `heap_size` is required, so the heap capacity can never be silently defaulted. | Encodes the other twelve `HeapMetadata` fields as a named map with `heap_size` omitted and verifies it does not decode as `HeapMetadata`. | -| `cms_heap_rejects_a_foreign_counter_type_name` | `counter_type` is pinned by the target, not merely checked for naming a wire counter. | Wraps metadata naming `counter_type: "i32"` — one of Count-Min's three wire counters, alongside `i64` and `f64` — around a valid `2x4` payload, so the envelope is well-formed and still names a counter this sketch does not hold, and verifies an `i64` `CountMinSketchWithHeap` rejects it. | -| `cms_heap_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x8` sketch with a heap of `4` over a hasher declaring its own `HashProfile`, verifies the counters and both heap entries round-trip, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | -| `cms_heap_refuses_a_heap_size_the_metadata_cannot_carry` | A `heap_size` past the metadata's `u32` field fails the encode rather than truncating. | Verifies a `2x4` sketch built at a heap capacity of `1 << 40` fails to serialize with a message naming the "exceeds the u32 metadata field" rule. | +| `count_min_sketch_with_heap_round_trip_serialization` | The envelope frames a sketch under kind_id `0x03 0x00`, and the bytes are stable across a round trip. | For a `3x8` regular-path `i64` sketch with a heap of `4` holding three weighted `u64` keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x03 0x00`, that the metadata carries `3x8`, `counter_type` `i64`, `mode` `regular`, `heap_size` `4`, and `key_type` `u64`, that dimensions, counters, heap length and capacity, and per-key estimates and heap counts all match, and that re-serializing reproduces the bytes. | +| `count_min_sketch_with_heap_byte_keys_round_trip` | A raw byte-array heap key reaches the wire as msgpack `bin`. | For a `2x4` `i64` sketch whose heap holds `[0xff, 0x00, 0xfe]` at `11`, verifies the metadata names `key_type` `bytes`, the decoded heap still finds the key by its own `DataInput::Bytes` at that count, and the bytes re-serialize identically. | +| `count_min_sketch_with_heap_f64_counters_round_trip` | The `f64` base counter reaches the wire beside a string-keyed heap. | For a `2x4` `Vector2D` sketch holding one `Str` heap key, verifies the metadata names `counter_type` `f64` and `key_type` `string`, that the counters and the heap key survive the decode, and that the bytes re-serialize identically. | +| `count_min_sketch_with_heap_counter_type_is_pinned_by_the_target` | The base counter type separates two otherwise equal sketches. | For `2x4` sketches holding numerically equal `i64` and `f64` cells, verifies the bytes differ, that `i64` bytes fail to decode as `f64`, and that `f64` bytes fail to decode as `i64`. | +| `count_min_sketch_with_heap_mode_in_metadata_round_trips` | The metadata `mode` pins which column derivation built the sketch. | Round-trips a `4x16` fast-path sketch, verifies the metadata names `mode` `fast` and the counters are preserved, then that the same bytes fail to decode as a `RegularPath` sketch. | +| `count_min_sketch_with_heap_rejects_foreign_kind_ids` | The neighbouring sketches' kind_ids are refused. | Verifies `CountSketchWithHeap`, `CountMinSketch`, and `CountSketch` envelopes at `3x8` each fail to decode as a `CountMinSketchWithHeap`. | +| `count_min_sketch_with_heap_rejects_zero_dimension_payload` | A zero dimension is a decode error, not a `Vector2D::from_fn` panic. | Verifies a crafted `4x0` envelope fails with a "must be non-zero" complaint. | +| `count_min_sketch_with_heap_rejects_dimension_length_mismatch` | The length check fires from the declared dimensions, before any allocation is sized from them. | Verifies a crafted envelope declaring `MATRIX_MAX_ROWS x 2^24` while carrying three counters fails with a "!= rows*cols" complaint. | +| `count_min_sketch_with_heap_rejects_serializing_zero_rows` | The encode side refuses a zero dimension. | Verifies a `0x4` sketch fails to serialize with a "non-zero" complaint. | +| `count_min_sketch_with_heap_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies `Vector2D::::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | +| `count_min_sketch_with_heap_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the thirteen `HeapMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HeapMetadata`. | +| `count_min_sketch_with_heap_metadata_rejects_a_missing_heap_size_key` | `heap_size` is required, so the heap capacity can never be silently defaulted. | Encodes the other twelve `HeapMetadata` fields as a named map with `heap_size` omitted and verifies it does not decode as `HeapMetadata`. | +| `count_min_sketch_with_heap_rejects_a_foreign_counter_type_name` | `counter_type` is pinned by the target, not merely checked for naming a wire counter. | Wraps metadata naming `counter_type: "i32"` — one of Count-Min's three wire counters, alongside `i64` and `f64` — around a valid `2x4` payload, so the envelope is well-formed and still names a counter this sketch does not hold, and verifies an `i64` `CountMinSketchWithHeap` rejects it. | +| `count_min_sketch_with_heap_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x8` sketch with a heap of `4` over a hasher declaring its own `HashProfile`, verifies the counters and both heap entries round-trip, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | +| `count_min_sketch_with_heap_refuses_a_heap_size_the_metadata_cannot_carry` | A `heap_size` past the metadata's `u32` field fails the encode rather than truncating. | Verifies a `2x4` sketch built at a heap capacity of `1 << 40` fails to serialize with a message naming the "exceeds the u32 metadata field" rule. | | `heap_every_key_type_round_trips_and_keeps_its_variant` | The `key_type` names the exact `OwnedDataInput` variant and is never widened. | Across all 13 wire key types - `i8`, `i16`, `i32`, `i64`, `isize`, `u8`, `u16`, `u32`, `u64`, `usize`, `f32`, `f64`, and `string` - verifies the metadata carries the expected name, the bytes re-serialize identically, and every decoded key is still found by its original `DataInput` at the weight it was given. | | `heap_refuses_mixed_and_128_bit_keys` | Keys the wire cannot carry refuse to serialize rather than being coerced. | Verifies a heap holding both an `I32` and an `I64` key fails with a "keys mix variants" complaint naming `key_type is i32`, that a lone `I128` or `U128` key fails with a "128-bit" complaint, and that a 128-bit key seated behind a `U64` one fails too. | | `heap_empty_emits_the_pinned_key_type_and_round_trips` | A heap monitoring nothing has one encoding. | For an empty `2x8` sketch with a heap of `8`, verifies the metadata carries the pinned `EMPTY_KEY_TYPE` and a `heap_size` of `8`, and that the decode holds no entries at capacity `8` with identical re-serialized bytes. | @@ -305,7 +305,7 @@ Delta tests: [`src/sketches/countminsketch_heap/delta.rs`](../src/sketches/count | `heap_does_not_allocate_a_declared_heap_size` | A declared `heap_size` is metadata: it bounds the entry count and is reported as the capacity, while the rebuild reserves from the entries it actually has. | Verifies an envelope declaring `heap_size` `u32::MAX` with two entries decodes, reports that capacity, and holds two entries. `rebuild_heap` checks the entry count against `heap_size` first, then seats the entries into `TopKHeap::with_expected_len(heap_size, entries.len())`, whose array and index each reserve `entries.len().min(heap_size).min(PREALLOCATED_SLOTS)` — so the declared `heap_size` sizes no allocation on its own. | | `heap_rejects_crafted_entry_sets` | Entry sets the heap could not have reached are refused. | Verifies three crafted envelopes each fail with their own complaint: two entries over a `heap_size` of `1`, the same key twice, and two keys against one count. | | `heap_rejects_entries_out_of_emitted_order` | Entries must arrive in descending count, ties by key order. | Verifies `i64` keys `[-1, 1]` at a count tie, and counts `[4, 9]`, each fail with a "not in descending count" complaint. | -| `cms_heap_rejects_too_many_rows` | The base matrix carries the seed list's row bound. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | +| `count_min_sketch_with_heap_rejects_too_many_rows` | The base matrix carries the seed list's row bound. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | | `decode_rejects_each_payload_rule` | A `0x03 0x80` payload breaking a cell or heap rule fails to decode. | For a `2x3` `i64` delta with heap size `2`, verifies cells `[0, 5]` beside heap `{1: 9, 2: 4}` decode, and that index `6` and a heap in ascending count each fail. | | `decode_rejects_another_kind` | A delta decodes only under its own kind_id. | Verifies the `cmsheap_i64_regular_2x3_delta` golden decodes and fails with its kind_id relabelled `0x03 0x00`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the matrix and the heap unchanged. | For a `2x3` state with heap size `2` holding `"kept"`, verifies a `3x2` delta, a delta with heap size `3` and a delta whose last cell overflows `i64::MAX` each fail, leaving the cells and the one-entry heap as they were. | @@ -320,45 +320,45 @@ Delta tests: [`src/sketches/countsketch_heap/delta.rs`](../src/sketches/countske | test_name | test_description | what_is_tested | | --- | --- | --- | -| `insert_and_estimate` | Repeated inserts increment `CountSketch` estimate for one key. | Inserts `"hello"` 5 times into `CountSketchWithHeap::new(5,256,10)` and verifies estimate is `5.0` within `1e-9`. | +| `update_and_estimate` | Repeated inserts increment `CountSketch` estimate for one key. | Inserts `"hello"` 5 times into `CountSketchWithHeap::new(5,256,10)` and verifies estimate is `5.0` within `1e-9`. | | `heap_tracks_top_k` | Heap keeps highest-frequency keys within top-k capacity. | Inserts keys `1..5` with frequencies `100,200,300,400,500` into `heap_size=3` sketch and verifies heap counts are exactly `[300,400,500]`. | | `merge_reconciles_heaps` | Merge combines counters and refreshes heap counts from merged sketch. | Merges two sketches containing `"merge_key"` counts `10` and `20`, then verifies estimate is `30.0` and heap count is `30`. | | `integer_heap_counts_truncate_the_median_toward_zero` | An integer sketch's heap holds its median truncated toward zero. | For a `2x64` `i64` sketch whose rows read `-2` and `-3` for `"flow"`, verifies the estimate is `-2.5` and the heap count `-2`. | | `f64_heap_counts_are_the_unrounded_median` | An `f64` sketch's heap holds the unrounded median. | On both paths, verifies `update_weighted("flow", -2.25)` leaves heap count `-2.25` and a merge with a copy leaves `-4.5`. | -| `insert_many_updates_estimate_and_heap` | `update_weighted` updates estimate and heap entry consistently. | Calls `update_weighted("many", 17)` and verifies estimate `17.0` plus heap entry count equals estimated count. | -| `bulk_insert_updates_multiple_keys` | `update_batch` updates multiple keys and heavy-hitter counts correctly. | Inserts stream `[7,8,7,9,7]`, verifies estimate for key `7` is `3.0`, and heap count for key `7` matches estimate cast to integer. | +| `update_weighted_updates_estimate_and_heap` | `update_weighted` updates estimate and heap entry consistently. | Calls `update_weighted("many", 17)` and verifies estimate `17.0` plus heap entry count equals estimated count. | +| `update_batch_updates_multiple_keys` | `update_batch` updates multiple keys and heavy-hitter counts correctly. | Inserts stream `[7,8,7,9,7]`, verifies estimate for key `7` is `3.0`, and heap count for key `7` matches estimate cast to integer. | | `clear_heap_keeps_cs_counters` | Clearing heap does not clear `CountSketch` counters. | After `update_weighted("persist",5)`, calls `clear_heap()`, verifies estimate remains `5.0`, then one more insert repopulates heap with updated estimate count. | | `from_storage_uses_storage_dimensions` | `from_storage` preserves backend dimensions and requested heap capacity. | Builds from `Vector2D::unfilled(4,128)` with `heap_size=9` and verifies `rows=4`, `cols=128`, `heap.capacity=9`. | | `merge_refreshes_existing_self_heap_entries` | Merge refreshes pre-existing self heap keys to merged estimates. | Merges sketches where `"a-key"` is updated on both sides (`120` and `40`), then verifies heap count for `"a-key"` equals merged estimate. | -| `fast_path_insert_and_estimate` | Fast path repeated inserts keep estimate exact for single key. | Inserts `"fast"` 7 times into fast-path sketch and verifies estimate is `7.0` within `1e-9`. | -| `fast_path_insert_many_and_bulk_insert` | Fast path batched APIs keep heap and estimate in sync. | Applies `update_weighted("fast-many",6)` plus bulk inserts adding 2 hits, then verifies estimate is `8.0` and heap count matches it. | +| `fast_path_update_and_estimate` | Fast path repeated inserts keep estimate exact for single key. | Inserts `"fast"` 7 times into fast-path sketch and verifies estimate is `7.0` within `1e-9`. | +| `fast_path_update_weighted_and_update_batch` | Fast path batched APIs keep heap and estimate in sync. | Applies `update_weighted("fast-many",6)` plus bulk inserts adding 2 hits, then verifies estimate is `8.0` and heap count matches it. | | `fast_path_heap_tracks_top_k` | Fast path heap preserves top-k ordering under weighted updates. | Inserts keys `1..5` with counts `100,200,300,400,500` via `update_weighted` and verifies heap counts `[300,400,500]`. | | `fast_path_merge_refreshes_existing_self_heap_entries` | Fast path merge refreshes self heap entries using merged totals. | Merges fast sketches where `"a-fast"` is updated on both sides (`120` and `40`) and verifies heap count equals merged estimate. | | `default_construction` | Default CountSketchWithHeap constructor uses expected dimensions and heap capacity. | Verifies `CountSketchWithHeap::, RegularPath>::default()` has `rows=3`, `cols=4096`, and `heap.capacity=DEFAULT_HEAP_SIZE`. | | `default_construction_fixed_backends_parity` | Default constructors across storage backends keep intended size/capacity contracts. | Verifies defaults for Fixed/Quick backends are `5x2048`, DefaultMatrix backends are `3x4096`, and all regular/fast variants use `DEFAULT_HEAP_SIZE`. | | `merge_requires_matching_dimensions_panics` | Merge panics on incompatible sketch dimensions. | Verifies merging `CountSketchWithHeap::new(5,256,4)` with `CountSketchWithHeap::new(6,256,4)` panics with dimension-mismatch message. | | `heap_entries_match_cs_estimates_after_mutations` | Every heap entry count matches current sketch estimate after updates and merge. | Checks heap-entry equality to `estimate(key)` both before and after merging another mutated sketch. | -| `bulk_insert_equivalent_to_repeated_insert` | Bulk insert is equivalent to repeated single inserts. | Compares bulk vs repeated insertion on same stream and verifies per-key estimates match within `1e-9` plus identical heap counts for keys `1..5`. | +| `update_batch_equivalent_to_repeated_update` | Bulk insert is equivalent to repeated single inserts. | Compares bulk vs repeated insertion on same stream and verifies per-key estimates match within `1e-9` plus identical heap counts for keys `1..5`. | | `regular_vs_fast_equivalence_on_same_stream` | Regular and fast wrappers agree on identical deterministic stream. | Feeds same 10-item string stream to both paths and verifies per-key estimates match within `1e-9` and heap counts match for `{alpha,beta,gamma,delta,epsilon}`. | | `merge_with_empty_other_and_empty_self` | Merge behavior is stable when one side is empty. | Verifies merging non-empty with empty leaves estimates/heap size unchanged and merging empty-self with non-empty reproduces estimates and heap visibility. | | `duplicate_candidate_keys_during_merge_do_not_corrupt_heap` | Duplicate merge candidates do not duplicate heap entries. | Merges sketches both containing `"dup"`; verifies heap count equals merged estimate, heap size stays within capacity, and only one heap entry exists for `"dup"`. | | `zipf_stream_top_k_recall_regular_fast_budget` | Regular path heap achieves high top-k recall on Zipf stream. | On Zipf stream (`rows=5`, `cols=4096`, `heap_size=16`, `domain=1024`, `exponent=1.1`, `N=20_000`), verifies heap size bound, entry-count consistency, and recall hits `>= 15` vs truth top-16. | | `zipf_stream_top_k_recall_fast_path_fast_budget` | Fast path heap achieves high top-k recall on Zipf stream. | Runs same Zipf setup in fast mode and verifies heap size bound, entry-count consistency, and recall hits `>= 15`. | | `zipf_stream_regular_fast_heap_overlap` | Regular and fast heaps substantially overlap on Zipf heavy hitters. | On shared Zipf stream (`heap_size=16`), verifies key overlap ratio between regular and fast top-k heaps is at least `0.8`. | -| `cs_heap_round_trip_serialization` | The envelope frames a sketch under kind_id `0x0a 0x00`, and the bytes are stable across a round trip. | For a `3x8` regular-path `i64` sketch with a heap of `4` holding three weighted `u64` keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x0a 0x00`, that the metadata carries `3x8`, `counter_type` `i64`, `mode` `regular`, `heap_size` `4`, and `key_type` `u64`, that dimensions, counters, heap length and capacity, and per-key estimates and heap counts all match, and that re-serializing reproduces the bytes. | -| `cs_heap_negative_counters_round_trip` | Signed cells reach the wire and come back unchanged. | Round-trips a `2x4` matrix of alternating positive and negative counters beside a `Str` heap key, and verifies the decoded slice equals the source, still holds a negative cell, keeps the heap key, and re-serializes identically. | -| `cs_heap_i32_round_trips_and_is_pinned_by_counter_type` | The `i32` wire config round-trips and its width is identity, not a detail. | Round-trips a `2x4` `Vector2D` sketch, verifies the metadata names `counter_type` `i32`, then that its bytes differ from the numerically equal `i64` sketch's, that `i32` bytes fail to decode as `i64`, and that `i64` bytes fail to decode as `i32`. | -| `cs_heap_mode_in_metadata_round_trips` | The metadata `mode` pins which column derivation built the sketch. | Round-trips a `4x16` fast-path sketch, verifies the metadata names `mode` `fast` and the counters are preserved, then that the same bytes fail to decode as a `RegularPath` sketch. | -| `cs_heap_rejects_foreign_kind_ids` | The neighbouring sketches' kind_ids are refused. | Verifies `CountMinSketchWithHeap`, `CountMinSketch`, and `CountSketch` envelopes at `3x8` each fail to decode as a `CountSketchWithHeap`. | -| `cs_heap_rejects_zero_dimension_payload` | A zero dimension is a decode error, not a `Vector2D::from_fn` panic. | Verifies a crafted `4x0` envelope fails with a "must be non-zero" complaint. | -| `cs_heap_rejects_dimension_length_mismatch` | The length check fires from the declared dimensions, before any allocation is sized from them. | Verifies a crafted envelope declaring `MATRIX_MAX_ROWS x 2^24` while carrying three counters fails with a "!= rows*cols" complaint. | -| `cs_heap_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies `Vector2D::::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | -| `cs_heap_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the thirteen `HeapMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HeapMetadata`. | -| `cs_heap_metadata_rejects_a_missing_key_type_key` | `key_type` is required, so the heap's key variant can never be silently defaulted. | Encodes the other twelve `HeapMetadata` fields as a named map with `key_type` omitted and verifies it does not decode as `HeapMetadata`. | -| `cs_heap_rejects_f64_into_an_integer_sketch` | An `f64`-labelled envelope decodes into neither integer sketch. | Wraps metadata naming `counter_type: "f64"` around a valid `2x4` payload and verifies both the `i64` and the `i32` sketch reject it. | -| `cs_heap_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x8` sketch with a heap of `4` over a hasher declaring its own `HashProfile`, verifies the counters and both heap entries round-trip, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | -| `cs_heap_refuses_a_heap_size_the_metadata_cannot_carry` | A `heap_size` past the metadata's `u32` field fails the encode rather than truncating. | Verifies a `2x4` sketch built at a heap capacity of `1 << 40` fails to serialize with a message naming the "exceeds the u32 metadata field" rule. | -| `cs_heap_rejects_too_many_rows` | The base matrix carries the seed list's row bound. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | +| `count_sketch_with_heap_round_trip_serialization` | The envelope frames a sketch under kind_id `0x0a 0x00`, and the bytes are stable across a round trip. | For a `3x8` regular-path `i64` sketch with a heap of `4` holding three weighted `u64` keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x0a 0x00`, that the metadata carries `3x8`, `counter_type` `i64`, `mode` `regular`, `heap_size` `4`, and `key_type` `u64`, that dimensions, counters, heap length and capacity, and per-key estimates and heap counts all match, and that re-serializing reproduces the bytes. | +| `count_sketch_with_heap_negative_counters_round_trip` | Signed cells reach the wire and come back unchanged. | Round-trips a `2x4` matrix of alternating positive and negative counters beside a `Str` heap key, and verifies the decoded slice equals the source, still holds a negative cell, keeps the heap key, and re-serializes identically. | +| `count_sketch_with_heap_i32_round_trips_and_is_pinned_by_counter_type` | The `i32` wire config round-trips and its width is identity, not a detail. | Round-trips a `2x4` `Vector2D` sketch, verifies the metadata names `counter_type` `i32`, then that its bytes differ from the numerically equal `i64` sketch's, that `i32` bytes fail to decode as `i64`, and that `i64` bytes fail to decode as `i32`. | +| `count_sketch_with_heap_mode_in_metadata_round_trips` | The metadata `mode` pins which column derivation built the sketch. | Round-trips a `4x16` fast-path sketch, verifies the metadata names `mode` `fast` and the counters are preserved, then that the same bytes fail to decode as a `RegularPath` sketch. | +| `count_sketch_with_heap_rejects_foreign_kind_ids` | The neighbouring sketches' kind_ids are refused. | Verifies `CountMinSketchWithHeap`, `CountMinSketch`, and `CountSketch` envelopes at `3x8` each fail to decode as a `CountSketchWithHeap`. | +| `count_sketch_with_heap_rejects_zero_dimension_payload` | A zero dimension is a decode error, not a `Vector2D::from_fn` panic. | Verifies a crafted `4x0` envelope fails with a "must be non-zero" complaint. | +| `count_sketch_with_heap_rejects_dimension_length_mismatch` | The length check fires from the declared dimensions, before any allocation is sized from them. | Verifies a crafted envelope declaring `MATRIX_MAX_ROWS x 2^24` while carrying three counters fails with a "!= rows*cols" complaint. | +| `count_sketch_with_heap_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies `Vector2D::::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | +| `count_sketch_with_heap_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the thirteen `HeapMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HeapMetadata`. | +| `count_sketch_with_heap_metadata_rejects_a_missing_key_type_key` | `key_type` is required, so the heap's key variant can never be silently defaulted. | Encodes the other twelve `HeapMetadata` fields as a named map with `key_type` omitted and verifies it does not decode as `HeapMetadata`. | +| `count_sketch_with_heap_rejects_f64_into_an_integer_sketch` | An `f64`-labelled envelope decodes into neither integer sketch. | Wraps metadata naming `counter_type: "f64"` around a valid `2x4` payload and verifies both the `i64` and the `i32` sketch reject it. | +| `count_sketch_with_heap_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x8` sketch with a heap of `4` over a hasher declaring its own `HashProfile`, verifies the counters and both heap entries round-trip, the bytes differ from the standard-profile sketch's over the same keys, and a standard-profile decode rejects them. | +| `count_sketch_with_heap_refuses_a_heap_size_the_metadata_cannot_carry` | A `heap_size` past the metadata's `u32` field fails the encode rather than truncating. | Verifies a `2x4` sketch built at a heap capacity of `1 << 40` fails to serialize with a message naming the "exceeds the u32 metadata field" rule. | +| `count_sketch_with_heap_rejects_too_many_rows` | The base matrix carries the seed list's row bound. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | | `decode_rejects_an_index_past_the_matrix` | A `0x0a 0x80` cell index must be below `rows * cols`. | For a `2x4` `i64` delta with heap size `2`, verifies indices `[0, 7]` decode and index `8` fails. | | `decode_rejects_another_kind` | A delta decodes only under its own kind_id. | Verifies the `csheap_i64_regular_2x4_delta` golden decodes and fails with its kind_id relabelled `0x0a 0x00`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the matrix and the heap unchanged. | For a `2x4` state with heap size `2` holding `"kept"`, verifies a `4x2` delta, a delta with heap size `3` and a delta whose last cell overflows `i64::MIN` each fail, leaving the cells and the one-entry heap as they were. | @@ -371,22 +371,22 @@ Wire tests: [`src/sketches/countsketch_l2_norm/wire.rs`](../src/sketches/countsk | test_name | test_description | what_is_tested | | --- | --- | --- | -| `countl2hh_rejects_column_bits_that_reach_the_sign_bits` | A geometry whose column and sign fields overlap panics at construction. | Verifies `CountSketchWithL2Norm::with_dimensions_and_seed_index(11, 2048, 0)` - 132 bits - panics with a message naming the hash bits. | -| `countl2hh_accepts_the_widest_disjoint_geometry` | The widest disjoint geometry is accepted. | Verifies `CountSketchWithL2Norm::with_dimensions_and_seed_index(10, 2048, 0)` - 120 bits - constructs. | -| `countl2hh_estimates_and_l2_are_consistent` | CountSketchWithL2Norm updates keep estimate and L2 consistent. | For `CountSketchWithL2Norm(3x32)`, applies `+5` then `-2` to one key, verifies estimates `5.0` then `3.0`, and asserts non-trivial L2 (`>= 3.0`). | -| `countl2hh_merge_combines_frequency_vectors` | CountSketchWithL2Norm merge combines per-key frequencies. | Merges two `CountSketchWithL2Norm(3x32)` sketches with same key counts `4` and `9`, then verifies merged estimate `== 13.0`. | -| `countl2hh_round_trip_serialization` | CountSketchWithL2Norm serialization round trip preserves estimate and L2. | Serializes/deserializes `CountSketchWithL2Norm::with_dimensions_and_seed_index(3,32,7)` after updates, verifying rows/cols and that both estimate and L2 remain unchanged (within `f64::EPSILON`). | -| `count_l2hh_round_trip_serialization` | The envelope frames a sketch under kind_id `0x19 0x00`, and the state survives a round trip. | For a `3x32` sketch seeded at `7` over three signed weighted inserts, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x19 0x00`, that the metadata carries `seed_index` `7` and `3x32`, and that the decode matches on dimensions, seed index, the counter array, the `l2` accumulators, and `l2`. | -| `count_l2hh_decoded_re_serializes_identically` | A decoded sketch re-serializes byte for byte. | Verifies the populated `3x32` sketch's decode re-encodes to the bytes it came from. | -| `count_l2hh_negative_counters_round_trip` | Signed cells reach the wire and come back unchanged. | Round-trips a `2x8` sketch fed two negative weights and verifies the decoded slice equals the source, still holds a negative cell, and re-serializes identically. | -| `count_l2hh_seed_index_travels_with_the_sketch` | The seed index is state, not a profile constant. | Verifies `2x8` sketches seeded at `0` and at `9` do not share bytes, and that the decoded seed-`9` sketch reports a `seed_index` of `9`. | -| `count_l2hh_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x32` sketch over a hasher declaring its own `HashProfile`, verifies the counter array round-trips, the bytes differ from the standard-profile sketch's over the same insert, and a standard-profile decode rejects them. | -| `count_l2hh_rejects_foreign_kind_ids` | The neighbouring universal-sketch kind_ids are refused. | Verifies `CountSketch`, `UnivMon`, `UnivMonTwoTier`, and `UnivMonQ` envelopes each fail to decode as a `CountSketchWithL2Norm`. | -| `count_l2hh_rejects_crafted_geometry` | Every geometry rule fires before an allocation is sized from it. | Verifies crafted envelopes carrying a zero `cols`, a `MATRIX_MAX_ROWS x 2^24` declaration against three counters, a five-entry `l2` array against two rows, and a negative `l2` accumulator each fail to decode. | -| `count_l2hh_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies a `2x4` sketch whose counts are replaced by `Vector2D::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | -| `l2hh_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the nine `CountSketchWithL2NormMetadata` fields plus a `bogus_field`, and the same fields with `cols` omitted, and verifies neither decodes as `CountSketchWithL2NormMetadata`. | -| `count_l2hh_empty_has_one_encoding` | A sketch holding nothing has exactly one encoding. | Verifies a fresh `3x32` sketch and one cleared after an insert serialize to identical bytes. | -| `countl2hh_rejects_too_many_rows` | CountSketchWithL2Norm carries the seed list's row bound on both sides. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | +| `count_sketch_with_l2_norm_rejects_column_bits_that_reach_the_sign_bits` | A geometry whose column and sign fields overlap panics at construction. | Verifies `CountSketchWithL2Norm::with_dimensions_and_seed_index(11, 2048, 0)` - 132 bits - panics with a message naming the hash bits. | +| `count_sketch_with_l2_norm_accepts_the_widest_disjoint_geometry` | The widest disjoint geometry is accepted. | Verifies `CountSketchWithL2Norm::with_dimensions_and_seed_index(10, 2048, 0)` - 120 bits - constructs. | +| `count_sketch_with_l2_norm_estimates_and_l2_are_consistent` | CountSketchWithL2Norm updates keep estimate and L2 consistent. | For `CountSketchWithL2Norm(3x32)`, applies `+5` then `-2` to one key, verifies estimates `5.0` then `3.0`, and asserts non-trivial L2 (`>= 3.0`). | +| `count_sketch_with_l2_norm_merge_combines_frequency_vectors` | CountSketchWithL2Norm merge combines per-key frequencies. | Merges two `CountSketchWithL2Norm(3x32)` sketches with same key counts `4` and `9`, then verifies merged estimate `== 13.0`. | +| `count_sketch_with_l2_norm_round_trip_preserves_estimate_and_l2` | CountSketchWithL2Norm serialization round trip preserves estimate and L2. | Serializes/deserializes `CountSketchWithL2Norm::with_dimensions_and_seed_index(3,32,7)` after updates, verifying rows/cols and that both estimate and L2 remain unchanged (within `f64::EPSILON`). | +| `count_sketch_with_l2_norm_round_trip_serialization` | The envelope frames a sketch under kind_id `0x19 0x00`, and the state survives a round trip. | For a `3x32` sketch seeded at `7` over three signed weighted inserts, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x19 0x00`, that the metadata carries `seed_index` `7` and `3x32`, and that the decode matches on dimensions, seed index, the counter array, the `l2` accumulators, and `l2`. | +| `count_sketch_with_l2_norm_decoded_re_serializes_identically` | A decoded sketch re-serializes byte for byte. | Verifies the populated `3x32` sketch's decode re-encodes to the bytes it came from. | +| `count_sketch_with_l2_norm_negative_counters_round_trip` | Signed cells reach the wire and come back unchanged. | Round-trips a `2x8` sketch fed two negative weights and verifies the decoded slice equals the source, still holds a negative cell, and re-serializes identically. | +| `count_sketch_with_l2_norm_seed_index_travels_with_the_sketch` | The seed index is state, not a profile constant. | Verifies `2x8` sketches seeded at `0` and at `9` do not share bytes, and that the decoded seed-`9` sketch reports a `seed_index` of `9`. | +| `count_sketch_with_l2_norm_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a `3x32` sketch over a hasher declaring its own `HashProfile`, verifies the counter array round-trips, the bytes differ from the standard-profile sketch's over the same insert, and a standard-profile decode rejects them. | +| `count_sketch_with_l2_norm_rejects_foreign_kind_ids` | The neighbouring universal-sketch kind_ids are refused. | Verifies `CountSketch`, `UnivMon`, `UnivMonTwoTier`, and `UnivMonQ` envelopes each fail to decode as a `CountSketchWithL2Norm`. | +| `count_sketch_with_l2_norm_rejects_crafted_geometry` | Every geometry rule fires before an allocation is sized from it. | Verifies crafted envelopes carrying a zero `cols`, a `MATRIX_MAX_ROWS x 2^24` declaration against three counters, a five-entry `l2` array against two rows, and a negative `l2` accumulator each fail to decode. | +| `count_sketch_with_l2_norm_rejects_serializing_an_unfilled_matrix` | The encode side refuses a matrix its own decoder would reject. | Verifies a `2x4` sketch whose counts are replaced by `Vector2D::unfilled(2, 4)`, which reserves eight cells without filling them, fails to serialize. | +| `count_sketch_with_l2_norm_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the nine `CountSketchWithL2NormMetadata` fields plus a `bogus_field`, and the same fields with `cols` omitted, and verifies neither decodes as `CountSketchWithL2NormMetadata`. | +| `count_sketch_with_l2_norm_empty_has_one_encoding` | A sketch holding nothing has exactly one encoding. | Verifies a fresh `3x32` sketch and one cleared after an insert serialize to identical bytes. | +| `count_sketch_with_l2_norm_rejects_too_many_rows` | CountSketchWithL2Norm carries the seed list's row bound on both sides. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | ### CountMinSketchWithSpaceSaving @@ -457,7 +457,7 @@ The `FoldCell` and `FoldEntry` tests live here because both types are declared i | test_name | test_description | what_is_tested | | --- | --- | --- | | `cell_starts_empty` | A `FoldCell::Empty` answers every query without holding anything. | Verifies `entry_count()` is `0`, `is_empty()` is true, and `get(42)` is `0`. | -| `cell_single_insert` | The first insert seats one entry inline. | After `add(10, 5)` into an `Empty` cell, verifies `entry_count()` is `1`, `get(10)` is `5`, `get(11)` is `0`, and the cell matches `FoldCell::Single`. | +| `cell_single_add` | The first insert seats one entry inline. | After `add(10, 5)` into an `Empty` cell, verifies `entry_count()` is `1`, `get(10)` is `5`, `get(11)` is `0`, and the cell matches `FoldCell::Single`. | | `cell_single_accumulates` | A repeat of the same `full_col` adds in place rather than upgrading. | After `add(10, 5)` then `add(10, 3)`, verifies `entry_count()` is still `1`, `get(10)` is `8`, and the cell still matches `FoldCell::Single`. | | `cell_collision_upgrades_to_collided` | A second distinct `full_col` is what upgrades the cell. | After `add(10, 5)` then `add(42, 3)`, verifies `entry_count()` is `2`, the cell matches `FoldCell::Collided`, and `get(10)` and `get(42)` read `5` and `3`. | | `cell_collided_accumulates` | A collided cell adds into the matching entry, not a new one. | Inserts `(10, 5)`, `(42, 3)`, `(10, 2)`, `(42, 7)` and verifies `get(10)` is `7`, `get(42)` is `10`, and `entry_count()` is still `2`. | @@ -468,16 +468,16 @@ The `FoldCell` and `FoldEntry` tests live here because both types are declared i | `cell_iter_empty` | An empty cell iterates nothing. | Verifies `FoldCell::Empty.iter().count()` is `0`; that single assertion is the whole test. | | `cell_iter_single` | An inline entry iterates as one pair. | For a cell holding `7` at `99`, verifies the collected items are exactly `[(7, 99)]`. | | `cell_iter_collided` | A collided cell iterates every pair it holds. | For a cell holding `7` at `10` and `15` at `20`, verifies the collected items, once sorted, are exactly `[(7, 10), (15, 20)]`. | -| `fold_cms_dimensions` | The fold level fixes the physical column count. | For `CountMinSketchFold::new(3, 4096, 4, 10)`, verifies `rows()` is `3`, `full_cols()` is `4096`, `fold_cols()` is `256` (`4096 / 2^4`), and `fold_level()` is `4`. | -| `fold_cms_level_zero_is_full` | `unfolded` builds a level-zero, full-width sketch. | For `CountMinSketchFold::unfolded(3, 1024, 10)`, verifies `fold_cols()` is `1024` and `fold_level()` is `0`. | -| `fold_cms_rejects_non_power_of_two` | A `full_cols` that is not a power of two is refused at construction. | Verifies `CountMinSketchFold::new(3, 1000, 0, 10)` panics with "full_cols must be a power of two". | -| `fold_cms_rejects_excessive_fold_level` | A fold level past the column width is refused at construction. | Verifies `CountMinSketchFold::new(3, 256, 9, 10)` panics with a message naming `fold_level`, `256` being `2^8` so `9` is one level too many. | -| `fold_cms_insert_query_single_key` | A weighted insert reads back at its weight. | Inserts `Str("hello")` at `7` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | -| `fold_cms_insert_accumulates` | Two weighted inserts of one key sum. | Inserts `Str("hello")` at `3` then `4` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | -| `fold_cms_absent_key_returns_zero` | An unseen key reads zero. | After inserting `Str("present")` at `10` into a `3x1024` level-4 sketch, verifies `estimate(Str("absent"))` is `0`. | -| `fold_cms_multiple_keys` | Estimates stay one-sided across a hundred weighted keys. | Inserts `U64(i)` at weight `i` for `i` in `0..100` into a `3x4096` level-4 sketch and verifies each estimate is at or above its true count; no upper bound is asserted. | -| `fold_cms_matches_standard_cms_exact` | A folded sketch answers exactly as the full-width CMS it targets. | For `rows=3`, `cols=256`, `fold_level=3` (32 physical columns) over `I32(0..50)` inserted once each, verifies every `estimate` equals the `CountMinSketch, RegularPath>::with_dimensions(3, 256)` estimate, and that `to_flat_counters()` has the same length as the standard sketch's storage slice and equals it cell for cell. | -| `fold_cms_matches_standard_cms_insert_many` | Weighted inserts match the full-width CMS's `update_weighted`. | For `rows=3`, `cols=512`, `fold_level=4` over `U64(0..30)` at weight `i + 1`, verifies every `estimate` equals the standard sketch's estimate after the same `update_weighted` calls. | +| `count_min_sketch_fold_dimensions` | The fold level fixes the physical column count. | For `CountMinSketchFold::new(3, 4096, 4, 10)`, verifies `rows()` is `3`, `full_cols()` is `4096`, `fold_cols()` is `256` (`4096 / 2^4`), and `fold_level()` is `4`. | +| `count_min_sketch_fold_level_zero_is_full` | `unfolded` builds a level-zero, full-width sketch. | For `CountMinSketchFold::unfolded(3, 1024, 10)`, verifies `fold_cols()` is `1024` and `fold_level()` is `0`. | +| `count_min_sketch_fold_rejects_non_power_of_two` | A `full_cols` that is not a power of two is refused at construction. | Verifies `CountMinSketchFold::new(3, 1000, 0, 10)` panics with "full_cols must be a power of two". | +| `count_min_sketch_fold_rejects_excessive_fold_level` | A fold level past the column width is refused at construction. | Verifies `CountMinSketchFold::new(3, 256, 9, 10)` panics with a message naming `fold_level`, `256` being `2^8` so `9` is one level too many. | +| `count_min_sketch_fold_update_estimate_single_key` | A weighted insert reads back at its weight. | Inserts `Str("hello")` at `7` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | +| `count_min_sketch_fold_update_accumulates` | Two weighted inserts of one key sum. | Inserts `Str("hello")` at `3` then `4` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | +| `count_min_sketch_fold_absent_key_returns_zero` | An unseen key reads zero. | After inserting `Str("present")` at `10` into a `3x1024` level-4 sketch, verifies `estimate(Str("absent"))` is `0`. | +| `count_min_sketch_fold_multiple_keys` | Estimates stay one-sided across a hundred weighted keys. | Inserts `U64(i)` at weight `i` for `i` in `0..100` into a `3x4096` level-4 sketch and verifies each estimate is at or above its true count; no upper bound is asserted. | +| `count_min_sketch_fold_matches_standard_cms_exact` | A folded sketch answers exactly as the full-width CMS it targets. | For `rows=3`, `cols=256`, `fold_level=3` (32 physical columns) over `I32(0..50)` inserted once each, verifies every `estimate` equals the `CountMinSketch, RegularPath>::with_dimensions(3, 256)` estimate, and that `to_flat_counters()` has the same length as the standard sketch's storage slice and equals it cell for cell. | +| `count_min_sketch_fold_matches_standard_cms_update_weighted` | Weighted inserts match the full-width CMS's `update_weighted`. | For `rows=3`, `cols=512`, `fold_level=4` over `U64(0..30)` at weight `i + 1`, verifies every `estimate` equals the standard sketch's estimate after the same `update_weighted` calls. | | `same_level_merge_adds_counts` | A same-level merge sums the two sides' counts. | Inserts `Str("user_001")` at `100` into one `3x1024` level-3 sketch and at `200` into another, then verifies `merge_same_level` leaves `estimate` at `300`. | | `same_level_merge_matches_standard_cms_merge` | A same-level merge answers as the merged full-width CMS. | For `rows=3`, `cols=512`, `fold_level=4` with `U64(0..20)` on the left and `U64(10..30)` on the right inserted once each, verifies every `estimate` after `merge_same_level` equals the estimate after `CountMinSketch::merge` for keys `0..30`. | | `unfold_merge_reduces_level` | `unfold_merge` drops the fold level by one. | For two empty `3x1024` level-3 sketches, verifies the result reports `fold_level()` `2` and `fold_cols()` `cols >> 2`; nothing is inserted, so only the geometry is checked. | @@ -491,7 +491,7 @@ The `FoldCell` and `FoldEntry` tests live here because both types are declared i | `heap_tracks_heavy_hitters` | The top-k heap carries the heaviest key at its estimate. | Inserts `Str("heavy")` 100 times, `Str("medium")` 10 times, and `Str("light")` once into a `3x1024` level-3 sketch with `heap_size=5`, then verifies the heap is not empty and its `"heavy"` entry reads `100`; the other two keys' entries are not checked. | | `heap_survives_same_level_merge` | A same-level merge refreshes the heap to the merged estimate. | Inserts `Str("user_x")` 50 times into one `3x1024` level-3 `heap_size=5` sketch and 70 times into another, then verifies the heap entry after `merge_same_level` reads `120`. | | `heap_survives_unfold_merge` | An unfolding merge carries the heap forward at the merged estimate. | Inserts `Str("endpoint_a")` 40 times into one `3x512` level-2 `heap_size=5` sketch and 60 times into another, then verifies the merged sketch's heap entry reads `100`. | -| `fold_cms_error_bound_zipf` | Folding does not widen the CMS error bound on a Zipf stream. | For a `3x4096` level-4 sketch with `heap_size=20` fed 200,000 draws over an 8,192-key domain at exponent `1.1` and seed `0x5eed_c0de`, counts the keys whose absolute error is under `(e / 4096) * 200_000` and requires that count to exceed `truth.len() * (1 - e^-3)`. | +| `count_min_sketch_fold_error_bound_zipf` | Folding does not widen the CMS error bound on a Zipf stream. | For a `3x4096` level-4 sketch with `heap_size=20` fed 200,000 draws over an 8,192-key domain at exponent `1.1` and seed `0x5eed_c0de`, counts the keys whose absolute error is under `(e / 4096) * 200_000` and requires that count to exceed `truth.len() * (1 - e^-3)`. | | `scenario_rate_limiting` | Per-user counts merge exactly across two epochs. | Merges two `3x4096` level-4 epoch sketches holding `user_001` at `350` and `350`, `user_002` at `10` and `5`, and `user_003` at `600` and `700`, and verifies `merge_same_level` leaves queries of exactly `700`, `15`, and `1300`. | | `scenario_error_frequency` | Per-endpoint error counts merge exactly across two epochs. | Merges two `3x4096` level-4 epoch sketches over four endpoint strings and verifies queries of exactly `350` for `/api/v1/search`, `210` for `/api/v1/login`, `101` for `/api/v2/recommend`, and `10` for `/api/v1/checkout`. | | `scenario_ddos_detection` | Three epochs — not a power of two — merge and still separate the threshold. | `hierarchical_merge` over three `3x4096` level-4 epoch sketches of four IP strings is verified to leave `10.0.0.42` at exactly `37_000`, above the `15_000` threshold, and `172.16.5.99` at exactly `9_055` and `10.0.0.43` at exactly `8_300`, both below it. | @@ -510,18 +510,18 @@ The shared `FoldCell` and `FoldEntry` behaviour is covered by the cell tests in | test_name | test_description | what_is_tested | | --- | --- | --- | -| `fold_cs_dimensions` | The fold level fixes the physical column count. | For `CountSketchFold::new(3, 4096, 4, 10)`, verifies `rows()` is `3`, `full_cols()` is `4096`, `fold_cols()` is `256` (`4096 / 2^4`), and `fold_level()` is `4`. | -| `fold_cs_level_zero_is_full` | `unfolded` builds a level-zero, full-width sketch. | For `CountSketchFold::unfolded(3, 1024, 10)`, verifies `fold_cols()` is `1024` and `fold_level()` is `0`. | -| `fold_cs_rejects_non_power_of_two` | A `full_cols` that is not a power of two is refused at construction. | Verifies `CountSketchFold::new(3, 1000, 0, 10)` panics with "full_cols must be a power of two". | -| `fold_cs_rejects_excessive_fold_level` | A fold level past the column width is refused at construction. | Verifies `CountSketchFold::new(3, 256, 9, 10)` panics with a message naming `fold_level`, `256` being `2^8` so `9` is one level too many. | -| `fold_cs_insert_query_single_key` | A weighted insert reads back at its weight through the median. | Inserts `Str("hello")` at `7` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | -| `fold_cs_insert_accumulates` | Two weighted inserts of one key sum. | Inserts `Str("hello")` at `3` then `4` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | -| `fold_cs_absent_key_returns_zero` | An unseen key reads zero. | After inserting `Str("present")` at `10` into a `3x1024` level-4 sketch, verifies `estimate(Str("absent"))` is `0`. | -| `fold_cs_multiple_keys` | Estimates stay close on both sides across a hundred weighted keys. | Inserts `U64(i)` at weight `i` for `i` in `0..100` into a `3x4096` level-4 sketch and verifies every estimate's absolute error is at most `10`, the signed estimator being two-sided. | -| `fold_cs_sign_application` | The per-row `±1` sign really reaches the stored counters. | Inserts 50 distinct `U64` keys at weight `1` into a `5x1024` level-4 sketch, walks every cell's entries, and verifies at least one count is positive and at least one is negative; no particular sign or cell is pinned. | -| `fold_cs_matches_standard_cs_exact` | A folded sketch answers exactly as the full-width Count Sketch it targets. | For `rows=3`, `cols=256`, `fold_level=3` (32 physical columns) over `I32(0..50)` inserted once each, verifies every `estimate` equals the `CountSketch, RegularPath>::with_dimensions(3, 256)` estimate cast to `i64`. | -| `fold_cs_matches_standard_cs_flat_counters` | The flat counter view is the full-width signed matrix. | For the same `3x256` level-3 sketch over `I32(0..50)`, verifies `to_flat_counters()` has the same length as the standard sketch's storage slice and equals it cell for cell. | -| `fold_cs_matches_standard_cs_insert_many` | Weighted inserts match the full-width Count Sketch's `update_weighted`. | For `rows=3`, `cols=512`, `fold_level=4` over `U64(0..30)` at weight `i + 1`, verifies every `estimate` equals the standard sketch's estimate cast to `i64` after the same `update_weighted` calls. | +| `count_sketch_fold_dimensions` | The fold level fixes the physical column count. | For `CountSketchFold::new(3, 4096, 4, 10)`, verifies `rows()` is `3`, `full_cols()` is `4096`, `fold_cols()` is `256` (`4096 / 2^4`), and `fold_level()` is `4`. | +| `count_sketch_fold_level_zero_is_full` | `unfolded` builds a level-zero, full-width sketch. | For `CountSketchFold::unfolded(3, 1024, 10)`, verifies `fold_cols()` is `1024` and `fold_level()` is `0`. | +| `count_sketch_fold_rejects_non_power_of_two` | A `full_cols` that is not a power of two is refused at construction. | Verifies `CountSketchFold::new(3, 1000, 0, 10)` panics with "full_cols must be a power of two". | +| `count_sketch_fold_rejects_excessive_fold_level` | A fold level past the column width is refused at construction. | Verifies `CountSketchFold::new(3, 256, 9, 10)` panics with a message naming `fold_level`, `256` being `2^8` so `9` is one level too many. | +| `count_sketch_fold_update_estimate_single_key` | A weighted insert reads back at its weight through the median. | Inserts `Str("hello")` at `7` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | +| `count_sketch_fold_update_accumulates` | Two weighted inserts of one key sum. | Inserts `Str("hello")` at `3` then `4` into a `3x1024` level-4 sketch and verifies `estimate` is `7`. | +| `count_sketch_fold_absent_key_returns_zero` | An unseen key reads zero. | After inserting `Str("present")` at `10` into a `3x1024` level-4 sketch, verifies `estimate(Str("absent"))` is `0`. | +| `count_sketch_fold_multiple_keys` | Estimates stay close on both sides across a hundred weighted keys. | Inserts `U64(i)` at weight `i` for `i` in `0..100` into a `3x4096` level-4 sketch and verifies every estimate's absolute error is at most `10`, the signed estimator being two-sided. | +| `count_sketch_fold_sign_application` | The per-row `±1` sign really reaches the stored counters. | Inserts 50 distinct `U64` keys at weight `1` into a `5x1024` level-4 sketch, walks every cell's entries, and verifies at least one count is positive and at least one is negative; no particular sign or cell is pinned. | +| `count_sketch_fold_matches_standard_cs_exact` | A folded sketch answers exactly as the full-width Count Sketch it targets. | For `rows=3`, `cols=256`, `fold_level=3` (32 physical columns) over `I32(0..50)` inserted once each, verifies every `estimate` equals the `CountSketch, RegularPath>::with_dimensions(3, 256)` estimate cast to `i64`. | +| `count_sketch_fold_matches_standard_cs_flat_counters` | The flat counter view is the full-width signed matrix. | For the same `3x256` level-3 sketch over `I32(0..50)`, verifies `to_flat_counters()` has the same length as the standard sketch's storage slice and equals it cell for cell. | +| `count_sketch_fold_matches_standard_cs_update_weighted` | Weighted inserts match the full-width Count Sketch's `update_weighted`. | For `rows=3`, `cols=512`, `fold_level=4` over `U64(0..30)` at weight `i + 1`, verifies every `estimate` equals the standard sketch's estimate cast to `i64` after the same `update_weighted` calls. | | `same_level_merge_adds_counts` | A same-level merge sums the two sides' counts. | Inserts `Str("user_001")` at `100` into one `3x1024` level-3 sketch and at `200` into another, then verifies `merge_same_level` leaves `estimate` at `300`. | | `same_level_merge_matches_standard_cs_merge` | A same-level merge answers as the merged full-width Count Sketch. | For `rows=3`, `cols=512`, `fold_level=4` with `U64(0..20)` on the left and `U64(10..30)` on the right inserted once each, verifies every `estimate` after `merge_same_level` equals the estimate after `CountSketch::merge` for keys `0..30`. | | `unfold_merge_reduces_level` | `unfold_merge` drops the fold level by one. | For two empty `3x1024` level-3 sketches, verifies the result reports `fold_level()` `2` and `fold_cols()` `cols >> 2`; nothing is inserted, so only the geometry is checked. | @@ -535,7 +535,7 @@ The shared `FoldCell` and `FoldEntry` behaviour is covered by the cell tests in | `heap_tracks_heavy_hitters` | The top-k heap carries the heaviest key at its estimate. | Inserts `Str("heavy")` 100 times, `Str("medium")` 10 times, and `Str("light")` once into a `3x1024` level-3 sketch with `heap_size=5`, then verifies the heap is not empty and its `"heavy"` entry reads `100`; the other two keys' entries are not checked. | | `heap_survives_same_level_merge` | A same-level merge refreshes the heap to the merged estimate. | Inserts `Str("user_x")` 50 times into one `3x1024` level-3 `heap_size=5` sketch and 70 times into another, then verifies the heap entry after `merge_same_level` reads `120`. | | `heap_survives_unfold_merge` | An unfolding merge carries the heap forward at the merged estimate. | Inserts `Str("endpoint_a")` 40 times into one `3x512` level-2 `heap_size=5` sketch and 60 times into another, then verifies the merged sketch's heap entry reads `100`. | -| `fold_cs_error_bound_zipf` | Folding does not widen the Count Sketch L2 error bound on a Zipf stream. | For a `3x4096` level-4 sketch with `heap_size=20` fed 200,000 draws over an 8,192-key domain at exponent `1.1` and seed `0x5eed_c0de`, counts the keys whose absolute error is under `sqrt(e / 4096)` times the stream's exact L2 norm and requires that count to exceed `truth.len() * (1 - e^-3)`. | +| `count_sketch_fold_error_bound_zipf` | Folding does not widen the Count Sketch L2 error bound on a Zipf stream. | For a `3x4096` level-4 sketch with `heap_size=20` fed 200,000 draws over an 8,192-key domain at exponent `1.1` and seed `0x5eed_c0de`, counts the keys whose absolute error is under `sqrt(e / 4096)` times the stream's exact L2 norm and requires that count to exceed `truth.len() * (1 - e^-3)`. | | `scatter_merge_matches_standard_cs_n1_to_n8` | The N-way scatter merge is exact for every N from 1 to 8. | For each `n` in `1..=8`, builds `n` `3x1024` level-3 epoch sketches over ten consecutive `U64` keys each, and verifies `hierarchical_merge` reports `fold_level()` `0` and matches the standard `3x1024` Count Sketch estimate for all `10 * n` keys. | | `unfold_to_single_pass_preserves_flat_counters` | A single-pass `unfold_to` is exact at every target level. | For a `3x256` level-4 sketch over `U64(0..40)` at weight `i + 1`, verifies that for target levels `3`, `2`, `1`, and `0` the result reports that level, `fold_cols()` of `cols >> target`, and `to_flat_counters()` identical to the source's. | | `unfold_to_same_level_returns_clone` | `unfold_to` at the current level is a no-op. | For a `3x256` level-3 sketch holding `Str("x")` at `42`, verifies `unfold_to(3)` reports `fold_level()` `3` and queries `42`. | @@ -565,7 +565,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_serde_round_trip_preserves_every_answer` | Serde round trip preserves state and the rebuilt key index. | Round-trips through `rmp-serde` and verifies residency, capacity, total, `min_count`, and per-key estimate/error all match, then confirms the decoded summary still takes inserts against its rebuilt index. | | `an_empty_summary_answers` | An empty summary answers every query without panicking. | Verifies `len`/`total`/`min_count` are zero, point queries return `0`, `is_guaranteed` is false, and `top_k`/`entries` are empty. | | `a_weighted_arrival_matches_repeating_it_under_eviction` | A weighted arrival matches repeating it once the summary is evicting. | Feeds the same Zipf stream key by key into two 64-counter summaries, one arrival at a time and one `update_weighted` per key, verifies the run saturates all 64 counters, and that `total`, `min_count`, and the whole sorted key/count/error entry list agree. | -| `bulk_insert_matches_repeated_inserts_and_a_zero_weight_is_inert` | `update_batch` is the loop it replaces, and a zero weight records nothing. | Verifies a 128-counter summary filled by `update_batch` matches one filled one at a time on `total` and on `top_k(usize::MAX)`, then that `update_weighted` at weight `0` for a resident key and for an absent one leaves `total`, residency, and `top_k` untouched. | +| `update_batch_matches_repeated_updates_and_a_zero_weight_is_inert` | `update_batch` is the loop it replaces, and a zero weight records nothing. | Verifies a 128-counter summary filled by `update_batch` matches one filled one at a time on `total` and on `top_k(usize::MAX)`, then that `update_weighted` at weight `0` for a resident key and for an absent one leaves `total`, residency, and `top_k` untouched. | | `a_zero_capacity_summary_still_answers` | A summary asked for no counters still holds one. | Verifies `with_capacity(0)` reports a capacity of `1`, and that after ten distinct keys residency is `1`, `total` is `10`, and the single held counter reads `10` with an error of `9`. | | `a_merge_into_an_under_full_summary_keeps_the_ceiling_honest` | A merge carries the peer's ceiling even when there are counters to spare. | Merges a one-counter summary that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, then verifies residency is `1` and below capacity, `upper_bound` for the dropped key 7 still covers its true `10`, and `is_guaranteed` is false for the key that survived. | | `a_merge_chain_stays_one_sided_against_the_truth` | A chain of merges over asymmetric capacities stays one-sided. | Shards the stream three ways by position into summaries of capacity 64, 512, and 7, merges them in turn, and verifies the merged `total` equals the stream length, residency stays within capacity, every true key's `upper_bound` covers its true count, every monitored key reads at or above its truth, and every key `is_guaranteed` accepts truly outranks the largest dropped key. | @@ -641,7 +641,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `an_empty_filter_rejects_every_probe` | A fresh filter reports nothing present. | Verifies `is_empty` and a zero insert count, and that probes disjoint from the member set are rejected on both hash paths. | | `the_measured_false_positive_rate_matches_what_the_sizing_predicts` | The delivered rate honours the target and matches `predicted_fpp`. | Across targets `0.1`, `0.01`, and `0.001`, measures the rate over 200,000 disjoint probes and verifies predicted and measured both stay at or under the target and agree within five binomial standard errors. | | `the_fill_based_estimate_tracks_the_measured_rate` | `estimated_fpp` reads the bits set, not the insert count. | Inserts the member set three times and verifies `estimated_fpp` still tracks the measured rate within a quarter either way although the insert count has tripled. | -| `repeated_inserts_leave_the_bits_unchanged` | Fill is a function of the distinct keys alone. | Builds one filter with single inserts and one with duplicated inserts over the same keys and verifies identical `count_ones` and `fill_ratio`. | +| `repeated_updates_leave_the_bits_unchanged` | Fill is a function of the distinct keys alone. | Builds one filter with single inserts and one with duplicated inserts over the same keys and verifies identical `count_ones` and `fill_ratio`. | | `a_union_equals_the_filter_of_the_concatenated_stream` | Merge is an exact union, which is what makes the filter shardable. | Splits a stream across two same-geometry filters, merges them, and verifies the same set bits and insert count as a single-pass filter, plus identical answers on members and non-members alike. | | `sizing_meets_the_target_and_is_one_power_of_two_from_missing_it` | `dimensions_for` is pinned to its contract rather than to its own expression. | For `(n, p)` pairs from `(1, 0.5)` through `(10_000, 1e-12)`, verifies every slice is a power of two and the row count stays within `BLOOM_MAX_SLICES`, that the chosen geometry's `predicted_fpp(n)` meets the target, and that halving `cols` misses it. | | `degenerate_geometries_still_answer` | Degenerate geometries answer rather than panic. | Verifies a one-bit filter answers and reports an `estimated_fpp` of `1.0` once set, and that a single-row filter still holds every member it was given. | @@ -659,7 +659,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `a_fast_path_union_equals_the_filter_of_the_concatenated_stream` | Merge is an exact union on the fast path too. | Splits 10,000 keys by parity across two `7 x 2^14` fast-path filters, merges them, and verifies the same set bits and insert count as a single-pass filter, with every key reported present. | | `merging_filters_of_different_widths_panics` | A merge across slice widths panics. | Verifies merging a `7 x 2^14` filter with a `7 x 2^13` one panics with "bit matrices must have the same dimensions". | | `merging_filters_of_different_slice_counts_panics` | A merge across slice counts panics. | Verifies merging a `7 x 2^14` filter with an `8 x 2^14` one panics with "bit matrices must have the same dimensions". | -| `bulk_insert_matches_inserting_one_at_a_time` | `update_batch` is the loop, not a different filter. | On both hash paths, verifies a filter filled by `update_batch` over 20,000 keys has the same set bits and insert count as one filled key by key. | +| `update_batch_matches_updating_one_at_a_time` | `update_batch` is the loop, not a different filter. | On both hash paths, verifies a filter filled by `update_batch` over 20,000 keys has the same set bits and insert count as one filled key by key. | | `with_capacity_builds_exactly_the_geometry_dimensions_for_reports` | The sizing helper describes the filter `with_capacity` actually builds. | Over `expected_items` `0`, `1`, `17`, `1_000` and `250_000` against targets `0.5`, `0.1`, `0.01`, `1e-4` and `1e-6`, verifies the built filter's rows and columns equal the pair `dimensions_for` reported for the same arguments. | | `the_slice_count_comes_from_the_bit_budget_not_the_target_alone` | The slice count tracks `expected_items` as well as the target rate. | Verifies `dimensions_for` chooses `5` slices at `(1, 0.05)`, `8` at `(1, 1/128)` and `14` at `(2, 1e-4)`, each exactly one over `round(log2(1/p))`; that the two agree for `expected_items` `1_000`, `100_000` and `10_000_000` across six targets; and that `(1_000_000, 1e-12)` saturates at `BLOOM_MAX_SLICES`. | | `a_target_outside_the_open_unit_interval_is_clamped_to_the_endpoints` | A target outside `(0, 1)` clamps onto the interval's own endpoints. | Verifies `0.0`, `-0.0`, `-1.0` and `f64::MIN` all size as `f64::MIN_POSITIVE` does, that `1.0`, `1.5` and `f64::MAX` all size as `1.0 - f64::EPSILON` does, and that both endpoint geometries are non-zero with a power-of-two width. | @@ -672,7 +672,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `a_serde_payload_past_the_slice_cap_is_rejected` | The plain serde form is a second door into the filter and fails closed on the same bound. | Encodes the serialized form over a `BLOOM_MAX_SLICES + 1` row grid and verifies the `rmp-serde` decode returns an error naming `BLOOM_MAX_SLICES` rather than panicking inside the decoder. | | `both_rate_reporters_raise_the_per_slice_rate_to_the_slice_count` | Both rate reporters use the slice count as the exponent. | For a `BLOOM_MAX_SLICES x 1024` filter holding 400 keys at a fill strictly inside `(0, 1)`, verifies `estimated_fpp` equals `fill_ratio^rows` and `predicted_fpp(400)` equals `(1 - e^(-400/cols))^rows`. | | `an_empty_filter_predicts_nothing_and_a_saturated_one_predicts_everything` | The two ends of the rate scale are reported exactly. | Verifies an empty filter reports a zero fill ratio, a zero `estimated_fpp` and `predicted_fpp(0) == 0`, then fills a `3x8` filter until its fill ratio is `1.0` and verifies `estimated_fpp` is exactly `1.0`. | -| `a_single_insert_sets_exactly_one_bit_in_every_slice` | Partitioning: one key touches exactly one bit per slice, whatever the width. | Across widths `1`, `2`, `7`, `64`, `100` and `1024` by row counts `1`, `5` and `BLOOM_MAX_SLICES`, verifies one insert sets exactly `rows` bits on both hash paths, that `fill_ratio` equals `rows / (rows * cols)`, and that the key reads present. | +| `a_single_update_sets_exactly_one_bit_in_every_slice` | Partitioning: one key touches exactly one bit per slice, whatever the width. | Across widths `1`, `2`, `7`, `64`, `100` and `1024` by row counts `1`, `5` and `BLOOM_MAX_SLICES`, verifies one insert sets exactly `rows` bits on both hash paths, that `fill_ratio` equals `rows / (rows * cols)`, and that the key reads present. | | `the_insert_counter_counts_calls_and_clear_returns_it_to_zero` | `inserted` counts calls rather than distinct keys, and `clear` resets it with the bits. | Verifies five inserts of one key count `5`, a `update_batch` of three more leaves `8`, and that `clear` returns the count, the bits and the fill ratio to zero. | | `merging_identical_geometries_sums_the_insert_counts` | A merge adds the counts and keeps both streams' members. | Merges two `5x512` filters holding 30 and 40 distinct keys and verifies the merged count is the sum, all 70 members read present, and the geometry is unchanged. | | `merging_mismatched_geometries_panics_instead_of_unioning_a_prefix` | A cross-geometry union panics rather than answering no about its own members. | Verifies merging a `5x512` filter with a `5x256` one panics in `BitMatrix::union_from`. | @@ -706,8 +706,8 @@ Wire tests: [`src/sketches/elastic/wire.rs`](../src/sketches/elastic/wire.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `init_with_dimensions_sizes_both_parts` | Both parts take the requested dimensions. | `with_heavy_and_light(12, 2, 256)` is verified to yield 12 heavy buckets and a 2x256 light layer. | -| `init_with_length_keeps_the_default_light_layer` | `with_heavy_buckets` sizes the heavy table alone and leaves the light layer at its defaults. | `with_heavy_buckets(8)` is verified to yield 8 heavy buckets over `DEFAULT_LIGHT_ROWS` x `DEFAULT_LIGHT_COLS`. | +| `with_heavy_and_light_sizes_both_parts` | Both parts take the requested dimensions. | `with_heavy_and_light(12, 2, 256)` is verified to yield 12 heavy buckets and a 2x256 light layer. | +| `with_heavy_buckets_keeps_the_default_light_layer` | `with_heavy_buckets` sizes the heavy table alone and leaves the light layer at its defaults. | `with_heavy_buckets(8)` is verified to yield 8 heavy buckets over `DEFAULT_LIGHT_ROWS` x `DEFAULT_LIGHT_COLS`. | | `an_empty_heavy_table_is_rejected` | A zero bucket count panics. | `with_heavy_buckets(0)` is verified to panic with "at least one heavy bucket" rather than divide by zero in the bucket index. | | `a_negative_heavy_table_is_rejected` | A negative bucket count panics. | `with_heavy_buckets(-1)` is verified to panic rather than widen to a huge `usize`. | | `an_empty_light_layer_is_rejected` | A zero-row light layer panics. | `with_heavy_and_light(8, 0, 4096)` is verified to panic with "non-empty light layer". | @@ -718,7 +718,7 @@ Wire tests: [`src/sketches/elastic/wire.rs`](../src/sketches/elastic/wire.rs) | `expansion_doubles_the_heavy_table` | Copy operation doubles bucket count. | After `expand_heavy()` on an 8-bucket table, verifies `heavy_buckets` and `heavy.len()` are both `16`. | | `expansion_preserves_every_existing_estimate` | Lemma 3.2 keeps estimates put across a doubling. | Records `estimate` for 24 flows over 8 buckets, doubles, and verifies every estimate is unchanged. | | `repeated_expansion_keeps_estimates_intact` | Two doublings in a row stay correct. | Same 24 flows through two `expand_heavy()` calls to `heavy_buckets = 32`, all estimates unchanged. | -| `an_insert_onto_a_stale_copy_replaces_it` | Incremental cleanup drops stale copies. | Finds a stale bucket after a doubling, inserts a key that hashes to it, and verifies the arrival takes the slot at `vote_pos = 1` while the displaced flow keeps its mass elsewhere. | +| `an_update_onto_a_stale_copy_replaces_it` | Incremental cleanup drops stale copies. | Finds a stale bucket after a doubling, inserts a key that hashes to it, and verifies the arrival takes the slot at `vote_pos = 1` while the displaced flow keeps its mass elsewhere. | | `merge_after_expansion_does_not_double_count` | Flushing an expanded table spills each flow once. | Doubles a 24-flow table, merges an empty peer, and verifies every estimate equals its true count rather than twice it. | | `merge_does_not_double_count_an_expanded_peer` | The peer's stale copies are skipped too. | Merges an expanded 24-flow peer into an unexpanded sketch and verifies no estimate doubles. | | `maximum_merging_does_not_double_count_an_expanded_peer` | Same for max merging. | As above through `merge_max`. | @@ -729,11 +729,11 @@ Wire tests: [`src/sketches/elastic/wire.rs`](../src/sketches/elastic/wire.rs) | `a_ratio_that_does_not_divide_the_table_is_rejected` | Lemma 3.2 needs `w' \| w`. | `compress_heavy(3)` on an 8-bucket table panics with "must divide the bucket count". | | `compression_after_expansion_does_not_double_count` | Stale copies are dropped before grouping. | Expands 12 buckets to 24 — putting each twin 12 apart — compresses by 3 so twins land in different groups, verifies no flow is resident twice, then merges an empty peer and verifies every estimate equals its true count. | | `expand_then_compress_returns_to_the_original_size` | Doubling and halving round-trips. | `expand_heavy()` then `compress_heavy(2)` returns an 8-bucket table to 8 buckets with no flow underestimated. | -| `heavy_only_insert_never_touches_the_light_layer` | Overload mode leaves every light counter alone. | Seeds the light layer through `update`, snapshots all `2x64` counters, then runs `update_heavy_only` across a vacant seat, a match, discarded arrivals, and a takeover, and verifies the snapshot is unchanged. | +| `heavy_only_update_never_touches_the_light_layer` | Overload mode leaves every light counter alone. | Seeds the light layer through `update`, snapshots all `2x64` counters, then runs `update_heavy_only` across a vacant seat, a match, discarded arrivals, and a takeover, and verifies the snapshot is unchanged. | | `heavy_only_takeover_inherits_the_evicted_flow_size` | Takeover carries the evicted flow's size to the arrival. | After 10 resident votes and `LAMBDA * 10` colliding arrivals, verifies the bucket becomes `(arrival, vote_pos=10, vote_neg=0)` rather than starting at `1`. | | `heavy_only_takeover_inherits_the_eviction_flag` | Takeover carries the bucket's flag to the arrival. | Seeds a resident bucket's `eviction` to `false` and to `true` in turn, drives a takeover through `update_heavy_only`, and verifies the arrival reads back the seeded value rather than a forced `true`. | | `heavy_only_takeover_discards_the_evicted_flow_as_designed` | The evicted flow's size is dropped, not spilled. | In the same scenario, verifies `estimate` on the evicted flow returns `0` against a true count of `10`. | -| `heavy_only_matches_insert_while_buckets_seat_and_match` | Seating and matching agree with the normal path. | Feeds 6 flows into a 16-bucket table through both paths and verifies every bucket field and every `estimate` result matches. | +| `heavy_only_matches_update_while_buckets_seat_and_match` | Seating and matching agree with the normal path. | Feeds 6 flows into a 16-bucket table through both paths and verifies every bucket field and every `estimate` result matches. | | `merge_keeps_uncontested_flows_in_the_heavy_part` | Merge leaves elephants in the heavy part. | Verifies post-merge `estimate` returns exactly `30` and `18`, that both flows are still resident with those vote counts, and that every bucket carries the eviction flag. | | `merge_keeps_the_larger_flow_on_a_contested_bucket` | A contested bucket goes to the larger flow. | With a 20-count and a 9-count flow on one bucket, verifies the 20 keeps the bucket and the loser reads back at `>= 9` from the light layer. | | `merge_keeps_the_peers_flow_when_it_is_the_larger` | Each side is sized against its own sketch. | A 3-count local flow loses its bucket to the peer's 50-count flow; querying the peer's flow against the local sketch would read near 0 and flip the outcome. | @@ -778,7 +778,7 @@ Wire tests: [`src/sketches/coco/wire.rs`](../src/sketches/coco/wire.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `insert_then_estimate_matches_full_value_for_partial_key` | Insert then estimate matches full value for partial key. | Core behavior for insert/query/update and deterministic semantics is validated; the substring query and `estimate` both return `5`. | +| `update_then_estimate_matches_full_value_for_partial_key` | Insert then estimate matches full value for partial key. | Core behavior for insert/query/update and deterministic semantics is validated; the substring query and `estimate` both return `5`. | | `estimate_with_udf_allows_custom_partial_matching` | Estimate with udf allows custom partial matching. | Core behavior for insert/query/update and deterministic semantics is validated. | | `tied_minimum_buckets_are_chosen_uniformly_at_random` | Buckets tied at the smallest value are picked uniformly. | Inserts one key into 2,000 fresh `32x4` tables seeded `0..2_000`, where all four mapped buckets tie at `0`, and verifies each row takes between 200 and 800 of the landings rather than row 0 taking all of them. | | `equal_seeds_build_identical_tables` | Two sketches built with one seed hold identical tables. | Splits 20,000 inserts of `flow::{(i*i + 7i) % 512}` by parity between two `64x2` tables seeded `seed` and `seed + 1`, merges them, and verifies two builds at seed `7` agree bucket for bucket and a build at seed `9` does not. | @@ -789,7 +789,7 @@ Wire tests: [`src/sketches/coco/wire.rs`](../src/sketches/coco/wire.rs) | `group_by_preserves_the_inserted_mass` | Grouping conserves the inserted mass under eviction. | Drives 400 inserts of weight 3 through an `8x2` table and verifies the grouped totals still sum to `1200`. | | `group_by_reproduces_the_papers_figure_seven` | Grouping reproduces the paper's worked example. | Groups `19.98.10.26\|80=521`, `19.98.10.26\|443=520`, and `34.52.73.17\|118=856` by srcip and verifies exactly two entries, `1041` and `856`. | | `a_key_occupies_at_most_one_bucket_per_row` | A key never gains a second home in the table. | After 64 inserts of one key into a `32x4` table, verifies exactly one bucket holds it and `estimate` returns `64`. | -| `estimate_key_never_exceeds_the_inserted_mass` | Point queries stay inside the table mass. | Over 500 weighted inserts across 40 keys in an `8x2` table, verifies the table mass equals the inserted mass and no per-key estimate exceeds it. | +| `estimate_never_exceeds_the_inserted_mass` | Point queries stay inside the table mass. | Over 500 weighted inserts across 40 keys in an `8x2` table, verifies the table mass equals the inserted mass and no per-key estimate exceeds it. | | `merge_combines_tables_without_losing_counts` | Merge combines tables without losing counts. | Merge behavior preserves expected aggregate semantics and internal invariants. | | `coco_round_trip_serialization` | The envelope frames a sketch under kind_id `0x0c 0x00`, and the state survives a round trip. | For an `with_dimensions(4, 8)` table holding two weighted keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x0c 0x00`, and that the decode keeps `cols` `8`, `rows` `4`, every bucket's key and value, and an `estimate` of `521`. | | `coco_rejects_foreign_kind_id` | Another sketch carrying `rows`/`cols` metadata is refused on its kind_id. | Verifies a `4x8` Count-Min envelope fails to decode as a `CocoSketch`. | @@ -915,9 +915,9 @@ Test file: [`src/sketch_framework/hashlayer.rs`](../src/sketch_framework/hashlay | test_name | test_description | what_is_tested | | --- | --- | --- | -| `test_insert_and_estimate` | Insert and frequency estimate accuracy on Zipf stream. | On Zipf stream (`N=10_000`, `domain=1000`, `exp=1.5`), builds default 2-sketch ensemble (CMS + Count, `3x4096`) and verifies average relative error for both CMS (index `0`) and Count (index `1`) frequency estimates over sampled keys is below `0.1`. | -| `test_insert_at` | Targeted insert updates only specified indices. | Inserts only at index `[0]` via `update_at`, then verifies CMS at index `0` has a positive estimate while Count at index `1` returns `0.0`. | -| `test_insert_with_hash_matches_insert` | Pre-computed hash insert matches regular insert. | Builds two identical ensembles; one uses `update()`, the other uses `hash_input()` + `update_with_hash()`. Verifies CMS estimates at index `0` are identical for a probe key. | +| `test_update_and_estimate` | Insert and frequency estimate accuracy on Zipf stream. | On Zipf stream (`N=10_000`, `domain=1000`, `exp=1.5`), builds default 2-sketch ensemble (CMS + Count, `3x4096`) and verifies average relative error for both CMS (index `0`) and Count (index `1`) frequency estimates over sampled keys is below `0.1`. | +| `test_update_at_indices` | Targeted insert updates only specified indices. | Inserts only at index `[0]` via `update_at`, then verifies CMS at index `0` has a positive estimate while Count at index `1` returns `0.0`. | +| `test_update_with_hash_matches_update` | Pre-computed hash insert matches regular insert. | Builds two identical ensembles; one uses `update()`, the other uses `hash_input()` + `update_with_hash()`. Verifies CMS estimates at index `0` are identical for a probe key. | | `test_hll_cardinality` | HLL-only ensemble cardinality accuracy. | Builds HLL-only ensemble (`HyperLogLog`), inserts Zipf stream, and verifies cardinality relative error vs true distinct count is `< 0.02`. | | `test_estimate_on_hll_returns_error` | Frequency query on HLL sketch returns error. | Calls `estimate()` on an HLL-only ensemble and verifies it returns `Err`. | | `test_cardinality_on_cms_returns_error` | Cardinality query on CMS sketch returns error. | Calls `cardinality()` on a CMS+Count ensemble and verifies it returns `Err`. | @@ -937,11 +937,11 @@ Wire tests: [`src/sketch_framework/univmon/wire.rs`](../src/sketch_framework/uni | test_name | test_description | what_is_tested | | --- | --- | --- | | `univmon_round_trip_serialization` | Univmon round trip serialization. | After weighted inserts, verifies non-empty serialization and round-trip preservation of configuration fields, `total_weight`, `L1/L2/entropy` (`<1e-6` drift), and cardinality (`< EPSILON` drift). | -| `update_populates_bucket_size_and_heavy_hitters` | Update populates bucket size and heavy hitters. | Inserting one hot key `40` times sets `total_weight=40`, tracks key in heavy-hitter heap with count `>=20`, and yields exact `L1=40` and `cardinality=1`. | +| `update_populates_total_weight_and_heavy_hitters` | Update populates bucket size and heavy hitters. | Inserting one hot key `40` times sets `total_weight=40`, tracks key in heavy-hitter heap with count `>=20`, and yields exact `L1=40` and `cardinality=1`. | | `merge_with_combines_heavy_hitters` | Merge with combines heavy hitters. | Merging sketches with disjoint heavy keys verifies merged left heap contains both contributions (`left=25`, `right=30`) while right heap retains `right=30`. | | `univmon_layers_use_different_seeds` | Univmon layers use different seeds. | Verifies hash outputs for the same key with seed indices `0..3` are all pairwise different. | | `univmon_cardinality_is_positive` | Univmon cardinality is positive. | After inserting `20` distinct flow keys, cardinality estimate is exactly `20.0`. | -| `univmon_bucket_size_tracked_correctly` | Univmon bucket size tracked correctly. | Inserts counts `100`, `200`, `150` for three flows and verifies `total_weight` equals total `450`. | +| `univmon_total_weight_tracked_correctly` | Univmon bucket size tracked correctly. | Inserts counts `100`, `200`, `150` for three flows and verifies `total_weight` equals total `450`. | | `univmon_basic_operation` | Univmon basic operation. | On fixed mixed workload, verifies exact aggregate metrics `cardinality=10.0` and `L1=131.0`. | | `test_statistical_accuracy` | Test statistical accuracy. | On heavy/medium/noise synthetic distribution, verifies relative error for both `L2` and `entropy` is below `0.15`. | | `univmon_random_data_matches_ground_truth_within_configured_tolerance` | Random weighted updates keep every metric inside its own tolerance. | Over `10_000` seeded random weighted updates across 5,000 keys into `new(256, 6, 8192, 16)`, requires relative error against the exact truth map at or under `0.07` for cardinality and at or under `0.05` for `L1`, `L2`, and entropy. | @@ -950,8 +950,8 @@ Wire tests: [`src/sketch_framework/univmon/wire.rs`](../src/sketch_framework/uni | `univmon_empty_has_one_encoding` | An empty pyramid has exactly one encoding. | Verifies a fresh `new(4, 2, 16, 3)` pyramid and one freed after an insert serialize to identical bytes carrying the pinned `EMPTY_KEY_TYPE`, and that the decode holds empty heaps and re-serializes identically. | | `univmon_empty_must_carry_the_u64_key_type` | An empty pyramid carries `key_type` `"u64"`. | Verifies an empty `new(4, 2, 16, 3)` relabelled `"string"` fails to decode with an "empty key set" complaint. | | `layer_seeds_are_distinct_and_never_the_layer_finder` | Every layer hashes at its own seed, never the layer finder's. | For `new(4, 2, 16, UNIVMON_MAX_LAYERS)`, verifies every layer's seed index is inside `SEED_LIST`, differs from `UNIVMON_BOTTOM_LAYER_SEED_INDEX`, and is distinct from every other layer's. | -| `init_univmon_rejects_a_layer_on_the_finder_seed` | A layer at the finder's seed index is refused. | Verifies `new(4, 2, 16, UNIVMON_BOTTOM_LAYER_SEED_INDEX + 1)` panics naming `UNIVMON_MAX_LAYERS`. | -| `init_univmon_rejects_column_bits_that_reach_the_sign_bits` | A layer geometry past the hash budget is refused. | Verifies `new(4, 11, 2048, 2)` panics naming the hash bits. | +| `new_rejects_a_layer_on_the_bottom_layer_seed_index` | A layer at the finder's seed index is refused. | Verifies `new(4, 2, 16, UNIVMON_BOTTOM_LAYER_SEED_INDEX + 1)` panics naming `UNIVMON_MAX_LAYERS`. | +| `new_rejects_column_bits_that_reach_the_sign_bits` | A layer geometry past the hash budget is refused. | Verifies `new(4, 11, 2048, 2)` panics naming the hash bits. | | `univmon_pins_its_hash_profile` | UnivMon hashes through the crate default, so it has one truthful profile. | Verifies a populated pyramid emits `DefaultXxHasher`'s profile id and seed list, and that the same payload re-framed under a custom profile is different bytes that fail to decode. | | `univmon_rejects_foreign_kind_ids` | The neighbouring universal-sketch kind_ids are refused. | Verifies `CountSketch`, `CountSketchWithL2Norm`, `UnivMonTwoTier`, and `UnivMonQ` envelopes each fail to decode as a `UnivMon`. | | `univmon_rejects_crafted_shapes` | Every shape rule fires before an allocation is sized from it. | Verifies six crafted metadata shapes fail - a `layers` of `u32::MAX` or `0`, a zero `cols` or `heap_size`, a `MATRIX_MAX_ROWS x 4096` sketch, and a `heap_size` of `1` - and that a short `heap_counts`, a short `candidate_complete`, and an `update_mode` of `7` each fail too. | @@ -974,26 +974,26 @@ Wire tests: [`src/sketch_framework/univmon_two_tier/wire.rs`](../src/sketch_fram | --- | --- | --- | | `pool_basic_take_put` | Pool basic take put. | Validates pool accounting: initial preallocation (`available=2`, `allocated=2`), on-demand allocation when empty (`allocated=3`), and reuse on put/take without further allocation. | | `pool_free_resets_sketch` | Pool free resets sketch. | Confirms returning a used sketch to pool resets state so retaken sketch has `total_weight=0` and near-zero `L2` in layer `0`. | -| `pyramid_basic_insert_and_query` | Pyramid basic insert and query. | For simple inserts, verifies exact aggregate state with `total_weight=65`, `L1` approximately `65` (`<1e-6`), and `cardinality=3`. | -| `pyramid_fast_insert_matches_standard` | Pyramid fast insert matches standard. | On identical 500-item stream, verifies standard vs fast paths keep identical `total_weight`, with `L1` deviation `<10%` and cardinality deviation `<15%`. | -| `pyramid_two_tier_dimensions` | Pyramid two tier dimensions. | Verifies two-tier layout metadata for configured pyramid (`layers=8`, `elephant_layers=4`). | -| `pyramid_free_resets_state` | Pyramid free resets state. | After bulk inserts, `clear()` resets sketch to empty baseline (`total_weight=0`, layer-0 `L2` approximately `0`). | -| `pyramid_merge_combines_data` | Pyramid merge combines data. | Merging disjoint halves verifies merged `L1` stays within `10%` of the sum of pre-merge `L1` values. | -| `pyramid_accuracy_zipf` | Pyramid accuracy Zipf. | On heavy/medium/light Zipf-like workload, requires relative error `<15%` for `L1`, `L2`, cardinality, and entropy. | -| `pyramid_fast_insert_accuracy` | Pyramid fast insert accuracy. | Using `update_weighted_terminal` only, requires relative error `<15%` for `L1`, `L2`, cardinality, and entropy versus exact frequency map. | -| `pyramid_memory_savings_vs_uniform` | Pyramid memory savings vs uniform. | Verifies pyramid column budget is smaller than uniform baseline and computed memory savings exceed `30%`. | -| `pyramid_round_trip_serialization` | The envelope frames a pyramid under kind_id `0x11 0x00`, and the bytes are stable across a round trip. | For `UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4)` fed four weighted string keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x11 0x00`, that the metadata carries that layout and `key_type` `string`, and that the decode matches on `total_weight`, L1, L2, cardinality, entropy, and the candidate flags, re-serializing identically. | -| `pyramid_two_tier_geometry_survives` | The two tiers are derived from the layer's position. | Verifies every decoded layer keeps its tier's dimensions - `3x16` for the two elephant layers and `2x8` for the mouse layers - its own seed index, and its counter array. | -| `pyramid_layers_with_different_heap_loads_round_trip` | Each layer's heap contents survive the round trip. | For `UnivMonTwoTier::new(8, 2, 3, 32, 2, 16, 4)` fed 40 weighted keys, verifies the layers carry different heap loads, that every layer's length and per-key counts match after a decode, and that the bytes re-serialize identically. | -| `pyramid_without_mouse_layers_round_trips` | The one-tier case still round-trips. | For a pyramid whose four layers are all elephants, verifies the decoded layers all read `2x16` and the bytes re-serialize identically. | -| `pyramid_empty_has_one_encoding` | An empty pyramid has exactly one encoding. | Verifies a fresh pyramid and one freed after an insert serialize to identical bytes carrying a `key_type` of `u64`, and that the decode re-serializes identically. | -| `pyramid_empty_must_carry_the_u64_key_type` | An empty two-tier pyramid carries `key_type` `"u64"`. | Verifies an empty `UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4)` relabelled `"string"` fails to decode with an "empty key set" complaint. | -| `pyramid_carries_update_mode_and_candidate_flags` | `update_mode` and `candidate_complete` are state, not derived. | For a terminal-only `UnivMonTwoTier::new(2, 1, 2, 8, 2, 8, 3)` fed 20 keys through `fast_insert`, verifies the payload carries `update_mode` `2` and that the decode preserves the candidate flags and cardinality and re-serializes identically. | -| `pyramid_pins_its_hash_profile` | The pyramid hashes through the crate default, so it has one truthful profile. | Verifies a populated pyramid emits `DefaultXxHasher`'s profile id and seed list, and that the same payload re-framed under a custom profile is different bytes that fail to decode. | -| `pyramid_rejects_foreign_kind_ids` | The neighbouring universal-sketch kind_ids are refused. | Verifies `CountSketch`, `CountSketchWithL2Norm`, `UnivMon`, and `UnivMonQ` envelopes each fail to decode as a `UnivMonTwoTier`. | -| `pyramid_rejects_crafted_shapes` | Every layout rule fires before an allocation is sized from it. | Verifies six crafted metadata layouts fail - a `layers` of `u32::MAX` or `0`, a `heap_size` of `0` or `1`, a zero `elephant_cols`, and a `MATRIX_MAX_ROWS x 4096` mouse tier - and that a short `heap_lens` and an `update_mode` of `9` each fail too. | -| `pyramid_rejects_serializing_an_inconsistent_layout` | The encode side refuses states its own decoder would reject. | Verifies a mouse layer holding the elephant tier's `3x16` dimensions fails to serialize, and that a layer hashing at another layer's seed index fails too. | -| `pyramid_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the fourteen `UnivMonTwoTierMetadata` fields plus a `bogus_field`, and the same fields with `mouse_cols` omitted, and verifies neither decodes as `UnivMonTwoTierMetadata`. | +| `univmon_two_tier_basic_update_and_query` | Pyramid basic insert and query. | For simple inserts, verifies exact aggregate state with `total_weight=65`, `L1` approximately `65` (`<1e-6`), and `cardinality=3`. | +| `univmon_two_tier_update_weighted_terminal_matches_standard` | Pyramid fast insert matches standard. | On identical 500-item stream, verifies standard vs fast paths keep identical `total_weight`, with `L1` deviation `<10%` and cardinality deviation `<15%`. | +| `univmon_two_tier_dimensions` | Pyramid two tier dimensions. | Verifies two-tier layout metadata for configured pyramid (`layers=8`, `elephant_layers=4`). | +| `univmon_two_tier_clear_resets_state` | Pyramid free resets state. | After bulk inserts, `clear()` resets sketch to empty baseline (`total_weight=0`, layer-0 `L2` approximately `0`). | +| `univmon_two_tier_merge_combines_data` | Pyramid merge combines data. | Merging disjoint halves verifies merged `L1` stays within `10%` of the sum of pre-merge `L1` values. | +| `univmon_two_tier_accuracy_zipf` | Pyramid accuracy Zipf. | On heavy/medium/light Zipf-like workload, requires relative error `<15%` for `L1`, `L2`, cardinality, and entropy. | +| `univmon_two_tier_update_weighted_terminal_accuracy` | Pyramid fast insert accuracy. | Using `update_weighted_terminal` only, requires relative error `<15%` for `L1`, `L2`, cardinality, and entropy versus exact frequency map. | +| `univmon_two_tier_memory_savings_vs_uniform` | Pyramid memory savings vs uniform. | Verifies pyramid column budget is smaller than uniform baseline and computed memory savings exceed `30%`. | +| `univmon_two_tier_round_trip_serialization` | The envelope frames a pyramid under kind_id `0x11 0x00`, and the bytes are stable across a round trip. | For `UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4)` fed four weighted string keys, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x11 0x00`, that the metadata carries that layout and `key_type` `string`, and that the decode matches on `total_weight`, L1, L2, cardinality, entropy, and the candidate flags, re-serializing identically. | +| `univmon_two_tier_geometry_survives` | The two tiers are derived from the layer's position. | Verifies every decoded layer keeps its tier's dimensions - `3x16` for the two elephant layers and `2x8` for the mouse layers - its own seed index, and its counter array. | +| `univmon_two_tier_layers_with_different_heap_loads_round_trip` | Each layer's heap contents survive the round trip. | For `UnivMonTwoTier::new(8, 2, 3, 32, 2, 16, 4)` fed 40 weighted keys, verifies the layers carry different heap loads, that every layer's length and per-key counts match after a decode, and that the bytes re-serialize identically. | +| `univmon_two_tier_without_mouse_layers_round_trips` | The one-tier case still round-trips. | For a pyramid whose four layers are all elephants, verifies the decoded layers all read `2x16` and the bytes re-serialize identically. | +| `univmon_two_tier_empty_has_one_encoding` | An empty pyramid has exactly one encoding. | Verifies a fresh pyramid and one freed after an insert serialize to identical bytes carrying a `key_type` of `u64`, and that the decode re-serializes identically. | +| `univmon_two_tier_empty_must_carry_the_u64_key_type` | An empty two-tier pyramid carries `key_type` `"u64"`. | Verifies an empty `UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4)` relabelled `"string"` fails to decode with an "empty key set" complaint. | +| `univmon_two_tier_carries_update_mode_and_candidate_flags` | `update_mode` and `candidate_complete` are state, not derived. | For a terminal-only `UnivMonTwoTier::new(2, 1, 2, 8, 2, 8, 3)` fed 20 keys through `fast_insert`, verifies the payload carries `update_mode` `2` and that the decode preserves the candidate flags and cardinality and re-serializes identically. | +| `univmon_two_tier_pins_its_hash_profile` | The pyramid hashes through the crate default, so it has one truthful profile. | Verifies a populated pyramid emits `DefaultXxHasher`'s profile id and seed list, and that the same payload re-framed under a custom profile is different bytes that fail to decode. | +| `univmon_two_tier_rejects_foreign_kind_ids` | The neighbouring universal-sketch kind_ids are refused. | Verifies `CountSketch`, `CountSketchWithL2Norm`, `UnivMon`, and `UnivMonQ` envelopes each fail to decode as a `UnivMonTwoTier`. | +| `univmon_two_tier_rejects_crafted_shapes` | Every layout rule fires before an allocation is sized from it. | Verifies six crafted metadata layouts fail - a `layers` of `u32::MAX` or `0`, a `heap_size` of `0` or `1`, a zero `elephant_cols`, and a `MATRIX_MAX_ROWS x 4096` mouse tier - and that a short `heap_lens` and an `update_mode` of `9` each fail too. | +| `univmon_two_tier_rejects_serializing_an_inconsistent_layout` | The encode side refuses states its own decoder would reject. | Verifies a mouse layer holding the elephant tier's `3x16` dimensions fails to serialize, and that a layer hashing at another layer's seed index fails too. | +| `univmon_two_tier_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the fourteen `UnivMonTwoTierMetadata` fields plus a `bogus_field`, and the same fields with `mouse_cols` omitted, and verifies neither decodes as `UnivMonTwoTierMetadata`. | ### UnivMon-Q @@ -1058,7 +1058,7 @@ Wire tests: [`src/sketch_framework/eh/wire.rs`](../src/sketch_framework/eh/wire. | --- | --- | --- | | `constructor_infers_merge_norm` | Constructor infers merge norm. | Verifies constructor infers `SketchNorm::L1` for the `CountMinSketch` payload and `SketchNorm::L2` for the `CountSketchWithL2Norm` payload. | | `l1_merge_invariant_same_size` | L1 merge invariant same size. | Under repeated updates with `k=2`, verifies L1 merge policy compacts buckets so `bucket_count < 10`. | -| `l2_merge_invariant_sum_l22` | L2 merge invariant sum l2_squared. | With `k=1` and weighted updates, verifies L2 merge rule keeps bucket count bounded (`bucket_count <= 2`). | +| `l2_merge_invariant_sum_l2_squared` | L2 merge invariant sum l2_squared. | With `k=1` and weighted updates, verifies L2 merge rule keeps bucket count bounded (`bucket_count <= 2`). | | `merge_recomputes_l2_mass` | Merge recomputes L2 mass. | After L2 merges, verifies bounded bucket count (`<=2`) and non-negative recomputed `l2_mass` for every payload bucket. | | `test_basic_insertion_and_query` | Test basic insertion and query. | After one update at `t=100`, verifies single bucket presence, exact min/max timestamps (`100`), and successful interval merge query for `[100,100]`. | | `eh_round_trip_serialization` | The envelope frames a histogram under kind_id `0x13 0x00`, and the state survives a round trip. | For `k = 2` over a 1,000-tick window of Count-Min buckets fed six timestamped updates, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x13 0x00`, and that the decode matches on `window`, `k`, `merge_norm`, every bucket's size, time range, and bit-exact `l2_mass`, and the bucket count. | @@ -1085,23 +1085,23 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | test_name | test_description | what_is_tested | | --- | --- | --- | -| `insert_routes_to_countl2hh_and_univmon` | Insert routes to countl2hh and univmon. | Verifies variant routing by checking `CountSketchWithL2Norm` estimate `>=9` after 9 inserts and `UnivMon` `total_weight=6` after 6 inserts. | -| `count_sketch_insert_and_query_round_trip` | Count insert and query round trip. | Confirms the `CountSketch` variant updates/query path by inserting one key and verifying returned estimate is at least `1.0`. | -| `ddsketch_insert_and_quantile_query_round_trip` | DDSketch insert and quantile query round trip. | Inserts `10,20,30` into DDSketch variant and verifies queried median (`q=0.5`) lies within `[10.0, 30.0]`. | +| `update_routes_to_count_sketch_with_l2_norm_and_univmon` | Insert routes to countl2hh and univmon. | Verifies variant routing by checking `CountSketchWithL2Norm` estimate `>=9` after 9 inserts and `UnivMon` `total_weight=6` after 6 inserts. | +| `count_sketch_update_and_query_round_trip` | Count insert and query round trip. | Confirms the `CountSketch` variant updates/query path by inserting one key and verifying returned estimate is at least `1.0`. | +| `ddsketch_update_and_quantile_query_round_trip` | DDSketch insert and quantile query round trip. | Inserts `10,20,30` into DDSketch variant and verifies queried median (`q=0.5`) lies within `[10.0, 30.0]`. | | `supports_norm_whitelist_is_enforced` | Supports norm whitelist is enforced. | Validates norm capability matrix: `CountMinSketch`/`CountSketch`/`DdSketch` support `L1` only, while `CountSketchWithL2Norm`/`UnivMon` support `L2` only. | -| `eh_sketch_list_round_trip_serialization` | The envelope frames a union under kind_id `0x14 0x00`, and the state survives a round trip. | For a `3x8` fast-path Count-Min variant holding one key, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x14 0x00`, and that the decode keeps a `sketch_type` of `CountMinSketch` and answers that key as the source does. | -| `eh_sketch_list_every_variant_round_trips` | Every variant this build carries round-trips and keeps its type. | For each populated variant, verifies the decode reports the same `sketch_type` and re-serializes to the bytes it came from. | -| `eh_sketch_list_kind_ids_are_build_independent` | The ten nested kind_ids and their registry names are the same in every build. | Verifies `variant_name` maps each of the ten ids to its name and `0xff 0xff` to `None`, and that every populated variant emits the id the table gives it. | -| `eh_sketch_list_rejects_an_experimental_kind_id` | An experimental variant's kind_id is refused without the feature. | Verifies crafted triples carrying the `CocoSketch`, `ElasticSketch`, and `UniformSampling` kind_ids each fail to decode, with a message naming the variant and "experimental" in builds without the feature. | -| `eh_sketch_list_rejects_an_unknown_kind_id` | An unknown kind_id is refused. | Verifies a Count-Min triple relabelled `0xff 0xff` fails with a "not a wire variant" complaint. | -| `eh_sketch_list_rejects_sibling_algorithm_kind_ids` | The nested ids are pinned to one algorithm each. | Verifies an HLL triple relabelled `0x01 0x01` or `0x01 0x03` fails to decode, and that a KLL triple relabelled `0x06 0x01` fails too. | -| `eh_sketch_list_rejects_a_mismatched_kind_id_and_descriptor` | A kind_id that does not match the blocks it carries is refused by the variant's own decoder. | Verifies an HLL triple relabelled with the Count-Min kind_id fails to decode. | -| `eh_sketch_list_rejects_foreign_kind_ids` | A Count-Min envelope and an `ExponentialHistogram` envelope are not union envelopes. | Verifies both fail to decode as an `EhBucketSketch`. | -| `eh_sketch_list_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes `metadata_version` plus a `bogus_field` as a named map and verifies it does not decode as `EhSketchListMetadata`. | -| `eh_sketch_list_metadata_rejects_a_missing_key` | `metadata_version` is required. | Encodes an empty named map and verifies it does not decode as `EhSketchListMetadata`. | -| `eh_sketch_list_rejects_crafted_blocks` | Crafted blocks fail closed with an error, never a panic. | Verifies a Count-Min triple whose descriptor is cut in half, and one whose state is three `0xc1` bytes, each fail to decode. | -| `eh_sketch_list_rejects_a_custom_hash_profile_descriptor` | A descriptor naming a custom hash profile is refused, since the variant's decoder pins the profile of the type it rebuilds. | Verifies a triple built from a custom-profile Count-Min fails to decode. | -| `eh_sketch_list_query_agrees_after_decode` | A decoded union answers a query the way the original did. | For every populated variant, verifies the source and the decode return the same answer for that variant's sample input. | +| `eh_bucket_sketch_round_trip_serialization` | The envelope frames a union under kind_id `0x14 0x00`, and the state survives a round trip. | For a `3x8` fast-path Count-Min variant holding one key, verifies the bytes open with the ASAPv1 magic, a `kind_id_len` of `2`, and `0x14 0x00`, and that the decode keeps a `sketch_type` of `CountMinSketch` and answers that key as the source does. | +| `eh_bucket_sketch_every_variant_round_trips` | Every variant this build carries round-trips and keeps its type. | For each populated variant, verifies the decode reports the same `sketch_type` and re-serializes to the bytes it came from. | +| `eh_bucket_sketch_kind_ids_are_build_independent` | The ten nested kind_ids and their registry names are the same in every build. | Verifies `variant_name` maps each of the ten ids to its name and `0xff 0xff` to `None`, and that every populated variant emits the id the table gives it. | +| `eh_bucket_sketch_rejects_an_experimental_kind_id` | An experimental variant's kind_id is refused without the feature. | Verifies crafted triples carrying the `CocoSketch`, `ElasticSketch`, and `UniformSampling` kind_ids each fail to decode, with a message naming the variant and "experimental" in builds without the feature. | +| `eh_bucket_sketch_rejects_an_unknown_kind_id` | An unknown kind_id is refused. | Verifies a Count-Min triple relabelled `0xff 0xff` fails with a "not a wire variant" complaint. | +| `eh_bucket_sketch_rejects_sibling_algorithm_kind_ids` | The nested ids are pinned to one algorithm each. | Verifies an HLL triple relabelled `0x01 0x01` or `0x01 0x03` fails to decode, and that a KLL triple relabelled `0x06 0x01` fails too. | +| `eh_bucket_sketch_rejects_a_mismatched_kind_id_and_descriptor` | A kind_id that does not match the blocks it carries is refused by the variant's own decoder. | Verifies an HLL triple relabelled with the Count-Min kind_id fails to decode. | +| `eh_bucket_sketch_rejects_foreign_kind_ids` | A Count-Min envelope and an `ExponentialHistogram` envelope are not union envelopes. | Verifies both fail to decode as an `EhBucketSketch`. | +| `eh_bucket_sketch_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes `metadata_version` plus a `bogus_field` as a named map and verifies it does not decode as `EhSketchListMetadata`. | +| `eh_bucket_sketch_metadata_rejects_a_missing_key` | `metadata_version` is required. | Encodes an empty named map and verifies it does not decode as `EhSketchListMetadata`. | +| `eh_bucket_sketch_rejects_crafted_blocks` | Crafted blocks fail closed with an error, never a panic. | Verifies a Count-Min triple whose descriptor is cut in half, and one whose state is three `0xc1` bytes, each fail to decode. | +| `eh_bucket_sketch_rejects_a_custom_hash_profile_descriptor` | A descriptor naming a custom hash profile is refused, since the variant's decoder pins the profile of the type it rebuilds. | Verifies a triple built from a custom-profile Count-Min fails to decode. | +| `eh_bucket_sketch_query_agrees_after_decode` | A decoded union answers a query the way the original did. | For every populated variant, verifies the source and the decode return the same answer for that variant's sample input. | | `nested_ddsketch_survives_a_serde_round_trip` | The nested DDSketch keeps its running state through a serde round trip. | Feeds `10.0`, `20.0`, `30.0`, `40.0` into an `EhBucketSketch::DdSketch(DdSketch::new(0.01))`, round-trips it through `rmp-serde`, and verifies the decode matches on `get_count`, `sum`, `min`, and `max` and that its `q = 0.5` query lands inside `[10.0, 40.0]`. | ### ExponentialHistogramUnivMon @@ -1179,10 +1179,10 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `an_inverted_control_band_is_normalised_rather_than_fatal` | An inverted min/max band collapses onto its floor instead of panicking. | Drives the controller against 500 queued items for 8 rounds with `min_threshold` `32` above `max_threshold` `8`, and verifies tau ends at exactly `32`. | | `the_controller_holds_tau_inside_the_dead_band` | A queue held at target settles tau after the initial transient. | Holds the channel at 10 items for 8 rounds at `target_queue_len` `10`, `alpha` `0.25` and a band of `1..=200`, and verifies tau ends at `32` - one step above its starting `31`, taken on the first evaluation's prediction of `20`, with every later prediction of `10` inside the `[7.5, 12.5]` dead band. | | `the_adaptive_controller_moves_tau_during_a_real_run` | A live controller keeps tau in its band and the parent within `workers * tau`. | Runs 200,000 inputs `U64(i % 4096)` through a `3x1024` `CountMinSketchOctoPlan` on 4 hash-partitioned workers from a starting tau of `60`, at `target_queue_len` `8`, `alpha` `0.25`, a `1us` interval and a band of `4..=120`, and verifies the settled tau is inside `4..=120` and that key `0`'s deficit against a single-threaded reference is between `0` and `4 * OCTO_MAX_PROMOTION_THRESHOLD`; whether and how far tau actually moved is deliberately not asserted. | -| `the_topk_aggregator_rebuilds_the_heavy_hitter_heap_from_worker_keys` | The aggregator builds a heavy-hitter heap out of keyed deltas alone. | Runs `U64(0..4)` at 4,000 occurrences each plus a light tail of `U64(1_000..6_000)` through a `4x2048` `CountMinSketchWithHeapOctoPlan` at a heap of `8` on 4 workers, and verifies the heap is non-empty and finds each of the four hot keys. | +| `the_count_min_sketch_with_heap_aggregator_rebuilds_the_heavy_hitter_heap_from_worker_keys` | The aggregator builds a heavy-hitter heap out of keyed deltas alone. | Runs `U64(0..4)` at 4,000 occurrences each plus a light tail of `U64(1_000..6_000)` through a `4x2048` `CountMinSketchWithHeapOctoPlan` at a heap of `8` on 4 workers, and verifies the heap is non-empty and finds each of the four hot keys. | | `a_parent_wider_than_its_workers_estimates_zero` | A parent geometry wider than the workers' reads zero, and `plan.aggregator()` is how not to hit it. | Runs 40,000 inputs `U64(i % 64)` through a `3x1024` `CountMinSketchOctoPlan` on 4 workers into a `5x1024` parent and verifies key `0` estimates `0`, then that the same run into `plan.aggregator()` estimates at least `600`. | -| `a_key_below_the_threshold_never_reaches_the_topk_aggregator` | A key that never reaches tau is absent from the parent and from the heap entirely. | For the 20 keys `U64(0..20)` over a `3x1024` `CountMinSketchWithHeapOctoPlan` at a heap of `32` on 4 workers, first asserts every key's `(worker, row, col)` cells are collision-free, then verifies that at `OCTO_COUNT_MIN_DEFAULT_THRESHOLD - 1` occurrences each the heap is empty and key `0` estimates `0`, while at `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` occurrences the heap holds all `20` keys and key `0` estimates at least `OCTO_COUNT_MIN_DEFAULT_THRESHOLD`. | -| `the_count_topk_aggregator_also_tracks_heavy_hitters` | The Count top-k aggregator keeps a heap too. | Runs `U64(0..3)` at 5,000 occurrences each plus a tail of `U64(500..3_000)` through a `5x2048` `CountSketchWithHeapOctoPlan` at a heap of `8` on 4 workers, and verifies the heap finds each of the three hot keys. | +| `a_key_below_the_threshold_never_reaches_the_count_min_sketch_with_heap_aggregator` | A key that never reaches tau is absent from the parent and from the heap entirely. | For the 20 keys `U64(0..20)` over a `3x1024` `CountMinSketchWithHeapOctoPlan` at a heap of `32` on 4 workers, first asserts every key's `(worker, row, col)` cells are collision-free, then verifies that at `OCTO_COUNT_MIN_DEFAULT_THRESHOLD - 1` occurrences each the heap is empty and key `0` estimates `0`, while at `OCTO_COUNT_MIN_DEFAULT_THRESHOLD` occurrences the heap holds all `20` keys and key `0` estimates at least `OCTO_COUNT_MIN_DEFAULT_THRESHOLD`. | +| `the_count_sketch_with_heap_aggregator_also_tracks_heavy_hitters` | The Count top-k aggregator keeps a heap too. | Runs `U64(0..3)` at 5,000 occurrences each plus a tail of `U64(500..3_000)` through a `5x2048` `CountSketchWithHeapOctoPlan` at a heap of `8` on 4 workers, and verifies the heap finds each of the three hot keys. | | `run_octo_ddsketch_tracks_quantiles_of_the_stream` | A finished DDSketch run answers quantiles within alpha plus the fleet's rank slack. | Runs 200,000 values `1.0 + (i * 7) % 999` through `DdOctoPlan` at alpha `0.01` on 4 workers and verifies the fleet's held-back share is under `0.05` and that the `0.1`, `0.5` and `0.9` quantiles each fall between the exact value at rank `q - slack` scaled by `1 - alpha` and the exact value at rank `q + slack` scaled by `1 + alpha`. | | `run_octo_univmon_reaches_the_deepest_layer` | The scaled per-layer threshold lets the deepest layer through. | Runs 60,000 inputs `U64(i % 4_096)` through a 12-layer `5x1024` `UnivMonOctoPlan` at a heap of `64` on 1 worker, and verifies layer `11` holds at least one non-zero counter. | | `run_octo_univmon_tracks_heavy_hitters_and_total_weight` | The reported total weight trails the stream without overshooting it, and layer 0 holds the hot keys. | Runs `U64(0..5)` at 4,000 occurrences each plus a tail of `U64(10_000..15_000)`, 25,000 inputs in all, through a 6-layer `3x256` `UnivMonOctoPlan` at a heap of `32` and a shared tau of `8` on 4 workers, and verifies `bucket_size` is at most 25,000 and above `0.9 * 25_000`, that layer 0's heap finds each of the five hot keys, and that no layer claims a complete candidate set. | @@ -1194,11 +1194,11 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `flushing_twice_is_harmless` | A second flush changes nothing. | Inserts 5,000 values into a `DdOctoPlan` runtime at alpha `0.01` on 2 workers, flushes twice, and verifies `finish` reports a count of `5_000`. | | `a_flushed_count_min_parent_matches_a_single_pass_exactly` | A finished Count-Min parent equals a single-threaded pass cell for cell. | Runs 40,000 inputs `U64(i % 512)` through a `3x1024` `CountMinSketchOctoPlan` on 4 workers and verifies all 3,072 counters equal a single-threaded reference's. | | `octo_runtime_close_is_idempotent` | Closing twice is allowed. | Calls `close` twice on a 2-worker `HllOctoPlan` runtime that saw no inserts and verifies `finish` estimates `0`. | -| `octo_runtime_insert_after_close_panics` | An insert after `close` panics. | Verifies inserting `U64(1)` into a closed 2-worker `HllOctoPlan` runtime panics with "cannot update after runtime has been closed". | +| `octo_runtime_update_after_close_panics` | An insert after `close` panics. | Verifies inserting `U64(1)` into a closed 2-worker `HllOctoPlan` runtime panics with "cannot update after runtime has been closed". | | `octo_runtime_empty_stream_finishes` | A runtime that saw nothing still finishes. | Verifies `finish` on a 4-worker `HllOctoPlan` runtime with no inserts estimates `0`. | | `octo_runtime_live_read_handle_tracks_the_aggregator` | A read handle reaches the live parent before the run is finished. | Inserts 64 inputs into a 2-worker counting runtime and polls the read handle, verifying it reads `0` before any insert, never goes backwards, never exceeds `64`, reaches exactly `64` inside a 10-second deadline, and that `finish` then reports `64`. | | `octo_runtime_close_preserves_queued_items` | `close` does not discard what is already queued. | Inserts 257 inputs into a 4-worker counting runtime, calls `close`, and verifies `finish` reports `257`. | -| `a_worker_picks_the_same_layers_as_a_single_threaded_insert` | A worker routes a key to the same layer set a single-threaded insert would. | For keys `U64(0..5_000)` over 6 layers, verifies `bottom_layer_for_hash(hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key), 6)` equals `UnivMon::bottom_layer_for` for every key and is under `6`. | +| `a_worker_picks_the_same_layers_as_a_single_threaded_update` | A worker routes a key to the same layer set a single-threaded insert would. | For keys `U64(0..5_000)` over 6 layers, verifies `bottom_layer_for_hash(hash64_seeded(UNIVMON_BOTTOM_LAYER_SEED_INDEX, key), 6)` equals `UnivMon::bottom_layer_for` for every key and is under `6`. | | `one_worker_at_threshold_one_reproduces_a_single_threaded_univmon` | At tau 1 one worker leaves the aggregator in a single-threaded UnivMon's exact state. | Drives a 20,000-input skewed stream through one `UnivMonOctoWorker` at `3x256` over 6 layers, a heap of `32` and a threshold of `1`, and verifies the aggregator matches `UnivMon::new(32, 3, 256, 6)` on `bucket_size`, on `candidates_complete()`, and per layer on the counter array, `l2` and the heavy-hitter heap contents, plus `estimate_g_sum`. | | `promotion_conserves_every_layer_counter` | Promoted plus residual reconstructs every layer counter, and no residual is over tau. | Drives a 30,000-input skewed stream through one worker at a threshold of `31` and verifies, for every cell of all 6 `3x256` layers, that the aggregator's counter plus the worker's residual equals the single-threaded counter, and that the residual's magnitude is under `31`. | | `completeness_is_withdrawn_exactly_where_the_layer_thresholds` | Completeness is withdrawn on exactly the layers whose scaled threshold is above one. | Drives a 5,000-input skewed stream through one worker at a base threshold of `8` and verifies every layer whose `univmon_layer_threshold(8, layer)` exceeds `1` reports incomplete candidates while every layer at `1` reports complete, and that both groups are non-empty. | @@ -1207,24 +1207,24 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `the_layer_threshold_tracks_how_little_traffic_a_deep_layer_sees` | `univmon_layer_threshold` halves tau per layer with a floor of one. | Verifies `univmon_layer_threshold(32, ..)` is `32` at layer `0`, `16` at layer `1`, `1` at layer `5` and still `1` at layer `40` without shift overflow, and that a base of `1` stays `1` at layer `0`. | | `the_worker_holds_one_byte_per_counter_and_no_keys` | A UnivMon worker holds one byte per counter and no key storage. | Verifies a 6-layer `3x256` `UnivMonOctoWorker` reports `counter_bytes()` of `3 * 256 * 6`, an eighth of the `i64` counters a UnivMon layer holds, and that processing 5,000 distinct 40-character string keys leaves `counter_bytes()` unchanged. | | `cm_promotion_is_lossless_modulo_the_child_residual` | Promoted counts plus the child's residual reconstruct a single pass. | Drives 20,000 keys of a zipf(1.1) stream over domain `512` at seed `9_101` through a `5x2048` `CountMinSketch, RegularPath>` child via `update_emit_delta`, replays every `CountMinSketchCellDelta` into a parent with `apply_delta`, and verifies for all 10,240 cells that parent plus child residual equals the single-pass reference cell and that every residual is at most `TAU - 1 = 30`. | -| `cm_deltas_are_well_formed_and_carry_exactly_one_promotion` | Every Count-Min delta carries one promotion window and addresses a real cell. | Over 20,000 zipf(1.1) keys, domain `512`, seed `9_102`, on a `5x2048` child, verifies the emitted `CountMinSketchCellDelta` list is non-empty and that each delta has `value == OCTO_COUNT_MIN_DEFAULT_THRESHOLD` (`31`), `row < 5` and `col < 2048`. | +| `count_min_sketch_cell_deltas_are_well_formed_and_carry_exactly_one_promotion` | Every Count-Min delta carries one promotion window and addresses a real cell. | Over 20,000 zipf(1.1) keys, domain `512`, seed `9_102`, on a `5x2048` child, verifies the emitted `CountMinSketchCellDelta` list is non-empty and that each delta has `value == OCTO_COUNT_MIN_DEFAULT_THRESHOLD` (`31`), `row < 5` and `col < 2048`. | | `cm_parent_holds_every_completed_promotion` | The parent holds every whole multiple of tau and nothing partial. | Over 30,000 zipf(1.1) keys, domain `512`, seed `9_103`, on `5x2048`, verifies every parent cell equals `31 * (reference_cell / 31)`, that the worst cell deficit is at most `30`, and that it is strictly above `0`. | -| `cm_octo_estimate_trails_the_single_thread_estimate_by_under_one_promotion` | An unflushed parent's point estimate is one-sided and lags by under one window. | Over 40,000 zipf(1.1) keys, domain `256`, seed `9_104`, on `5x2048`, verifies for each of the keys `U64(0..256)` that `parent.estimate()` never exceeds the single-threaded `reference.estimate()` and trails it by at most `30`. | +| `count_min_sketch_octo_estimate_trails_the_single_thread_estimate_by_under_one_promotion` | An unflushed parent's point estimate is one-sided and lags by under one window. | Over 40,000 zipf(1.1) keys, domain `256`, seed `9_104`, on `5x2048`, verifies for each of the keys `U64(0..256)` that `parent.estimate()` never exceeds the single-threaded `reference.estimate()` and trails it by at most `30`. | | `cm_fast_path_promotes_on_the_same_schedule_as_the_regular_path` | The fast path promotes exactly the mask value and loses no counts. | Over 20,000 zipf(1.1) keys, domain `512`, seed `9_105`, through a `5x2048` `CountMinSketch, FastPath>` child into a `FastPath` parent, verifies at least one promotion fired, that every emitted delta's `value` is `OCTO_COUNT_MIN_DEFAULT_THRESHOLD`, and that parent plus child equals a `FastPath` single-pass reference on all 10,240 cells; the regular path is never run, so the promotion *schedules* of the two paths are not compared despite the name - only the per-cell promotion rule and conservation are asserted. | -| `cm_delta_addressing_survives_columns_past_the_u16_ceiling` | Delta row/col fields address a geometry wider than 65,536 columns. | Drives 60,000 zipf(1.1) keys, domain `4_096`, seed `9_108`, through a `3x100_000` `CountMinSketch, RegularPath>` child straight into a parent, and verifies parent plus child equals the single-pass reference on all 300,000 cells. | -| `cm_delta_application_is_order_independent` | Delivery order does not change the Count-Min parent. | Captures the deltas of 15,000 zipf(1.1) keys, domain `512`, seed `9_106`, on `5x2048`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_106)`, and verifies the two parents' full 10,240-cell vectors are equal. | +| `count_min_sketch_cell_delta_addressing_survives_columns_past_the_u16_ceiling` | Delta row/col fields address a geometry wider than 65,536 columns. | Drives 60,000 zipf(1.1) keys, domain `4_096`, seed `9_108`, through a `3x100_000` `CountMinSketch, RegularPath>` child straight into a parent, and verifies parent plus child equals the single-pass reference on all 300,000 cells. | +| `count_min_sketch_cell_delta_application_is_order_independent` | Delivery order does not change the Count-Min parent. | Captures the deltas of 15,000 zipf(1.1) keys, domain `512`, seed `9_106`, on `5x2048`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_106)`, and verifies the two parents' full 10,240-cell vectors are equal. | | `cm_sharded_children_conserve_counts_against_a_single_pass` | Sharding preserves every count, with one window of slack per shard. | Deals 40,000 zipf(1.1) keys, domain `512`, seed `9_107`, round-robin across `4` `5x2048` children into one parent, and verifies per cell that the parent plus the sum of the four residuals equals the single-pass reference and that the summed residual is at most `4 * 30 = 120`. | -| `count_deltas_carry_exactly_the_signed_threshold` | A Count-sketch delta is always plus or minus the mask value. | Over 30,000 zipf(1.1) keys, domain `256`, seed `9_201`, through a `5x2048` `CountSketch, RegularPath>` child, verifies the delta list is non-empty and that every delta has `value.unsigned_abs() == OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` (`31`), `row < 5` and `col < 2048`. | +| `count_sketch_cell_deltas_carry_exactly_the_signed_threshold` | A Count-sketch delta is always plus or minus the mask value. | Over 30,000 zipf(1.1) keys, domain `256`, seed `9_201`, through a `5x2048` `CountSketch, RegularPath>` child, verifies the delta list is non-empty and that every delta has `value.unsigned_abs() == OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` (`31`), `row < 5` and `col < 2048`. | | `count_promotion_is_lossless_modulo_the_child_residual` | Promoted signed counts plus the residual reconstruct a single pass. | Over 30,000 zipf(1.1) keys, domain `256`, seed `9_202`, on `5x2048`, verifies for all 10,240 cells that parent plus child equals the single-pass reference and that each residual's magnitude is at most `30`. | -| `count_octo_estimate_stays_within_one_residual_of_the_single_thread_estimate` | The signed parent's estimate stays within one window either way. | Over 40,000 zipf(1.1) keys, domain `256`, seed `9_203`, on `5x2048`, verifies for each of `U64(0..256)` that the absolute gap between `parent.estimate()` and `reference.estimate()` is at most `30.0`. | -| `count_fast_path_conserves_counts_like_the_regular_path` | The signed fast path promotes the mask magnitude and loses no counts. | Over 20,000 zipf(1.1) keys, domain `256`, seed `9_204`, through a `5x2048` `CountSketch, FastPath>` child into a `FastPath` parent, verifies every emitted delta's magnitude is `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` and that parent plus child equals a `FastPath` single-pass reference on all 10,240 cells; the regular path is not run, so the two paths are never compared against each other. | -| `count_delta_application_is_order_independent` | Delivery order does not change the Count-sketch parent. | Captures the deltas of 15,000 zipf(1.1) keys, domain `256`, seed `9_205`, on `5x2048`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_205)`, and asserts the two 10,240-cell vectors are equal; that single equality is the whole assertion. | +| `count_sketch_octo_estimate_stays_within_one_residual_of_the_single_thread_estimate` | The signed parent's estimate stays within one window either way. | Over 40,000 zipf(1.1) keys, domain `256`, seed `9_203`, on `5x2048`, verifies for each of `U64(0..256)` that the absolute gap between `parent.estimate()` and `reference.estimate()` is at most `30.0`. | +| `count_sketch_fast_path_conserves_counts_like_the_regular_path` | The signed fast path promotes the mask magnitude and loses no counts. | Over 20,000 zipf(1.1) keys, domain `256`, seed `9_204`, through a `5x2048` `CountSketch, FastPath>` child into a `FastPath` parent, verifies every emitted delta's magnitude is `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` and that parent plus child equals a `FastPath` single-pass reference on all 10,240 cells; the regular path is not run, so the two paths are never compared against each other. | +| `count_sketch_cell_delta_application_is_order_independent` | Delivery order does not change the Count-sketch parent. | Captures the deltas of 15,000 zipf(1.1) keys, domain `256`, seed `9_205`, on `5x2048`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_205)`, and asserts the two 10,240-cell vectors are equal; that single equality is the whole assertion. | | `count_sharded_children_conserve_signed_counts` | Sharding the signed sketch preserves every count. | Deals 40,000 zipf(1.1) keys, domain `256`, seed `9_206`, round-robin across `3` `5x2048` children into one parent, and verifies per cell that the parent plus the three residuals equals the single-pass reference; no bound on the residual size is asserted here. | | `hll_promotion_reproduces_the_single_thread_registers_exactly` | Max-register promotion leaves parent and child identical to one pass. | Feeds the 60,000 distinct keys `U64(0..60_000)` through a default `HyperLogLog` (p14) child into a parent, and verifies `OCTO_HLL_DEFAULT_THRESHOLD == 0`, that the parent's and the child's `registers_as_slice()` both equal a single-pass reference's, and that `parent.estimate()` equals `reference.estimate()`. | | `hll_promotion_is_exact_for_every_variant_and_precision` | Promotion is register-exact at every precision and estimator. | Feeds 40,000 values `i.wrapping_mul(2_654_435_761)` through an inline `check!` macro that runs one child, one parent and one single-pass reference per type, asserting both the parent's and the child's registers equal the reference's for `HyperLogLogP12`, `HyperLogLog` (p14), `HyperLogLogP16`, `HyperLogLogP12`, `HyperLogLog` and `HyperLogLogP16`. | -| `hll_deltas_are_strictly_increasing_per_register_and_never_repeat` | Each register's delta stream is strictly increasing and replays its final value. | Over the 30,000 distinct keys `U64(0..30_000)` on a default `HyperLogLog`, verifies every delta's `pos` is inside the register array, that its `value` is strictly above the best value promoted for that register so far, and that the resulting per-register maxima equal the child's `registers_as_slice()`. | +| `hll_register_deltas_are_strictly_increasing_per_register_and_never_repeat` | Each register's delta stream is strictly increasing and replays its final value. | Over the 30,000 distinct keys `U64(0..30_000)` on a default `HyperLogLog`, verifies every delta's `pos` is inside the register array, that its `value` is strictly above the best value promoted for that register so far, and that the resulting per-register maxima equal the child's `registers_as_slice()`. | | `hll_duplicate_keys_promote_nothing_after_the_first_improvement` | Only the first insert of a key can improve a register. | Inserts `U64(7)` 1,000 times into a default `HyperLogLog` through `update_emit_delta` and verifies exactly `1` delta was emitted. | -| `hll_delta_application_is_order_independent` | Max is commutative, so register delivery order is free. | Captures the deltas of `U64(0..30_000)`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_301)`, and verifies the two parents' register slices are equal. | +| `hll_register_delta_application_is_order_independent` | Max is commutative, so register delivery order is free. | Captures the deltas of `U64(0..30_000)`, applies them in order and again after `shuffle` with `StdRng::seed_from_u64(9_301)`, and verifies the two parents' register slices are equal. | | `hll_sharded_children_match_a_single_pass_exactly` | Sharded HLL promotion carries no partition penalty. | Deals the 50,000 distinct keys `U64(0..50_000)` round-robin across `4` default `HyperLogLog` children into one parent and verifies the parent's `registers_as_slice()` equals a single-pass reference's. | | `hll_cardinality_survives_the_delta_round_trip_exactly` | Delta-fed registers match a single pass byte for byte, and the estimate stays in band. | Over the 100,000 distinct keys `U64(0..100_000)` through a default `HyperLogLog`, verifies the parent's registers equal the reference's, that `parent.estimate()` equals `reference.estimate()`, and that the relative error against the exact `100_000` is under `0.025` (p14's 1.04/sqrt(2^14) is about 0.81%). | | `run_octo_cm_matches_a_single_threaded_replay_of_the_same_partition` | The Count-Min runtime is cell-identical to a deterministic replay at every worker count. | Under `--features octo-runtime`, runs 40,000 zipf(1.1) keys, domain `512`, seed `9_401`, through `run_octo` with `CountMinSketchOctoPlan::new(5, 2048)` at `queue_capacity` `4096` and no core pinning, for `1, 2, 3, 4, 7` workers, and verifies all 10,240 parent cells equal those of a `HashByKey` single-threaded replay that flushes its children at end of stream. | @@ -1233,17 +1233,17 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `run_octo_ddsketch_matches_a_single_threaded_replay` | The DDSketch runtime's buckets are a pure function of the partition. | Under `--features octo-runtime`, runs 40,000 values `1.0 + (i * 7) % 999` through `run_octo` with `DdOctoPlan::new(0.01)` on `4` workers and verifies `get_count()` and the list of non-zero `(positive_store_offset + index, count)` pairs equal those of a single-threaded `DdSketchWorker` replay driven at `OCTO_DDSKETCH_DEFAULT_THRESHOLD` (`4`) and then flushed; the raw store offset and length are deliberately not compared. | | `run_octo_is_deterministic_across_repeated_runs` | Two runs of the same stream give the same parent. | Under `--features octo-runtime`, runs 30,000 zipf(1.1) keys, domain `512`, seed `9_403`, through `CountMinSketchOctoPlan::new(5, 2048)` on `4` workers twice and verifies the two 10,240-cell vectors are equal. | | `streaming_runtime_matches_the_batch_helper` | `OctoRuntime` and `run_octo` end in the same parent state. | Under `--features octo-runtime`, runs 30,000 zipf(1.1) keys, domain `512`, seed `9_404`, on `4` workers once through `run_octo` and once insert-by-insert through `OctoRuntime` plus `finish`, both with `CountMinSketchOctoPlan::new(5, 2048)`, and verifies the 10,240-cell vectors are equal. | -| `insert_batch_matches_element_wise_inserts` | `update_batch` is element-wise `update`. | Under `--features octo-runtime`, runs 20,000 zipf(1.1) keys, domain `512`, seed `9_405`, through two `CountMinSketchOctoPlan::new(5, 2048)` runtimes on `3` workers - one per-input `update`, one single `update_batch` - and verifies the finished parents' 10,240-cell vectors are equal. | +| `update_batch_matches_element_wise_updates` | `update_batch` is element-wise `update`. | Under `--features octo-runtime`, runs 20,000 zipf(1.1) keys, domain `512`, seed `9_405`, through two `CountMinSketchOctoPlan::new(5, 2048)` runtimes on `3` workers - one per-input `update`, one single `update_batch` - and verifies the finished parents' 10,240-cell vectors are equal. | | `degenerate_config_is_clamped_rather_than_rejected` | Zero workers and zero queue capacity clamp to one instead of dropping data. | Under `--features octo-runtime`, runs the 5,000 distinct keys `U64(0..5_000)` through `run_octo` with `num_workers: 0` and `queue_capacity: 0` on `HllOctoPlan`, and verifies the parent's registers equal a single-threaded reference's. | | `a_one_slot_queue_applies_backpressure_without_deadlocking` | A one-slot queue still finishes the whole stream correctly. | Under `--features octo-runtime`, runs the 20,000 distinct keys `U64(0..20_000)` through `run_octo` with `4` workers and `queue_capacity: 1` on `HllOctoPlan`, and asserts the parent's registers equal a single-threaded reference's; the assertion is that the run terminates and is exact - no queue occupancy or blocking is observed, so "applies backpressure" is not itself measured. | -| `a_borrowed_key_may_be_dropped_the_moment_insert_returns` | A borrowed key can be freed the instant `update` returns. | Under `--features octo-runtime`, inserts 20,000 `DataInput::Str` values `format!("session-{i:020}")` into an `OctoRuntime` over `HllOctoPlan` on `4` workers, dropping each owned `String` immediately after the call, and verifies the finished parent's registers equal a single-threaded reference's over the same 20,000 strings. | +| `a_borrowed_key_may_be_dropped_the_moment_update_returns` | A borrowed key can be freed the instant `update` returns. | Under `--features octo-runtime`, inserts 20,000 `DataInput::Str` values `format!("session-{i:020}")` into an `OctoRuntime` over `HllOctoPlan` on `4` workers, dropping each owned `String` immediately after the call, and verifies the finished parent's registers equal a single-threaded reference's over the same 20,000 strings. | | `a_borrowed_key_reaches_a_keyed_aggregator_by_value` | A key-storing aggregator receives borrowed keys by value. | Under `--features octo-runtime`, inserts `hot-0`, `hot-1`, `hot-2` as borrowed `Str` values 4,000 times each (each `String` dropped right after `update`) into an `OctoRuntime` over `CountMinSketchWithHeapOctoPlan::new(4, 2048)` with a top-`8` `CountMinSketchWithHeapOctoAggregator` on `4` workers, and verifies the parent's `heap().find()` returns a hit for all three keys; the heap counts are not checked. | | `an_empty_stream_finishes_with_a_pristine_parent` | An empty stream leaves every counter at zero. | Under `--features octo-runtime`, calls `run_octo(&[], ...)` with `CountMinSketchOctoPlan::new(5, 2048)` on `4` workers and verifies all 10,240 parent cells are `0`. | | `a_user_defined_worker_and_aggregator_round_trip_every_input` | The public `OctoPlan`/`OctoWorker`/`OctoAggregator` traits work from outside the crate. | Under `--features octo-runtime`, defines a `SumWorker`/`SumPlan`/`SumAggregator` whose delta is `(worker_id, 1)`, runs `U64(0..10_001)` through `run_octo` on `3` workers under `OctoPartition::RoundRobin`, and verifies the per-worker loads sum to exactly `10_001` and that each worker `w` received exactly the count of stream indices with `i % 3 == w`. | | `the_read_handle_observes_a_monotone_prefix_of_the_final_state` | The live read handle is monotone and reaches the live total before `finish`. | Under `--features octo-runtime`, drives `U64(0..4_000)` through an `OctoRuntime` over `SumPlan` on `2` workers, and verifies the handle reads `0` before the first insert, that every reading sampled at each 500th insert is non-decreasing and at most `4_000`, that it then reaches exactly `4_000` inside a 10-second `Instant`-based deadline while the runtime is still open, and that `finish` reports `4_000`. | | `reading_through_a_stale_handle_panics` | Reading a handle after `finish` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "Octo runtime has been finished")]` test that takes a `read_handle()` from a 2-worker `SumPlan` runtime, calls `finish()`, then calls `with_parent` on the stale handle. | | `close_is_idempotent_and_preserves_already_queued_work` | Closing twice is allowed and drains what was queued. | Under `--features octo-runtime`, inserts `U64(0..3_001)` into a 4-worker `SumPlan` runtime, calls `close()` twice, and verifies `finish` reports a per-worker total of exactly `3_001`. | -| `inserting_after_close_panics` | An insert after `close` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "cannot update after runtime has been closed")]` test that closes a 2-worker `SumPlan` runtime and then inserts `U64(1)`. | +| `updating_after_close_panics` | An insert after `close` panics. | Under `--features octo-runtime`, a `#[should_panic(expected = "cannot update after runtime has been closed")]` test that closes a 2-worker `SumPlan` runtime and then inserts `U64(1)`. | | `a_finished_run_carries_only_count_mins_own_one_sided_error` | A flushed Count-Min run has only Count-Min's own one-sided error left. | Under `--features octo-runtime`, runs 200,000 zipf(1.1) keys, domain `4_096`, seed `9_501`, through `CountMinSketchOctoPlan::new(5, 4096)` on `4` workers and verifies for every one of the 4,096 keys that the estimate is at or above the exact `FreqTruth` count and at most `exact + (e / 4096) * N` above it. | | `run_octo_hll_cardinality_error_stays_within_three_sigma` | The HLL runtime is register-exact and its cardinality is within three sigma. | Under `--features octo-runtime`, runs the 200,000 distinct keys `U64(0..200_000)` through `HllOctoPlan` on `4` workers and verifies the parent's registers equal a single-threaded reference's exactly, then that the relative error of `estimate()` against `200_000` is under `0.025`. | | `run_octo_count_frequencies_track_a_zipf_stream` | A flushed Count-sketch run holds the Count-sketch L2 bound on the heavy keys. | Under `--features octo-runtime`, runs 200,000 zipf(1.1) keys, domain `2_048`, seed `9_502`, through `CountSketchOctoPlan::new(5, 4096)` on `4` workers, then for the truth's top `64` keys collects any estimate whose absolute error exceeds `3 * \|\|f\|\|_2 / sqrt(4096)` and asserts the violation list is empty. | @@ -1277,10 +1277,10 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `run_octo_elastic_matches_a_single_threaded_replay_at_one_worker` | At one worker the Elastic runtime is bit-exact against a sequential replay. | Under `--features octo-runtime`, runs 40,000 zipf(1.1) keys, domain `2_048`, seed `23_003`, through `run_octo` with `ElasticOctoPlan::new(128, 3, 1024)` on `1` worker, repeated `3` times, and verifies every heavy bucket's `(flow_id, vote_pos, vote_neg, eviction)` and all `3 * 1024` light counters equal those of a flushed single-threaded `HashByKey` replay. | | `coco_online_accuracy_beats_sketch_merge` | Coco's online relative error and heavy-hitter F1 both favour delta promotion. | Over `3` streams of 200,000 zipf(1.1) keys, domain `5_000`, seeds `24_001..24_004`, drives a `4`-worker `512x2` Octo Coco replay, an ideal single-pass `CocoSketch` and a sketch-merge baseline sized to Octo's counter spend, scoring at `8` off-boundary checkpoints the mean relative error over the truth's top `32` keys and the F1 of the reported set against the `HH_ALPHA = 0.0002` heavy-hitter threshold, and verifies the baseline was not starved, that Octo's relative error is below half the baseline's, that Octo's F1 is above the baseline's, that the ideal pass leads Octo, and that Octo's relative error is below `0.10`. | | `elastic_online_accuracy_beats_sketch_merge` | Elastic's online relative error and F1 both favour delta promotion. | Over 300,000 zipf(1.1) keys, domain `5_000`, seed `24_101`, drives a `4`-worker Octo Elastic replay at `1_024` buckets and a `3x4096` light layer, an ideal single-pass `ElasticSketch`, and a sketch-merge baseline sized to Octo's spend, scoring at `8` off-boundary checkpoints the mean relative error over the truth's top `32` keys and F1 at the `0.0002` heavy-hitter threshold, and verifies the baseline was funded, that Octo's relative error is below half the baseline's, that Octo's F1 exceeds `1.2x` the baseline's, that the ideal pass leads Octo, and that Octo's relative error is below `0.06`. | -| `cm_top_k_point_estimates_trail_the_single_thread_reference_by_at_most_k_tau` | The Count-Min top-k aggregator trails a single thread by at most `k*tau` and keeps Count-Min's own bound. | Drives 200,000 zipf(1.1) keys, domain `8_192`, seed `0x0C70_1101`, through `CountMinSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both `HashByKey` and `RoundRobin`, and per route tallies over every truth key that the estimate is at or below a `CountMinSketchWithHeap` reference's and no more than `k*tau = 124` under it, that it is at or above `truth - 124`, that it is under `CountMinSpec::new(5, 2048).simultaneous_bound(..., SIMULTANEOUS_LEVEL = 1e-3)` with zero violations tolerated, and that violations of the marginal `e*(N-f)/w` bound occur at a rate no worse than `e^-5`. | -| `count_top_k_point_estimates_satisfy_the_l2_bound_within_the_promotion_residual` | The Count top-k aggregator holds the L2 bound widened by exactly `k*tau`. | Drives the same 200,000-key stream through `CountSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both routes, and per route tallies over every truth key that the absolute gap to a `CountSketchWithHeap` reference is at most `k*tau = 124`, that the absolute error against truth is within `CountSketchSpec::new(5, 2048).scale_at(simultaneous_kappa(distinct, 1e-3), \|\|f_-i\|\|_2) + 124` with zero violations, and that violations of `sqrt(3/w)*\|\|f_-i\|\|_2 + 124` occur at a rate no worse than the spec's marginal failure probability. | +| `count_min_sketch_with_heap_point_estimates_trail_the_single_thread_reference_by_at_most_k_tau` | The Count-Min top-k aggregator trails a single thread by at most `k*tau` and keeps Count-Min's own bound. | Drives 200,000 zipf(1.1) keys, domain `8_192`, seed `0x0C70_1101`, through `CountMinSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both `HashByKey` and `RoundRobin`, and per route tallies over every truth key that the estimate is at or below a `CountMinSketchWithHeap` reference's and no more than `k*tau = 124` under it, that it is at or above `truth - 124`, that it is under `CountMinSpec::new(5, 2048).simultaneous_bound(..., SIMULTANEOUS_LEVEL = 1e-3)` with zero violations tolerated, and that violations of the marginal `e*(N-f)/w` bound occur at a rate no worse than `e^-5`. | +| `count_sketch_with_heap_point_estimates_satisfy_the_l2_bound_within_the_promotion_residual` | The Count top-k aggregator holds the L2 bound widened by exactly `k*tau`. | Drives the same 200,000-key stream through `CountSketchWithHeapOctoPlan::new(5, 2048)` on `4` workers with a top-`32` aggregator under both routes, and per route tallies over every truth key that the absolute gap to a `CountSketchWithHeap` reference is at most `k*tau = 124`, that the absolute error against truth is within `CountSketchSpec::new(5, 2048).scale_at(simultaneous_kappa(distinct, 1e-3), \|\|f_-i\|\|_2) + 124` with zero violations, and that violations of `sqrt(3/w)*\|\|f_-i\|\|_2 + 124` occur at a rate no worse than the spec's marginal failure probability. | | `top_k_plans_recover_the_heavy_hitters_their_promotion_floor_guarantees` | Both top-k plans recover every heavy hitter above the promotion floor, with a consistent heap. | Over the same 200,000-key stream and both routes, drives `CountMinSketchWithHeapOctoPlan` and `CountSketchWithHeapOctoPlan` at `5x2048` on `4` workers with a top-`32` aggregator and verifies per plan and route that the heap holds at most `32` entries with no duplicate key, that at least `16` of the true top `32` clear the `k*tau = 124` floor (a premise check), that *every* true top-32 key above that floor is present, that no heap key's true count is below the true 32nd count minus `124`, and that each heap entry's stored count equals the aggregator sketch's own estimate for that key. | -| `a_key_below_the_promotion_threshold_never_reaches_the_top_k_aggregator` | A flow below tau is absent from the top-k aggregator entirely, and the first crossing ships exactly tau. | On `CountMinSketchWithHeapOctoPlan::new(5, 2048)` with one worker and a top-`32` aggregator, sends `U64(0xC01D)` `TAU - 1 = 30` times and verifies zero deltas shipped, that the aggregator estimates `0` for it, and that the heap is empty; then sends one more arrival and verifies at least one delta was promoted, that every delta's `key` is `DataInput::to_owned_input` of that key, that the aggregator then estimates exactly `TAU = 31`, and that the heap holds exactly `1` entry. | +| `a_key_below_the_promotion_threshold_never_reaches_the_count_min_sketch_with_heap_aggregator` | A flow below tau is absent from the top-k aggregator entirely, and the first crossing ships exactly tau. | On `CountMinSketchWithHeapOctoPlan::new(5, 2048)` with one worker and a top-`32` aggregator, sends `U64(0xC01D)` `TAU - 1 = 30` times and verifies zero deltas shipped, that the aggregator estimates `0` for it, and that the heap is empty; then sends one more arrival and verifies at least one delta was promoted, that every delta's `key` is `DataInput::to_owned_input` of that key, that the aggregator then estimates exactly `TAU = 31`, and that the heap holds exactly `1` entry. | | `hll_octo_plan_is_register_exact_and_in_band_under_both_partitions` | `HllOctoPlan` is register-exact and in cardinality band under either partition. | Drives 60,000 zipf(1.1) keys, domain `1_024`, seed `0x0C70_ACC0`, through `HllOctoPlan` on `4` workers under `HashByKey` and `RoundRobin`, flushes each worker, and per route verifies the parent's `registers_as_slice()` equals a single-threaded `HyperLogLog`'s exactly and that `estimate()` passes `CardinalityConfidenceSpec::hll(14, 4.0)` against the exact distinct count. | | `dd_octo_plan_is_bucket_exact_after_flush_and_holds_alpha_under_both_partitions` | `DdOctoPlan` is bucket-exact after a flush and holds alpha under either partition. | Turns the same 60,000-key stream into values `1.0 + key`, drives them through `DdOctoPlan::new(0.01)` on `4` workers under both routes, flushes, and per route verifies `get_count()` equals a single-pass `DdSketch`'s, that every bucket over the union of both stores' index ranges matches the reference (offset differences allowed for), and that the quantiles `0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0` all satisfy `RelativeQuantileSpec::core(0.01)` against the exact order statistics, with zero violations tolerated. | | `univmon_octo_plan_reports_the_delivered_l1_and_holds_its_f2_bound_under_both_partitions` | The UnivMon parent's L1 is exactly the delivered weight, and its F2 holds the AMS bound. | Drives the same 60,000-key stream through `UnivMonOctoPlan::new(5, 4096, 8)` with heap `64` on `4` workers under both routes, recording the largest `total_weight` each worker stamped on a delta, and per route verifies the delivered sum never exceeds `N`, that `parent.estimate_l1()` equals that sum *exactly* (`UnivMonOctoWorker` has no flush, so mass after a worker's last delta stays in flight and the printed context names how much), and that `estimate_l2().powi(2)` passes `SecondMomentSpec::new(5, 4096)`'s `sqrt(2*kappa/w)` relative band against the exact F2. | @@ -1296,8 +1296,8 @@ Test file: [`src/sketch_framework/tumbling.rs`](../src/sketch_framework/tumbling | `pool_take_returns_preallocated` | A take off a pre-allocated pool hands back a sketch without allocating. | For a `SketchPool` of `4` over `CountMinSketchFoldConfig { rows: 3, full_cols: 1024, fold_level: 3, heap_size: 10 }`, verifies `available` and `total_allocated` both start at `4`, and that one `take` leaves `available` at `3` with `total_allocated` still `4`. | | `pool_take_allocates_when_empty` | An empty pool allocates rather than failing. | For a `SketchPool` of `0` over the `3x1024` fold-3 config, verifies `available` and `total_allocated` both start at `0`, and that one `take` raises `total_allocated` to `1`. | | `pool_put_recycles` | A returned sketch goes back on the free list instead of being dropped. | For a `SketchPool` of `1` over the `3x1024` fold-3 config, verifies `available` is `0` after a `take` and back to `1` after the `put`, with `total_allocated` still `1`. | -| `fold_cms_clear_resets_to_empty` | `CountMinSketchFold::clear` empties both the counters and the heap. | Inserts keys `U64(0..50)` at weight `1` into a `CountMinSketchFold::new(3, 1024, 3, 10)` built directly, verifies key `0` reads above `0`, then after `clear` verifies all fifty keys read `0` and `heap()` is empty. | -| `fold_cs_clear_resets_to_empty` | `CountSketchFold::clear` empties both the counters and the heap. | Inserts keys `U64(0..50)` at weight `1` into a `CountSketchFold::new(3, 1024, 3, 10)` built directly, verifies key `0` reads non-zero, then after `clear` verifies all fifty keys read `0` and `heap()` is empty. | +| `count_min_sketch_fold_clear_resets_to_empty` | `CountMinSketchFold::clear` empties both the counters and the heap. | Inserts keys `U64(0..50)` at weight `1` into a `CountMinSketchFold::new(3, 1024, 3, 10)` built directly, verifies key `0` reads above `0`, then after `clear` verifies all fifty keys read `0` and `heap()` is empty. | +| `count_sketch_fold_clear_resets_to_empty` | `CountSketchFold::clear` empties both the counters and the heap. | Inserts keys `U64(0..50)` at weight `1` into a `CountSketchFold::new(3, 1024, 3, 10)` built directly, verifies key `0` reads non-zero, then after `clear` verifies all fifty keys read `0` and `heap()` is empty. | | `kll_clear_resets_to_empty` | `Kll::clear` resets the count and the CDF. | Feeds `F64(0..1000)` into a `Kll::new(200, 8)` built directly, verifies `count()` is above `0`, then after `clear` verifies `count()` is `0` and `cdf().query(0.5)` is `0.0`. | | `univmon_q_tumbling_merges_quantiles_and_universal_metrics` | A `TumblingWindow` merge carries the quantile and the distinct count exactly. | For `TumblingWindow::new(100, 4, cfg, 2)` over `UnivMonQConfig { levels: 8, width: 256, cols_halving_period: 0, depth: 5, counter_bits: 64, candidates: 64, ordered_samples: 64, seed_index: 5 }` fed `F64(value % 50)` at timestamps `0..300`, verifies `query_all` reports `count` `300`, `quantile(0.5)` of `Some(24.0)`, and `estimate_cardinality` of `50.0`. | | `zero_window_size_panics` | A `window_size` of zero is refused at construction. | Verifies `TumblingWindow::::new(0, 5, cfg, 4)` over the `3x1024` fold-3 config panics with `"window_size must be > 0"`. | @@ -1306,25 +1306,25 @@ Test file: [`src/sketch_framework/tumbling.rs`](../src/sketch_framework/tumbling | `window_pool_recycles_on_eviction` | Eviction returns sketches to the pool, cleared, and the pool allocates nothing after priming. | For `TumblingWindow::new(100, 2, cfg, 4)` over the `3x1024` fold-3 config fed keys `U64(0..6)` at timestamps `w * 100` - six windows opened, five closed, three evicted - verifies `closed_count()` is `2`, `pool_available()` is exactly `1`, and `pool_total_allocated()` is still the initial `4`, then verifies the active window (running on the sketch window 2 handed back) estimates `1` for its own key `U64(5)` and `0` for window 2's key `U64(2)`. | | `query_all_matches_manual_merge` | `query_all` answers exactly as one sketch fed the same stream. | For `TumblingWindow::new(100, 10, cfg, 4)` over the `3x1024` fold-3 config fed keys `U64(0..20)` at timestamps `i * 30`, and a `CountMinSketchFold::new(3, 1024, 3, 10)` fed the same twenty keys, verifies `query_all` and the manual sketch return equal estimates for every key. | | `query_recent_selects_subset` | `query_recent(n)` covers the `n` newest closed windows plus the active one, and nothing older. | For `TumblingWindow::new(100, 10, cfg, 4)` over the `3x1024` fold-3 config holding `Str("old")` at weight `5` in window 0, `Str("new")` at `10` in window 1, and `Str("active")` at `7` in the active window 2, verifies `query_recent(1)` reads `new` as `10`, `active` as `7`, and `old` as `0`. | -| `fold_cms_tumbling_hierarchical_merge` | `query_all_hierarchical` on `CountMinSketchFold` keeps most estimates inside the Count-Min bound. | For `TumblingWindow::new(10_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xCAFE_BABE`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `80_000` samples at timestamps `i`, verifies over `90%` of distinct keys have `\|est - truth\|` under `(e / 4096) * L1`. | +| `count_min_sketch_fold_tumbling_hierarchical_merge` | `query_all_hierarchical` on `CountMinSketchFold` keeps most estimates inside the Count-Min bound. | For `TumblingWindow::new(10_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xCAFE_BABE`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `80_000` samples at timestamps `i`, verifies over `90%` of distinct keys have `\|est - truth\|` under `(e / 4096) * L1`. | | `kll_tumbling_quantile_accuracy` | A merged `TumblingWindow` puts the median inside a 2% rank band. | For `TumblingWindow::new(5_000, 4, cfg, 6)` over `KllConfig { k: 200, m: 8, seed: None }` fed `20_000` seeded (`0xDEAD_BEEF`) uniform values in `[0.0, 1_000_000.0)` at timestamps `i`, verifies `query_all().cdf().query(0.5)` falls between the sorted values at ranks `ceil(0.48n)` and `ceil(0.52n)`. | | `flush_closes_active_window` | `flush` closes the active window even mid-window, and the data stays queryable. | For `TumblingWindow::new(100, 10, cfg, 4)` over the `3x1024` fold-3 config holding `Str("x")` at weight `5` from `t = 10`, verifies `closed_count` goes from `0` to `1` across `flush(50)`, that `active_sketch()` then reads `x` as `0`, and that `query_all` still reads it as `5`. | -| `fold_cs_tumbling_basic` | A `TumblingWindow` merge sums one key's weight across windows. | For `TumblingWindow::new(100, 10, cfg, 4)` over `CountSketchFoldConfig { rows: 3, full_cols: 1024, fold_level: 3, heap_size: 10 }` fed `Str("hello")` at weights `5`, `3`, and `2` at timestamps `0`, `100`, and `200`, verifies `query_all` reads the key as exactly `10`. | -| `fold_cs_tumbling_hierarchical_merge` | `query_all_hierarchical` on `CountSketchFold` unfolds to level `0` and answers every key exactly. | For `TumblingWindow::new(100, 10, cfg, 6)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 2, heap_size: 10 }` fed keys `U64(0..40)` once each across four windows, verifies the merged sketch reports `fold_level()` of `0` and that no key of the forty is off its true `1` by more than `1`. | -| `fold_cms_tumbling_accuracy_zipf` | A flat `query_all` on `CountMinSketchFold` meets the Count-Min bound at its own confidence. | For `TumblingWindow::new(31_250, 16, cfg, 18)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xACC0_BAC1`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `500_000` samples, verifies the share of keys within `(e / 4096) * L1` is at or above `(1 - e^-3) * 100`, and reports mean and max absolute error. | -| `fold_cms_hierarchical_vs_flat_merge` | A hierarchical merge and an unfolded flat merge are the same answer, key for key. | Feeds a seeded (`0xF1A7_CAFE`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `100_000` samples into two identical `TumblingWindow::new(12_500, 8, cfg, 10)` managers over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 3, heap_size: 20 }`, and verifies `query_all_hierarchical` reaches `fold_level` `0` and returns estimates exactly equal to `query_all().unfold_full()` for every key in the truth map. | -| `fold_cs_tumbling_accuracy_zipf` | A flat `query_all` on `CountSketchFold` meets the Count Sketch L2 bound at its own confidence. | For `TumblingWindow::new(31_250, 16, cfg, 18)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xC5_ACCA`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `500_000` samples, verifies the share of keys within `sqrt(e / 4096) * L2` is at or above `(1 - e^-5) * 100`, and reports mean and max absolute error. | +| `count_sketch_fold_tumbling_basic` | A `TumblingWindow` merge sums one key's weight across windows. | For `TumblingWindow::new(100, 10, cfg, 4)` over `CountSketchFoldConfig { rows: 3, full_cols: 1024, fold_level: 3, heap_size: 10 }` fed `Str("hello")` at weights `5`, `3`, and `2` at timestamps `0`, `100`, and `200`, verifies `query_all` reads the key as exactly `10`. | +| `count_sketch_fold_tumbling_hierarchical_merge` | `query_all_hierarchical` on `CountSketchFold` unfolds to level `0` and answers every key exactly. | For `TumblingWindow::new(100, 10, cfg, 6)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 2, heap_size: 10 }` fed keys `U64(0..40)` once each across four windows, verifies the merged sketch reports `fold_level()` of `0` and that no key of the forty is off its true `1` by more than `1`. | +| `count_min_sketch_fold_tumbling_accuracy_zipf` | A flat `query_all` on `CountMinSketchFold` meets the Count-Min bound at its own confidence. | For `TumblingWindow::new(31_250, 16, cfg, 18)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xACC0_BAC1`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `500_000` samples, verifies the share of keys within `(e / 4096) * L1` is at or above `(1 - e^-3) * 100`, and reports mean and max absolute error. | +| `count_min_sketch_fold_hierarchical_vs_flat_merge` | A hierarchical merge and an unfolded flat merge are the same answer, key for key. | Feeds a seeded (`0xF1A7_CAFE`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `100_000` samples into two identical `TumblingWindow::new(12_500, 8, cfg, 10)` managers over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 3, heap_size: 20 }`, and verifies `query_all_hierarchical` reaches `fold_level` `0` and returns estimates exactly equal to `query_all().unfold_full()` for every key in the truth map. | +| `count_sketch_fold_tumbling_accuracy_zipf` | A flat `query_all` on `CountSketchFold` meets the Count Sketch L2 bound at its own confidence. | For `TumblingWindow::new(31_250, 16, cfg, 18)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xC5_ACCA`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `500_000` samples, verifies the share of keys within `sqrt(e / 4096) * L2` is at or above `(1 - e^-5) * 100`, and reports mean and max absolute error. | | `kll_tumbling_multi_quantile_accuracy` | Five quantiles of a merged `TumblingWindow` each land in a 2% rank band. | For `TumblingWindow::new(12_500, 8, cfg, 10)` over `KllConfig { k: 200, m: 8, seed: None }` fed `100_000` seeded (`0x411_00171`) uniform values in `[0.0, 10_000_000.0)`, verifies `query_all().cdf().query(q)` for `q` in `0.10`, `0.25`, `0.50`, `0.75`, `0.90` sits between the sorted values at ranks `q - 0.02` and `q + 0.02`. | | `kll_tumbling_distribution_shift` | Across a mid-stream distribution change the merged CDF keeps the shape of both halves. | For `TumblingWindow::new(12_500, 8, cfg, 10)` over `KllConfig { k: 400, m: 8, seed: None }` fed `50_000` seeded (`0xFA_ACE1`) `normal(100, 10)` values then `50_000` seeded (`0xFA_ACE2`) `normal(500, 50)` values, verifies `query_all().cdf()` puts `p10` below `200.0`, `p50` between `50.0` and `600.0`, `p90` above `350.0`, and `p10 < p50 < p90`. | | `kll_tumbling_seeded_is_byte_identical_across_instances` | A `KllConfig` seed makes the whole tumbling pipeline reproducible, and a different seed does not alias onto it. | Feeds `20_000` seeded (`0x5EED_0001`) uniform values in `[0.0, 1_000_000.0)` at timestamps `i` through two independent `TumblingWindow::new(1000, 4, KllConfig { k: 200, m: 8, seed: Some(0x5EED_CAFE) }, 6)` managers - twenty windows, so the run rotates and evicts - and verifies `query_all().to_asapv1()` is identical; a third manager at seed `0x5EED_CAFE + 1` is verified to produce different bytes. | | `tumbling_eviction_correctness` | After eviction the merge answers the retained windows, and evicted-only keys stay inside the bound. | For `TumblingWindow::new(10_000, 4, cfg, 6)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xE01C_0100`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `80_000` samples over eight windows, verifies over `90%` of the retained truth - windows `3..8`, the four newest closed plus the active - is within `(e / 4096) * retained_L1`, and that every key appearing only in windows `0..3` reads at or under that same bound rather than exactly `0`. | | `tumbling_query_recent_accuracy` | `query_recent` answers its own subset's truth, and excluded keys stay inside the bound. | For `TumblingWindow::new(10_000, 6, cfg, 8)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` retaining all six windows of a seeded (`0xBEC3_0A00`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `60_000` samples, verifies over `90%` of the truth for windows `2..6` is within `(e / 4096) * recent_L1` under `query_recent(3)`, and that every key appearing only in windows `0` and `1` reads at or under that bound. | -| `fold_cms_tumbling_heap_correctness` | The merged `CountMinSketchFold` heap recovers most of the true heavy hitters, with counts and root ordering intact. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xBEAF_C0DE`) Zipf(`domain = 10_000`, `exponent = 1.3`) stream of `200_000` samples, verifies `query_all().heap().heap()` holds exactly `20` residents and holds at least `80%` of the exact top-`20` keys as `OwnedDataInput::U64` entries, that every resident's count is positive and no greater than the merged estimate for its key, that the merged estimate is at or above the exact count (Count-Min's one-sided error), and that the min-heap's root is the smallest resident. | -| `fold_cs_tumbling_query_recent_accuracy` | `query_recent` on `CountSketchFold` meets the L2 bound over its own subset. | For `TumblingWindow::new(10_000, 6, cfg, 8)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` retaining all six windows of a seeded (`0xC5_0A01`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `60_000` samples, verifies over `90%` of the truth for windows `2..6` is within `sqrt(e / 4096) * recent_L2` under `query_recent(3)`, and that every key appearing only in windows `0` and `1` has `\|est\|` at or under that bound. | +| `count_min_sketch_fold_tumbling_heap_correctness` | The merged `CountMinSketchFold` heap recovers most of the true heavy hitters, with counts and root ordering intact. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xBEAF_C0DE`) Zipf(`domain = 10_000`, `exponent = 1.3`) stream of `200_000` samples, verifies `query_all().heap().heap()` holds exactly `20` residents and holds at least `80%` of the exact top-`20` keys as `OwnedDataInput::U64` entries, that every resident's count is positive and no greater than the merged estimate for its key, that the merged estimate is at or above the exact count (Count-Min's one-sided error), and that the min-heap's root is the smallest resident. | +| `count_sketch_fold_tumbling_query_recent_accuracy` | `query_recent` on `CountSketchFold` meets the L2 bound over its own subset. | For `TumblingWindow::new(10_000, 6, cfg, 8)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` retaining all six windows of a seeded (`0xC5_0A01`) Zipf(`domain = 5000`, `exponent = 1.1`) stream of `60_000` samples, verifies over `90%` of the truth for windows `2..6` is within `sqrt(e / 4096) * recent_L2` under `query_recent(3)`, and that every key appearing only in windows `0` and `1` has `\|est\|` at or under that bound. | | `kll_tumbling_query_recent_accuracy` | `query_recent` on `Kll` answers the quantiles of its own subset. | For `TumblingWindow::new(12_500, 8, cfg, 10)` over `KllConfig { k: 200, m: 8, seed: None }` fed `100_000` seeded (`0xA11_0072`) uniform values in `[0.0, 10_000_000.0)`, verifies `query_recent(3).cdf().query(q)` for `q` in `0.10`, `0.25`, `0.50`, `0.75`, `0.90` sits inside a `0.03` rank band of the `50_000` values in windows `4..8`. | -| `fold_cs_tumbling_heap_correctness` | The merged `CountSketchFold` heap recovers most of the true heavy hitters, with counts and root ordering intact. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xC5_4EA9`) Zipf(`domain = 10_000`, `exponent = 1.3`) stream of `200_000` samples, verifies `query_all().heap().heap()` holds exactly `20` residents and holds at least `80%` of the exact top-`20` keys as `OwnedDataInput::U64` entries, that every resident's count is positive, that each resident's merged estimate is within `2%` of its exact count (Count Sketch's error is two-sided), and that the min-heap's root is the smallest resident. | -| `fold_cms_tumbling_vs_monolithic` | Splitting a `CountMinSketchFold` stream into windows costs at most half again the error. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` and a `CountMinSketchFold::new(3, 4096, 4, 20)` both fed a seeded (`0xDE_AD01`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `200_000` samples, verifies the merged window's mean absolute error is at most `1.5` times the single sketch's, and reports both. | -| `fold_cs_tumbling_vs_monolithic` | Splitting a `CountSketchFold` stream into windows costs at most half again the error. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` and a `CountSketchFold::new(5, 4096, 4, 20)` both fed a seeded (`0xDE_AD02`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `200_000` samples, verifies the merged window's mean absolute error is at most `1.5` times the single sketch's, and reports both. | +| `count_sketch_fold_tumbling_heap_correctness` | The merged `CountSketchFold` heap recovers most of the true heavy hitters, with counts and root ordering intact. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed a seeded (`0xC5_4EA9`) Zipf(`domain = 10_000`, `exponent = 1.3`) stream of `200_000` samples, verifies `query_all().heap().heap()` holds exactly `20` residents and holds at least `80%` of the exact top-`20` keys as `OwnedDataInput::U64` entries, that every resident's count is positive, that each resident's merged estimate is within `2%` of its exact count (Count Sketch's error is two-sided), and that the min-heap's root is the smallest resident. | +| `count_min_sketch_fold_tumbling_vs_monolithic` | Splitting a `CountMinSketchFold` stream into windows costs at most half again the error. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` and a `CountMinSketchFold::new(3, 4096, 4, 20)` both fed a seeded (`0xDE_AD01`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `200_000` samples, verifies the merged window's mean absolute error is at most `1.5` times the single sketch's, and reports both. | +| `count_sketch_fold_tumbling_vs_monolithic` | Splitting a `CountSketchFold` stream into windows costs at most half again the error. | For `TumblingWindow::new(25_000, 8, cfg, 10)` over `CountSketchFoldConfig { rows: 5, full_cols: 4096, fold_level: 4, heap_size: 20 }` and a `CountSketchFold::new(5, 4096, 4, 20)` both fed a seeded (`0xDE_AD02`) Zipf(`domain = 10_000`, `exponent = 1.1`) stream of `200_000` samples, verifies the merged window's mean absolute error is at most `1.5` times the single sketch's, and reports both. | | `kll_tumbling_vs_monolithic` | Splitting a `Kll` stream into windows costs at most `0.02` of rank. | For `TumblingWindow::new(12_500, 8, cfg, 10)` over `KllConfig { k: 200, m: 8, seed: None }` and a `Kll::new(200, 8)` both fed `100_000` seeded (`0xDE_AD03`) uniform values in `[0.0, 10_000_000.0)`, verifies the merged window's worst rank error over `q` in `0.10`, `0.25`, `0.50`, `0.75`, `0.90` is at most the single sketch's plus `0.02`, and reports both. | | `tumbling_single_window_accuracy` | With `window_size` past the stream length nothing closes and every payload still answers. | Over `50_000` samples with `window_size` set to `50_001` and `max_windows` `10`, pool cap `4`, verifies `closed_count()` is `0` in three subsections and that `query_all` holds: a `3x4096` fold-4 `CountMinSketchFold` on a seeded (`0x51_0001`) Zipf(`5000`, `1.1`) stream over `90%` within `(e / 4096) * L1`; a `5x4096` fold-4 `CountSketchFold` on a seeded (`0x51_0002`) stream over `90%` within `sqrt(e / 4096) * L2`; and a `Kll { k: 200, m: 8 }` on seeded (`0x51_0003`) uniform values in `[0.0, 1_000_000.0)` with `q` `0.25`, `0.50`, `0.75` inside a `0.02` rank band. | | `tumbling_very_small_windows` | Five hundred rotations under a fifty-window cap still answer the retained tail. | For `TumblingWindow::new(10, 50, cfg, 52)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 1024, fold_level: 3, heap_size: 10 }` fed a seeded (`0x77_1100`) Zipf(`domain = 500`, `exponent = 1.2`) stream of `5000` samples at timestamps `i` - `500` windows, of which only the newest `50` closed plus the active survive - verifies over `85%` of the truth built from samples `4500..5000` is within `(e / 1024) * L1` under `query_all`. | @@ -1734,7 +1734,7 @@ Test file: [`tests/e2e/frequency.rs`](../tests/e2e/frequency.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `countl2hh_weighted_turnstile_satisfies_the_l2_median_bound` | CountSketchWithL2Norm's point estimates obey the Count Sketch L2 bound and its `l2_squared` tracks F2 within the AMS bound under a weighted turnstile stream. | Builds `CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 11)`, feeds `zipf_u64(30_000, 512, 1.3, 1005)` through `update_weighted` at weights `1 + (i % 5)`, then applies a `-10` decrement to the hottest key from `truth.top_k(1)`. Asserts `CountSketchSpec::new(4, 2048).assert_contract` over `estimate` (zero violations of the simultaneous `sqrt(kappa/w)` residual-L2 band at `delta = 1e-3`, marginal rate at `kappa = 3` no greater than `P[Bin(4, 1/3) >= 2]`), and that `SecondMomentSpec::new(4, 2048).check(l2_squared(), truth.f2())` holds, i.e. the F2 relative error is at most `sqrt(2*3/2048)` (about `0.0541`). | +| `count_sketch_with_l2_norm_weighted_turnstile_satisfies_the_l2_median_bound` | CountSketchWithL2Norm's point estimates obey the Count Sketch L2 bound and its `l2_squared` tracks F2 within the AMS bound under a weighted turnstile stream. | Builds `CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 11)`, feeds `zipf_u64(30_000, 512, 1.3, 1005)` through `update_weighted` at weights `1 + (i % 5)`, then applies a `-10` decrement to the hottest key from `truth.top_k(1)`. Asserts `CountSketchSpec::new(4, 2048).assert_contract` over `estimate` (zero violations of the simultaneous `sqrt(kappa/w)` residual-L2 band at `delta = 1e-3`, marginal rate at `kappa = 3` no greater than `P[Bin(4, 1/3) >= 2]`), and that `SecondMomentSpec::new(4, 2048).check(l2_squared(), truth.f2())` holds, i.e. the F2 relative error is at most `sqrt(2*3/2048)` (about `0.0541`). | ### Frequency: CountMinSketchFold and CountSketchFold @@ -1742,7 +1742,7 @@ Test file: [`tests/e2e/frequency.rs`](../tests/e2e/frequency.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `fold_cms_foldcs_counts_and_hierarchical_merge` | Weighted counts are exact on sparse dimensions and survive a same-level merge, a signed update, and a hierarchical merge. | On `CountMinSketchFold::::new(3, 2048, 0, 32)` sketches: 100 inserts of `Str("alpha")` at weight 2 into each of two sketches plus 100 of `Str("beta")` at weight 1 into the second, verifying `query` reads `200`, `200` and `100`; after `merge_same_level`, `400` for alpha and `100` for beta. On `CountSketchFold::::new(3, 2048, 0, 32)`, `Str("gamma")` at `+60` then `-20` is verified to query `40`. Finally `CountMinSketchFold::hierarchical_merge` of two sketches built by `build_fold_cms` (30 and 45 inserts at weight 3) is verified to query `90` and `135`. | +| `count_min_sketch_fold_and_count_sketch_fold_counts_and_hierarchical_merge` | Weighted counts are exact on sparse dimensions and survive a same-level merge, a signed update, and a hierarchical merge. | On `CountMinSketchFold::::new(3, 2048, 0, 32)` sketches: 100 inserts of `Str("alpha")` at weight 2 into each of two sketches plus 100 of `Str("beta")` at weight 1 into the second, verifying `query` reads `200`, `200` and `100`; after `merge_same_level`, `400` for alpha and `100` for beta. On `CountSketchFold::::new(3, 2048, 0, 32)`, `Str("gamma")` at `+60` then `-20` is verified to query `40`. Finally `CountMinSketchFold::hierarchical_merge` of two sketches built by `build_fold_cms` (30 and 45 inserts at weight 3) is verified to query `90` and `135`. | | `folded_sketches_keep_their_own_bounds_through_a_sixteen_way_merge` | Folded sketches keep their own theorems, evaluated at the folded width, through a sixteen-way hierarchical merge. | Splits `zipf_u64(240_000, 10_000, 1.1, 1009)` into 16 windows of 15,000, giving each window a `CountMinSketchFold` and a `CountSketchFold` of `3x4096` at `fold_level = 4` and `top_k = 20`, then `hierarchical_merge`s each family. Asserts `CountMinSpec::new(3, 256)` over `CountMinSketchFold::estimate` and `CountSketchSpec::new(3, 256)` over `CountSketchFold::estimate` - the bounds are evaluated at the folded width `4096 >> 4 = 256`, not the unfolded 4096 - with the specs' usual rules: zero violations for one-sidedness and both simultaneous bounds at `delta = 1e-3`, and the marginal bounds as rate pins at `e^-3` and `P[Bin(3, 1/3) >= 2]`. | ### Frequency: Portable Count-Min and Count Sketch @@ -1813,10 +1813,10 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `kll_bulk_update_is_byte_identical_to_the_update_loop` | `Kll::update_batch` produces the same sketch as a loop of `update`, not merely one in the same rank band. | Over the 9 `bulk_cases(0, 20_000)` streams (the six `rank_streams` shapes plus `exponential(1e-3)` seed `3007`, `log-uniform` on `gamma = 1.01/0.99` bucket edges seed `3005`, and a 10-element `sequential-10`), builds two `Kll` at `k=200` sharing the seed `kll_bulk_seed(case)` and asserts equal `count()`, equal `to_asapv1()` output, and bit-identical `quantile(q)` (`to_bits()`) at every `q` in `[0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0]`. | -| `kll_dynamic_bulk_update_is_byte_identical_to_the_update_loop` | `KllDynamic::update_batch` holds the same byte-level equality on its own wire form. | Same equality on `KllDynamic` at `k=200`, `n=10_000`, over the 9 `bulk_cases(1, n)` streams with seed `kll_bulk_seed(0x0100 + case)`: asserts equal `count()`, equal `to_asapv1()`, and equal `quantile(q).to_bits()` over the same 7-point q grid. | -| `kll_bulk_update_on_a_degenerate_slice_matches_the_loop` | The two degenerate slice lengths behave as a no-op and as a single `update`. | Feeds `1.0, 2.0, 3.0` into a `k=200` sketch at `kll_bulk_seed(0x0200)`, then asserts `update_batch(&[])` leaves `count()`, `quantile(0.5).to_bits()` and `to_asapv1()` unchanged; and that a fresh sketch given `update_batch(&[42.0])` serializes byte-identically to one given `update(&42.0)`. | -| `kll_bulk_update_data_input_matches_the_loop_and_stops_at_the_first_non_numeric` | The `DataInput` batch path equals the loop and aborts at the first non-numeric element with the prefix applied. | Wraps 5_000 `normal_f64(100, 20)` values (seed `8001`) as `DataInput::F64` and asserts `update_batch_data_input` and a loop of `update_data_input` serialize identically at `k=200`, seed `kll_bulk_seed(0x0300)`. Then feeds `[F64(1.0), String("x"), F64(2.0)]` and asserts the call returns `Err` while `count() == 1`, and that `update_batch_data_input(&[])` returns `Ok` leaving `count() == 0`. | +| `kll_update_batch_is_byte_identical_to_the_update_loop` | `Kll::update_batch` produces the same sketch as a loop of `update`, not merely one in the same rank band. | Over the 9 `bulk_cases(0, 20_000)` streams (the six `rank_streams` shapes plus `exponential(1e-3)` seed `3007`, `log-uniform` on `gamma = 1.01/0.99` bucket edges seed `3005`, and a 10-element `sequential-10`), builds two `Kll` at `k=200` sharing the seed `kll_bulk_seed(case)` and asserts equal `count()`, equal `to_asapv1()` output, and bit-identical `quantile(q)` (`to_bits()`) at every `q` in `[0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0]`. | +| `kll_dynamic_update_batch_is_byte_identical_to_the_update_loop` | `KllDynamic::update_batch` holds the same byte-level equality on its own wire form. | Same equality on `KllDynamic` at `k=200`, `n=10_000`, over the 9 `bulk_cases(1, n)` streams with seed `kll_bulk_seed(0x0100 + case)`: asserts equal `count()`, equal `to_asapv1()`, and equal `quantile(q).to_bits()` over the same 7-point q grid. | +| `kll_update_batch_on_a_degenerate_slice_matches_the_loop` | The two degenerate slice lengths behave as a no-op and as a single `update`. | Feeds `1.0, 2.0, 3.0` into a `k=200` sketch at `kll_bulk_seed(0x0200)`, then asserts `update_batch(&[])` leaves `count()`, `quantile(0.5).to_bits()` and `to_asapv1()` unchanged; and that a fresh sketch given `update_batch(&[42.0])` serializes byte-identically to one given `update(&42.0)`. | +| `kll_update_batch_data_input_matches_the_loop_and_stops_at_the_first_non_numeric` | The `DataInput` batch path equals the loop and aborts at the first non-numeric element with the prefix applied. | Wraps 5_000 `normal_f64(100, 20)` values (seed `8001`) as `DataInput::F64` and asserts `update_batch_data_input` and a loop of `update_data_input` serialize identically at `k=200`, seed `kll_bulk_seed(0x0300)`. Then feeds `[F64(1.0), String("x"), F64(2.0)]` and asserts the call returns `Err` while `count() == 1`, and that `update_batch_data_input(&[])` returns `Ok` leaving `count() == 0`. | ### Quantiles: KLL Geometry Axes @@ -1918,7 +1918,7 @@ Test file: [`tests/e2e/heavy_hitters.rs`](../tests/e2e/heavy_hitters.rs) | `elastic_merge_max_never_reads_below_either_side_on_disjoint_flow_sets` | Maximum merging never reads a flow below its own side. | Builds two `ElasticSketch::with_heavy_and_light(64, 3, 1_024)` sketches on disjoint streams (16 elephants at 300 repeats plus 2,000 distinct mice each, prefixed `l-` and `r-`), calls `merge_max`, and verifies for every flow of either stream that the merged `query` is at least that flow's own side's `query`. | | `elastic_merge_max_and_sum_merging_agree_on_every_heavy_flow` | Maximum merging never reads a heavy flow above summing, and never loses it. | Merges the same two disjoint `64 / 3x1024` sketches twice, once with `merge_max` and once with `merge`, then for every flow `heavy_hitters(200)` reports on the summed side verifies the maximum-merged `query` is at most the summed one and strictly above `0`; the two merges are only ordered, not asserted equal, so the name's "agree" is an inequality here. | | `elastic_merge_heavy_and_absorb_evicted_carry_a_transferred_resident` | A transferred resident and an absorbed eviction both land where a query can reach them. | On fresh `ElasticSketch::with_heavy_and_light(64, 3, 1_024)` sketches, verifies `merge_heavy("transferred", 500, false)` makes `query` read exactly `500` and a following `merge_heavy("transferred", 0, true)` leaves it at `500`; that `merge_heavy("spilled", 400, true)` followed by `absorb_evicted("spilled", 90)` still reads at least `400`; and that `absorb_evicted("mouse", 75)` with no heavy bucket reads at least `75` through the light layer. | -| `elastic_insert_heavy_only_never_writes_the_light_layer` | The heavy-only insert path leaves the light layer untouched. | Sends 64 flows named `flow-{e:03}` at 20 records each through `update_heavy_only` on `ElasticSketch::with_heavy_and_light(8, 3, 256)` and verifies `light.estimate(&DataInput::String(id))` is exactly `0` for all 64 flows; what the heavy table stored is not checked. | +| `elastic_update_heavy_only_never_writes_the_light_layer` | The heavy-only insert path leaves the light layer untouched. | Sends 64 flows named `flow-{e:03}` at 20 records each through `update_heavy_only` on `ElasticSketch::with_heavy_and_light(8, 3, 256)` and verifies `light.estimate(&DataInput::String(id))` is exactly `0` for all 64 flows; what the heavy table stored is not checked. | | `elastic_expansion_keeps_every_resident_readable_and_compression_restores_the_width` | Expansion keeps every resident readable and compression keeps every one of them present. | Over `ElasticSketch::with_heavy_and_light(64, 3, 1_024)` fed 24 `x-` elephants at 250 repeats plus 2,000 mice, records `heavy_hitters(200)` and `full_bucket_count(100)`, calls `expand_heavy()` and verifies every recorded flow's `query` is still at least its pre-expansion size and `full_bucket_count(100)` did not fall, then calls `compress_heavy(2)` and verifies every recorded flow still queries above `0`; the bucket count after compression is never read, so the name's "restores the width" is not asserted. | | `elastic_heavy_changes_reports_only_flows_that_moved_past_the_threshold` | `heavy_changes` reports exactly the flows whose size moved past the threshold it was given. | Builds two `64 / 3x1024` windows over 16 `w-` elephants at 200 repeats plus 2,000 mice, the second carrying 600 extra `w-elephant-000` records, and verifies `heavy_changes(&first, 100)` includes `w-elephant-000`, that every reported entry's `\|after - before\|` exceeds `100`, that `heavy_changes(&first, 10_000)` is empty, and that the second window compared against itself at a threshold of `0` is empty. | | `a_heavy_bucket_seats_evicts_and_reports_vacancy` | A heavy bucket seats a flow, hands the slot to a takeover, and reports its own vacancy. | Verifies `HeavyBucket::new()` is vacant; that `occupy("first")` leaves it non-vacant with `flow_id` `first`, `vote_pos` `1`, `vote_neg` `0` and no `eviction` flag; that `occupy_many("bulk", 40)` sets `vote_pos` to `40`; that `evict_many("takeover", 12)` returns `bulk` and leaves `flow_id` `takeover` with `vote_pos` and `vote_neg` both `12` and `eviction` raised; and that `evict("single")` returns `takeover` and leaves `vote_pos` at `1`. | @@ -1982,7 +1982,7 @@ Test file: [`tests/e2e/frameworks.rs`](../tests/e2e/frameworks.rs) | `hydra_univmon_head_l1_is_exact_per_subpopulation` | A weighted L1 survives the fan-out exactly in every subpopulation. | Ingests `zipf_u64(30_000, 1000, 1.2, 4_660)` as `U32` under `h2_keys(30_000, 4_650)` with weight `1 + i % 7` into a `5x256` `Hydra` of `UnivMon::new(32, 5, 256, 8)`, asserts the lattice has exactly 8 members, and `assert_eq!`s `L1Norm` against the exact weighted total for each; no other metric is checked. | | `hydra_shard_merge_preserves_answers_for_every_counter` | A two-shard merge reproduces the single pass for the Count Sketch, HLL, KLL and UnivMon heads. | For each counter, ingests `h2_keys(24_000, 4_670)` into one single-pass and two shard grids (`5x256`, columns `[region, service]`), splitting by `uniform_u64(24_000, 2, 4_675)`, merges, and probes the three `h2_masks("eu-west", "auth")` keys: `3x64` `CountSketch` cells must match the single pass exactly on all four endpoints; `HyperLogLog(Default)` exactly on `Cardinality`; `Kll::with_k_and_seed(200, 4_681)` fed shard-separated `normal_f64(_, 300.0, 20.0, 4_680)` and `normal_f64(_, 700.0, 20.0, 4_685)` modes must land in the exact rank band of half-width `eps(200)` at `q` in `[0.1, 0.25, 0.5, 0.75, 0.9]`; and `UnivMon::new(32, 5, 256, 8)` over `zipf_u64(24_000, 800, 1.2, 4_690)` must give an `L1Norm` exactly equal to both the exact weighted total and the single pass, with `L2Norm` inside `univmon_l2_band(exact, 5, 256)`, i.e. `[L2*sqrt(1-b), L2*sqrt(1+b)]` for `b = sqrt(2*3/256)`. Despite the name, Count-Min is not one of the counters exercised here. | | `hydra_query_frequency_is_the_frequency_query_it_wraps` | `query_frequency` answers exactly what the `Frequency` query it wraps answers. | Feeds 25 records of `Str("event")` under each of the four `[eu, us] x [alice, bob]` keys into a `4x4096` `Hydra` of `4x4096` fast-path `CountMinSketch`, then `assert_eq!`s `Hydra::query_frequency` against `query_key` with `HydraQuery::Frequency` on five keys (`[eu, alice]`, `[us, bob]`, `[eu, None]`, `[None, bob]`, `[apac, None]`), and verifies a 1-element key is an error while `[apac, nobody]` answers exactly `0.0`. Only agreement with the wrapped query is pinned; no accuracy band is asserted. | -| `hydra_query_quantile_is_the_cumulative_query_it_wraps` | `query_cdf` answers the `Cdf` query it wraps, inside KLL's rank error. | Ingests 20_000 `uniform_u64(20_000, 1_000_000, 4_101)` draws as `F64` under key `s0` into a `4x512` `Hydra` of `Kll::with_k_and_seed(200, 0x5EED_0700)`, then for `x` in `{100_000, 250_000, 500_000, 750_000, 900_000}` `assert_eq!`s `query_cdf(key, x)` against `query_key(key, Cdf(x))` - the wrapper returns a CDF, not a quantile - and requires it within `eps(200)` (about `0.0165`) of the exact CDF; also that `ghost` reads exactly `0.0` and that an empty key `&[]` is an error. | +| `hydra_query_cdf_is_the_cumulative_query_it_wraps` | `query_cdf` answers the `Cdf` query it wraps, inside KLL's rank error. | Ingests 20_000 `uniform_u64(20_000, 1_000_000, 4_101)` draws as `F64` under key `s0` into a `4x512` `Hydra` of `Kll::with_k_and_seed(200, 0x5EED_0700)`, then for `x` in `{100_000, 250_000, 500_000, 750_000, 900_000}` `assert_eq!`s `query_cdf(key, x)` against `query_key(key, Cdf(x))` - the wrapper returns a CDF, not a quantile - and requires it within `eps(200)` (about `0.0165`) of the exact CDF; also that `ghost` reads exactly `0.0` and that an empty key `&[]` is an error. | ### Frameworks: UnivMon Pipelines @@ -1990,8 +1990,8 @@ Test file: [`tests/e2e/frameworks.rs`](../tests/e2e/frameworks.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `univmon_weighted_metrics_and_fast_insert_parity` | UnivMon's weighted L1 is exact and its L2, entropy and cardinality hold their bands, with `fast_insert` tracking the standard path. | Drives two `UnivMon::new(32, 5, 2048, 8)` sketches over `zipf_u64(20_000, 1000, 1.2, 4003)` with weight `1 + i % 7`, one through `update` and one through `fast_insert`, then asserts both `estimate_l1()` equal the exact weighted total with `assert_eq!`; `estimate_l2()` inside `univmon_l2_band(exact, 5, 2048)` (`b = sqrt(2*3/2048)`, about `0.054`); `estimate_entropy()` in `[0.88, 1.12]` times the exact bits; `estimate_cardinality()` in `[0.94, 1.06]` times the exact distinct count; and the fast path's `estimate_l2()` in `[0.85, 1.15]` times exact. | -| `univmon_pyramid_weighted_metrics` | UnivMonTwoTier's weighted L1 is exact and its L2 and cardinality hold their bands. | Drives `UnivMonTwoTier::with_defaults()` over `zipf_u64(15_000, 800, 1.3, 4004)` with weight `1 + i % 3` and asserts `estimate_l1()` equals the exact weighted total with `assert_eq!`, `estimate_l2()` is in `[0.85, 1.15]` times exact, and `estimate_cardinality()` in `[0.80, 1.20]` times the exact distinct count; entropy is not checked. | +| `univmon_weighted_metrics_and_update_weighted_terminal_parity` | UnivMon's weighted L1 is exact and its L2, entropy and cardinality hold their bands, with `fast_insert` tracking the standard path. | Drives two `UnivMon::new(32, 5, 2048, 8)` sketches over `zipf_u64(20_000, 1000, 1.2, 4003)` with weight `1 + i % 7`, one through `update` and one through `fast_insert`, then asserts both `estimate_l1()` equal the exact weighted total with `assert_eq!`; `estimate_l2()` inside `univmon_l2_band(exact, 5, 2048)` (`b = sqrt(2*3/2048)`, about `0.054`); `estimate_entropy()` in `[0.88, 1.12]` times the exact bits; `estimate_cardinality()` in `[0.94, 1.06]` times the exact distinct count; and the fast path's `estimate_l2()` in `[0.85, 1.15]` times exact. | +| `univmon_two_tier_weighted_metrics` | UnivMonTwoTier's weighted L1 is exact and its L2 and cardinality hold their bands. | Drives `UnivMonTwoTier::with_defaults()` over `zipf_u64(15_000, 800, 1.3, 4004)` with weight `1 + i % 3` and asserts `estimate_l1()` equals the exact weighted total with `assert_eq!`, `estimate_l2()` is in `[0.85, 1.15]` times exact, and `estimate_cardinality()` in `[0.80, 1.20]` times the exact distinct count; entropy is not checked. | | `univmon_generic_g_sum_reproduces_every_named_estimator` | The generic g-sum reproduces the L2, cardinality and entropy estimators. | Over `UnivMon::new(32, 5, 2_048, 8)` fed `zipf_u64(20_000, 1_000, 1.2, 4_201)` with weight `1 + i % 7`, `assert_eq!`s `estimate_g_sum(\|x\| x*x, false).sqrt()` against `estimate_l2()`, `estimate_g_sum(\|_\| 1.0, true)` against `estimate_cardinality()`, and `estimate_g_sum(\|_\| 0.0, false)` against `0.0`; requires `estimate_entropy()` to equal `log2(L1) - gsum(x*log2 x)/L1` within `1e-9`; and checks `estimate_g_sum(\|x\| x, false)` only against `[0.50, 1.50]` times the exact L1, so the identity g-sum's agreement with L1 is not pinned tightly. | | `univmon_g_sum_is_zero_on_an_empty_sketch` | An empty UnivMon reports zero for every estimator. | On a fresh `UnivMon::new(32, 5, 256, 4)`, `assert_eq!`s `estimate_l1()`, `estimate_entropy()`, `estimate_g_sum(\|x\| x*x, false)` and `estimate_cardinality()` each against `0.0`. | | `a_univmon_pool_recycles_a_returned_sketch_and_hands_back_a_cleared_one` | A returned sketch becomes available again and comes back cleared without a new allocation. | On `UnivMonPool::new(2, 16, 3, 256, 4)`: asserts `available() == 2` and `total_allocated() == 2` when fresh, `available() == 0` after two `take()`s with `total_allocated()` still `2`, and `total_allocated() == 3` after a third `take()` past capacity; then fills the first sketch with `zipf_u64(5_000, 512, 1.1, 4_202)` inserts (checking `estimate_l1() > 0.0`), `put()`s it back and asserts `available() == 1`, and requires the next `take()` to report `estimate_l1() == 0.0` with `total_allocated()` still `3`. | @@ -2028,7 +2028,7 @@ Test file: [`tests/e2e/windows.rs`](../tests/e2e/windows.rs) | `every_eh_variant_selects_the_documented_merge_norm` | Each `EhBucketSketch` payload selects the merge norm its enclosing histogram uses. | For `CountSketchWithL2Norm` (`CountSketchWithL2Norm::with_dimensions(3, 512)`) and `UnivMon` (`UnivMon::new(32, 3, 512, 4)`) asserts `supports_norm(SketchNorm::L2)` and `!supports_norm(SketchNorm::L1)`, and that `ExponentialHistogram::new(8, 1_000_000, proto).merge_norm` is `SketchNorm::L2`; for `CountMinSketch` and `CountSketch` (`Vector2D` `FastPath` at `3x512`), `CocoSketch(4, 512)`, `ElasticSketch(512)`, `Hll`, `Kll(k=200, seed 0x5EED_0100)` and `DdSketch(alpha=0.01)` asserts `supports_norm(SketchNorm::L1)` and `merge_norm == SketchNorm::L1`. | | `eh_count_min_variant_conforms_to_the_count_min_model_over_the_retained_window` | A Count-Min payload satisfies Count-Min's theorem over the histogram's full retained span. | Feeds `10_000` Zipf(1.1) keys over a `2_048` domain (seed `0x0E11_0001`), one per timestamp `t = 0..9_999`, into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, merges the full retained span with `query_interval(payload.first().min_time, payload.last().max_time)`, and runs `CountMinSpec::new(3, 512).assert_contract` over the exact truth for that span: one-sidedness `est >= f` with zero violations, the simultaneous budget `b*(N-f)/w` with `b = (D/1e-3)^(1/3)` and zero violations, and the marginal `e*(N-f)/w` as a rate pin at `e^-3 = 0.0498`. | | `eh_count_sketch_variant_conforms_to_the_l2_model_over_the_retained_window` | A Count Sketch payload satisfies the two-sided L2 bound over the retained span. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketch(CountSketch::, FastPath>::with_dimensions(3, 512))`; the full-span merge is judged by `CountSketchSpec::new(3, 512).assert_contract`, which requires `\|est - f\|` under `sqrt(kappa/512)*\|\|f_-i\|\|_2` at the simultaneous `kappa` (union-bounded to `delta = 1e-3` over every probed key, zero violations) and pins the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` violation rate at `P[Bin(3, 1/3) >= 2] = 7/27`. | -| `eh_countl2hh_variant_satisfies_the_l2_bound_over_the_retained_window` | A `CountSketchWithL2Norm` payload keeps the L2 bound of the Count Sketch matrix underneath it. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketchWithL2Norm(CountSketchWithL2Norm::with_dimensions(3, 512))`; the full-span merge is held to `CountSketchSpec::new(3, 512).assert_contract` - the simultaneous L2 band at `delta = 1e-3` with zero violations and the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` rate pin - reading frequencies through `merged.query(&DataInput::U64(k))`. | +| `eh_count_sketch_with_l2_norm_variant_satisfies_the_l2_bound_over_the_retained_window` | A `CountSketchWithL2Norm` payload keeps the L2 bound of the Count Sketch matrix underneath it. | Same `10_000`-key Zipf(1.1) stream, domain `2_048`, seed `0x0E11_0001`, `k = 8`, window `1_000_000`, payload `EhBucketSketch::CountSketchWithL2Norm(CountSketchWithL2Norm::with_dimensions(3, 512))`; the full-span merge is held to `CountSketchSpec::new(3, 512).assert_contract` - the simultaneous L2 band at `delta = 1e-3` with zero violations and the marginal `sqrt(3/512)*\|\|f_-i\|\|_2` rate pin - reading frequencies through `merged.query(&DataInput::U64(k))`. | | `eh_elastic_variant_stays_one_sided_over_the_retained_window` | The Elastic payload never reads a retained heavy key below its true count. | For `ElasticSketch(1024)` inside `ExponentialHistogram::new(8, 1_000_000, ..)`, feeds `10_000` Zipf(1.1) draws over a `2_048` domain (seed `0x0E11_0001`) as `DataInput::String("f{k}")` at `t = 0..9_999`, merges the full retained span, and asserts `est >= c` for the truth's top `32` keys with zero violations. No upper bound is placed on the reported counts, and keys outside the top `32` are not probed. | | `eh_coco_variant_conserves_the_retained_mass` | The merged Coco window holds exactly the retained mass. | Same EH, stream and merge over `CocoSketch(CocoSketch::with_dimensions_and_seed(4, 1024, COCO_SEED))`, and verifies the merged table's recorded mass equals the number of events in the retained span. | | `eh_hll_variant_satisfies_the_register_error_model_over_the_retained_window` | A merged HLL payload lands inside the register error model for the retained span's cardinality. | Feeds `10_000` uniform draws over `200_000` (seed `0x0E11_0001`) into `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::Hll(HyperLogLog::::new()))`, merges the full retained span, and checks `merged.query(&DataInput::Str("card"))` against the exact distinct count of that span under `CardinalityConfidenceSpec::hll(14, 4.0)` - a relative band of `4 * 1.04 / 2^7 = 0.0325` at `m = 2^14` registers. This is a single trial, aggregated by `assert_independent_binomial` at `p = 6.3e-5`, so no violation is tolerated. | @@ -2046,7 +2046,7 @@ Test file: [`tests/e2e/windows.rs`](../tests/e2e/windows.rs) | --- | --- | --- | | `eh_expires_buckets_past_the_window_and_reports_its_retained_span` | Buckets older than the window are dropped whole, so the retained span can run longer than the window, and it is reported honestly. | With `ExponentialHistogram::new(8, 100, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))` updated with `Str("req")` at every `t` in `0..500` divisible by `3`, asserts `max_time()` is `498`, `cover(min_time(), 498)` is true, `cover(500, 600)` is false, and that every retained bucket's `max_time` is at or past the cutoff `498 - 100 = 398`; the full-span merge is then required only to read `>=` the number of retained multiples of `3`, so the exact retained count is not pinned and `min_time()` is not pinned to a value. The survival test is per bucket, not per event: a bucket lives while its `max_time` reaches the cutoff, so its `min_time` may lie before the cutoff and `min_time()` may sit up to the oldest bucket's own span earlier than `max_time - 100`. | | `an_exponential_histogram_expires_against_the_window_length_it_was_last_given` | Expiry is lazy — it runs inside the insert path, so `update_window` takes effect on the next insert and never on its own — and widening the window cannot resurrect expired buckets. | With `ExponentialHistogram::new(8, 10_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))` fed `200` updates of `U64(t % 32)` at timestamps `t*10` (`0` to `1_990`), asserts the retained span starts at `0`; after `update_window(100)` and one update at `t = 2_000`, asserts the span start moved strictly later and `max_time() == Some(2_000)`; after `update_window(10_000)` and an update at `t = 2_010`, asserts the span start is still at or past the narrow one. The window changes are checked by inequality only: no exact retained span is pinned after either call. | -| `an_exponential_histogram_custom_bucket_update_matches_repeated_inserts` | `update_with` reproduces `update`'s bucket structure and answers, and a doubled custom insert never reads lower. | Runs two `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))` over `t = 0..499` with key `U64(t % 64)`, one through `update` and one through `update_with` performing a single `update`, then asserts equal `bucket_count()` and, for all `64` keys, exactly equal estimates from the two full-span merges. A third histogram whose `update_with` inserts the key twice is asserted only to read `>=` the single-insert histogram per key - the doubling is not pinned to a factor of two. | +| `an_exponential_histogram_custom_bucket_update_matches_repeated_updates` | `update_with` reproduces `update`'s bucket structure and answers, and a doubled custom insert never reads lower. | Runs two `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))` over `t = 0..499` with key `U64(t % 64)`, one through `update` and one through `update_with` performing a single `update`, then asserts equal `bucket_count()` and, for all `64` keys, exactly equal estimates from the two full-span merges. A third histogram whose `update_with` inserts the key twice is asserted only to read `>=` the single-insert histogram per key - the doubling is not pinned to a factor of two. | ### Windows: Tumbling Windows and Sketch Pools @@ -2054,9 +2054,9 @@ Test file: [`tests/e2e/windows.rs`](../tests/e2e/windows.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `tumbling_fold_cs_windows_are_exact_and_answers_satisfy_the_l2_bound` | Every `CountSketchFold` window slice covers exactly its time range and answers inside the L2 bound at the folded width. | With `CountSketchFoldConfig { rows: 3, full_cols: 4_096, fold_level: 2, heap_size: 32 }` in `TumblingWindow::new(500, 16, cfg, 4)` fed `3_000` Zipf(1.1) keys over a `512` domain (seed `0x0E11_0001`) at `t = 0..2_999`, asserts `closed_count() == 5`, then runs `CountSketchSpec::new(3, 1_024)` (`4_096 >> 2`) against exact truth for `active_sketch()` (the last `500` keys), `query_all()` (all `3_000`) and `query_recent(2)` (the last `1_500`). After `flush(3_000)`, inserting `U64(7)` at `t = 3_000` must make the recycled active sketch read exactly `1`, and `closed_count()` must be at least `6`. | +| `tumbling_count_sketch_fold_windows_are_exact_and_answers_satisfy_the_l2_bound` | Every `CountSketchFold` window slice covers exactly its time range and answers inside the L2 bound at the folded width. | With `CountSketchFoldConfig { rows: 3, full_cols: 4_096, fold_level: 2, heap_size: 32 }` in `TumblingWindow::new(500, 16, cfg, 4)` fed `3_000` Zipf(1.1) keys over a `512` domain (seed `0x0E11_0001`) at `t = 0..2_999`, asserts `closed_count() == 5`, then runs `CountSketchSpec::new(3, 1_024)` (`4_096 >> 2`) against exact truth for `active_sketch()` (the last `500` keys), `query_all()` (all `3_000`) and `query_recent(2)` (the last `1_500`). After `flush(3_000)`, inserting `U64(7)` at `t = 3_000` must make the recycled active sketch read exactly `1`, and `closed_count()` must be at least `6`. | | `tumbling_univmon_q_windows_carry_exact_aggregates_through_rotation` | UnivMon-Q's exact aggregates stay exact on every window slice and survive pool recycling. | With `UnivMonQConfig { levels: 6, width: 1_024, depth: 5, candidates: 256, ordered_samples: 512, ..default() }` in `TumblingWindow::new(500, 16, cfg, 4)` fed `3_000` uniform values over `100_000` (seed `0x0E11_0001`) at `t = 0..2_999`, asserts `closed_count() == 5` and, for `query_all()` (all `3_000`), `query_recent(1)` (the last `1_000`) and `active_sketch()` (the last `500`), that `count()` equals the slice length and `min()`/`max()` equal the slice's exact extremes; after `flush(3_000)` and one insert of `42.0`, asserts `count() == 1` and `min() == max() == Some(42.0)`. UnivMon-Q's estimators are not exercised here. | -| `tumbling_fold_cms_hierarchical_merge_covers_every_observation` | `query_all_hierarchical` on `CountMinSketchFold` never narrows the sketch, never underestimates, and never exceeds the flat merge. | With `CountMinSketchFoldConfig { rows: 3, full_cols: 4_096, fold_level: 2, heap_size: 32 }` in `TumblingWindow::new(500, 16, cfg, 4)` fed `3_000` Zipf(1.1) keys over a `512` domain (seed `0x0E11_0001`), asserts `query_all_hierarchical().fold_cols() >= query_all().fold_cols()`, runs `CountMinSpec::new(3, hierarchical.fold_cols()).assert_contract` against the exact truth for all `3_000` observations, and asserts per key that the hierarchical estimate is at or above the true count and at or below the flat `query_all()` estimate, plus `total_entries() > 0`. | +| `tumbling_count_min_sketch_fold_hierarchical_merge_covers_every_observation` | `query_all_hierarchical` on `CountMinSketchFold` never narrows the sketch, never underestimates, and never exceeds the flat merge. | With `CountMinSketchFoldConfig { rows: 3, full_cols: 4_096, fold_level: 2, heap_size: 32 }` in `TumblingWindow::new(500, 16, cfg, 4)` fed `3_000` Zipf(1.1) keys over a `512` domain (seed `0x0E11_0001`), asserts `query_all_hierarchical().fold_cols() >= query_all().fold_cols()`, runs `CountMinSpec::new(3, hierarchical.fold_cols()).assert_contract` against the exact truth for all `3_000` observations, and asserts per key that the hierarchical estimate is at or above the true count and at or below the flat `query_all()` estimate, plus `total_entries() > 0`. | | `tumbling_window_pool_accounting_tracks_every_recycled_sketch` | The window pool's allocation accounting balances against the windows that hold its sketches. | With `KllConfig { k: 200, m: 8, seed: Some(0x7001_0001) }` in `TumblingWindow::new(100, 2, cfg, 4)`, asserts `pool_total_allocated() == 4` and `pool_available() == 3` before any insert; after `800` inserts of `U64(t % 97)` at `t = 0..799`, asserts `closed_count() == 2`, `pool_available() > 0`, `pool_total_allocated() >= 4`, and the invariant `pool_available() + closed_count() + 1 == pool_total_allocated()`. | | `a_sketch_pool_reuses_a_returned_sketch_before_allocating_a_new_one` | A returned sketch is handed back cleared and in preference to a fresh allocation. | On `SketchPool::::new(2, KllConfig { k: 200, m: 8, seed: Some(0x7001_0002) })` asserts `available() == 2` and `total_allocated() == 2`; two `take()` calls leave `available() == 0` with `total_allocated()` still `2`; a third `take()` raises `total_allocated()` to `3`; the first sketch is dirtied with `1_000` `update` calls and returned with `put`, after which `available() == 1`, the next `take()` returns a sketch with `count() == 0`, and `total_allocated()` is still `3`. | @@ -2119,8 +2119,8 @@ Test file: [`tests/e2e/matrix_instances.rs`](../tests/e2e/matrix_instances.rs) | `csheap_{regular_path_instances_satisfy_the_l2_bound, fast_path_instances_conform_to_the_l2_model}` | Every `CountSketchWithHeap` instance carries the L2 bound and keeps its heap consistent with the sketch. | One shared body per storage: three `CountSketchWithHeap` sketches over `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, whole stream to one and even/odd halves to two `merge`d shards, then `CountSketchSpec::new(rows, cols)` over both - simultaneous `sqrt(kappa/w)*\|\|f_-i\|\|_2` at the union-bounded `kappa` for `delta = 1e-3` with zero tolerance, marginal band at `kappa = 3` rate-pinned - and `assert_heap_matches_sketch` with `heap_count_from_estimate` as the estimate-to-entry map, requiring every heap entry's `count` to equal `heap_count_from_estimate(estimate)` and at most one entry to sit below the true k-th count. Nine instances each: `Vector2D`, `Vector2D`, `DefaultMatrixI32`, `DefaultMatrixI64`, `DefaultMatrixI128` from `Default` at `3x4096`, `FixedMatrix`, `QuickMatrixI64`, `QuickMatrixI128` from `Default` at `5x2048`, and `Vector2D` built explicitly by `CountSketchWithHeap::new(3, 4096, 32)` because it has no `Default`. Generated as `csheap_regular_path_instances_satisfy_the_l2_bound` (`RegularPath`) and `csheap_fast_path_instances_conform_to_the_l2_model` (`FastPath`). | | `countmin_both_paths_are_exact_on_a_collision_free_workload` | On a collision-free workload both hashing paths return exact counts on every storage backend. | Inserts the eight well-separated keys `1`, `7`, `4_242`, `90_210`, `1_000_003`, `2_147_483_647`, `4_294_967_311`, `9_007_199_254_740_993` at multiplicities `10, 20, ..., 80` into a `RegularPath` and a `FastPath` `CountMinSketch` over the same storage, and requires both `estimate` calls to equal the true count exactly for every key, with zero violations. Run over all ten storages: `Vector2D`, `Vector2D`, `Vector2D`, `Vector2D`, `FixedMatrix`, `DefaultMatrixI32`, `QuickMatrixI64`, `QuickMatrixI128`, `DefaultMatrixI64`, `DefaultMatrixI128`. | | `countsketch_both_paths_are_exact_on_a_collision_free_workload` | The signed family is likewise exact on both paths when nothing collides. | Same eight keys at multiplicities `10, 20, ..., 80` into a `RegularPath` and a `FastPath` `CountSketch` over the same storage, requiring both `estimate` values to equal the true count exactly with zero violations, over nine storages: `Vector2D`, `Vector2D`, `Vector2D`, `FixedMatrix`, `DefaultMatrixI32`, `QuickMatrixI64`, `QuickMatrixI128`, `DefaultMatrixI64`, `DefaultMatrixI128`. | -| `cmsheap_inert_instances_construct_and_report_their_geometry` | The `CountMinSketchWithHeap` instances whose counter type fails the insert bound still construct and answer their geometry. | For each of six instances, verifies `rows()`, `cols()`, `count_min_sketch().rows()` and an empty `heap()`: `CountMinSketchWithHeap::, _>::new(3, 4096, 32)` on both paths at `3x4096`, `CountMinSketchWithHeap::::default()` on both paths at `5x2048`, and `CountMinSketchWithHeap::::default()` on both paths at `3x4096`. This is constructibility only: the test does not assert that `update`, `update_weighted`, `estimate` or `merge` are uncallable, and adding such an impl would leave every assertion here passing. | -| `csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping` | An `i128` `CountSketchWithHeap` holds mass past `i64::MAX` while its heap entry saturates at the ceiling. | For each of six instances, inserts `update_weighted(U64(0xC0FF_EE01), 1 << 40)` and requires `estimate` to equal `2^40` exactly and the heap entry to carry the same `i64`; then, on a fresh sketch, `update_weighted` of `i64::MAX as i128 * 4` and requires `estimate >= i64::MAX as f64`, `heap_count_from_estimate(estimate) == i64::MAX`, and the heap entry to read exactly `i64::MAX` rather than a wrapped negative; then merges a second sketch carrying the same overflow weight and requires the heap entry to stay at `i64::MAX`. Instances: `CountSketchWithHeap::, _>::new(3, 4096, 32)` on both paths, and `QuickMatrixI128` and `DefaultMatrixI128` from `Default` on both paths. | +| `count_min_sketch_with_heap_inert_instances_construct_and_report_their_geometry` | The `CountMinSketchWithHeap` instances whose counter type fails the insert bound still construct and answer their geometry. | For each of six instances, verifies `rows()`, `cols()`, `count_min_sketch().rows()` and an empty `heap()`: `CountMinSketchWithHeap::, _>::new(3, 4096, 32)` on both paths at `3x4096`, `CountMinSketchWithHeap::::default()` on both paths at `5x2048`, and `CountMinSketchWithHeap::::default()` on both paths at `3x4096`. This is constructibility only: the test does not assert that `update`, `update_weighted`, `estimate` or `merge` are uncallable, and adding such an impl would leave every assertion here passing. | +| `count_sketch_with_heap_i128_counters_saturate_into_the_heap_instead_of_wrapping` | An `i128` `CountSketchWithHeap` holds mass past `i64::MAX` while its heap entry saturates at the ceiling. | For each of six instances, inserts `update_weighted(U64(0xC0FF_EE01), 1 << 40)` and requires `estimate` to equal `2^40` exactly and the heap entry to carry the same `i64`; then, on a fresh sketch, `update_weighted` of `i64::MAX as i128 * 4` and requires `estimate >= i64::MAX as f64`, `heap_count_from_estimate(estimate) == i64::MAX`, and the heap entry to read exactly `i64::MAX` rather than a wrapped negative; then merges a second sketch carrying the same overflow weight and requires the heap entry to stay at `i64::MAX`. Instances: `CountSketchWithHeap::, _>::new(3, 4096, 32)` on both paths, and `QuickMatrixI128` and `DefaultMatrixI128` from `Default` on both paths. | | `countmin_counter_widths_carry_the_mass_their_type_allows` | Each `CountMinSketch` counter width carries a count the next narrower one cannot. | On `3x64` `RegularPath` sketches keyed by `U64(0xFEED_FACE)`: `Vector2D` returns `i32::MAX - 1` after `update_weighted` of that weight; `Vector2D` returns `i32::MAX as i64 * 4`; `Vector2D` returns `i64::MAX as i128 * 4`. Then merges two `Vector2D` `3x64` sketches each holding `i64::MAX` and requires the result to be exactly `i64::MAX as i128 * 2`. | | `countsketch_counter_widths_carry_signed_mass_in_both_directions` | Each `CountSketch` counter width reaches the negative end of its range without wrapping. | On `3x64` `Vector2D<_>` `RegularPath` sketches keyed by `U64(0xDEAD_BEEF)`: `i32` cancels `+i32::MAX/2` against `-(i32::MAX/2)` to exactly `0.0`; `i64` reads `i32::MAX as i64 * 4` and, after a further `-2x` that weight, exactly its negation; `i128` reads `i64::MAX as i128 * 4` and, after a further `-2x`, exactly its negation. | | `countmin_holds_its_contract_across_the_depth_and_width_axis_on_both_paths` | Count-Min's one-sided and simultaneous bounds hold across the whole depth-by-width grid on both paths. | Over the sixteen geometries `d` in `{1, 2, 3, 9}` by `w` in `{64, 512, 4_096, 8_192}`, builds `CountMinSketch::, RegularPath>` and `CountMinSketch::, FastPath>` by `with_dimensions`, feeds `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, and for each path asserts no key underestimates and no key's excess exceeds `b*(N-f)/w` with `b = (D/1e-3)^(1/d)`, both with zero tolerance. The marginal `e*(N-f)/w` bound is not checked here, and the mean excess `countmin_axis_contract` returns is discarded. | @@ -2132,7 +2132,7 @@ Test file: [`tests/e2e/matrix_instances.rs`](../tests/e2e/matrix_instances.rs) | `countmin_merge_max_dominates_both_sides_on_disjoint_key_sets` | `merge_max` dominates both inputs and stays below the summing merge. | Splits `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)` by key parity into two `CountMinSketch::, RegularPath>::with_dimensions(4, 2_048)` sketches, records every key's pre-merge estimate on its own side, and after `merge_max` asserts each key's estimate is at or above both its pre-merge value and its side's true count. It then builds the same pair under `merge` and asserts the elementwise-max estimate never exceeds the elementwise-sum estimate, for the left key set. | | `countmin_merge_max_is_idempotent_and_absorbs_an_empty_sketch` | `merge_max` with an empty or identical sketch leaves every estimate untouched. | Fills a `CountMinSketch::, RegularPath>::with_dimensions(4, 2_048)` from `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, records every distinct key's estimate, then asserts that estimate is unchanged after `merge_max` with a fresh empty `4x2_048` sketch and after `merge_max` with a clone of itself. | | `countmin_precomputed_hash_entry_points_match_the_value_entry_points` | Count-Min's precomputed-hash and bulk entry points agree with the value entry points. | At `4x2_048` on `CountMinSketch::, FastPath>` over `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, builds one sketch by `update`, one by `update_with_hash` on `hash_for_matrix(4, 2_048, ..)` digests, and one by a single `update_batch_with_hashes` over the same digest vector, then asserts for every distinct key that all three `estimate` values are equal and that `estimate_with_hash` on the key's digest equals `estimate`. | -| `countmin_weighted_batch_entry_points_match_a_loop_of_single_inserts` | Count-Min's weighted batch entry points match a loop of `update_weighted`. | At `4x2_048` on `CountMinSketch::, FastPath>`, assigns each position of `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)` the weight `(i % 5) + 1` and builds three sketches: a loop of `update_weighted`, one `update_weighted_batch` over the value/weight pairs, and one `update_weighted_batch_with_hashes` over the `hash_for_matrix` digest/weight pairs, asserting all three estimates agree for every distinct key. Finally, `update_weighted_with_hash` of the digest of `U64(7)` with weight `9` must make `estimate(U64(7))` exactly `9`. | +| `countmin_weighted_batch_entry_points_match_a_loop_of_single_updates` | Count-Min's weighted batch entry points match a loop of `update_weighted`. | At `4x2_048` on `CountMinSketch::, FastPath>`, assigns each position of `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)` the weight `(i % 5) + 1` and builds three sketches: a loop of `update_weighted`, one `update_weighted_batch` over the value/weight pairs, and one `update_weighted_batch_with_hashes` over the `hash_for_matrix` digest/weight pairs, asserting all three estimates agree for every distinct key. Finally, `update_weighted_with_hash` of the digest of `U64(7)` with weight `9` must make `estimate(U64(7))` exactly `9`. | | `countsketch_precomputed_hash_entry_points_match_the_value_entry_points` | Count Sketch's precomputed-hash entry points agree with the value entry points. | At `5x2_048` on `CountSketch::, FastPath>` over `zipf_u64(40_000, 4_096, 1.1, 0x10BE_C700)`, builds one sketch by `update` and one by `update_with_hash` on `hash_for_matrix(5, 2_048, ..)` digests, and asserts for every distinct key that the two `estimate` values are equal and that `estimate_with_hash` equals `estimate`. Finally, `update_weighted_with_hash` of the digest of `U64(11)` with weight `6` must make `estimate(U64(11))` exactly `6.0`. | ### Numeric Type Coverage @@ -2203,13 +2203,13 @@ Feature: the whole file is `#![cfg(feature = "experimental")]`, so these rows ru | test_name | test_description | what_is_tested | | --- | --- | --- | -| `eh_univ_optimized_map_tier_exact_windows` | An interval inside the retained map tier is answered exactly, key by key. | Drives `ExponentialHistogramUnivMon::with_defaults(2, 100)` with 150 updates at `t` in `0..150`, key `U32(t % 10)` and weight `(t % 3) + 1`, then requires `query_interval(120, 149)` to return `EhUnivMonQueryResult::Exact` whose `total_count` equals the exact summed weight over `120..=149`, whose `counts.len()` equals the exact distinct-key count, and whose per-key entry equals the exact per-key weight for every key. | -| `eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it` | The map tier promotes into the sketch tier, whose interval answer keeps L1 exact and puts L2 and entropy in their measured bands. | Feeds 80,000 Zipf(1.1) draws over a 30,000-key domain (stream seed `4242`) into `ExponentialHistogramUnivMon::new(k=2, window=100_000_000, heap=32, rows=3, cols=512, layers=2)`, asserts at least two `sketch_buckets` exist, queries exactly `[sketch_buckets[0].min_time, sketch_buckets.last().max_time]` and requires an `EhUnivMonQueryResult::Sketch`; then `estimate_l1()` must equal the window's exact total weight by equality, `estimate_l2()` must lie in `[0.90, 1.10] ×` the exact L2 norm, and `estimate_entropy()` in `[0.90, 1.40] ×` the exact entropy in bits (bands measured on this configuration). | -| `eh_univ_optimized_sketch_tier_cardinality_is_documented_as_unrecoverable` | The sketch tier's cardinality estimate is a near-total underestimate, and that is pinned rather than hidden in a tolerance. | On the same stream and configuration (80,000 Zipf(1.1) draws over 30,000 keys, seed `4242`, `k=2`, `rows=3`, `cols=512`, `layers=2`), asserts the premise that `ceil(log2(distinct))` exceeds the `2` sampling layers, and that `estimate_cardinality()` over the promoted span is below `0.10 ×` the exact distinct count. Only that upper bound is asserted — the reported value itself is not pinned — and the assertion fails loudly if F0 recovery ever starts working. | -| `eh_univ_optimized_answers_a_mixed_map_and_sketch_interval` | A span crossing the tier boundary is answered as a Sketch and still carries the window's exact L1. | Feeds 60,000 updates `t` in `0..60_000` with key `U64(t % 20_000)` and weight `1` into `ExponentialHistogramUnivMon::new(k=2, window=1_000_000, heap=32, rows=3, cols=512, layers=2)`, asserts both `sketch_buckets` and `exact_buckets` are non-empty, and requires `query_interval(sketch_buckets[0].min_time, exact_buckets.last().max_time)` to return an `EhUnivMonQueryResult::Sketch` whose `estimate_l1()` equals the exact window weight. Only L1 is asserted on the mixed span. | -| `eh_univ_optimized_expires_buckets_past_the_window` | Expiry drops whole buckets past the window on both tiers, and the retained span and bucket count stay bounded. | Feeds 40,000 updates with key `U64(t % 8_000)` into `ExponentialHistogramUnivMon::new(k=8, window=5_000, heap=32, rows=3, cols=512, layers=2)`, then asserts `max_time() == 39_999`, that `min_time() <= max_time()`, that every retained `sketch_buckets` and `exact_buckets` entry has `max_time >= max_time - 5_000` (a bucket may still reach back before the cutoff with its `min_time`), that `cover(min_time, max_time)` is `true` and `cover(0, max_time)` is `false`, and that 30,000 further updates leave `bucket_count()` at most double its earlier value. The span-ordering check is trivially satisfiable, and the growth bound is a `2×` loosening rather than a fixed ceiling. | -| `eh_univ_optimized_reuses_pooled_sketches_without_leaking_state` | A sketch recycled through the pool carries no counters from a previous window. | Feeds 80,000 updates with key `U64(t % 20_000)` into `ExponentialHistogramUnivMon::new(k=8, window=20_000, heap=32, rows=3, cols=512, layers=2)` — long enough that buckets are created, promoted, merged and expired repeatedly — and asserts `query_interval(min_time(), max_time()).estimate_l1()` equals the exact weight of the retained window, which a pooled sketch holding stale counters would inflate. L1 is the only quantity asserted. | -| `eh_univ_optimized_map_tier_matches_exact_per_key_counts_on_a_skewed_stream` | The map tier's per-key counts on a weighted skewed stream are the exact reference the sketch tier is scored against. | Feeds 4,000 Zipf(1.2) draws over a 64-key domain (stream seed `9_001`) with weight `1 + (t % 3)` into `ExponentialHistogramUnivMon::with_defaults(4, 1_000_000)`, then requires `query_interval(0, 3_999)` to return `EhUnivMonQueryResult::Exact` whose `total_count` equals the exact total weight and whose `counts` entry matches the exact weight for every key in the truth. | +| `exponential_histogram_univmon_exact_tier_windows` | An interval inside the retained map tier is answered exactly, key by key. | Drives `ExponentialHistogramUnivMon::with_defaults(2, 100)` with 150 updates at `t` in `0..150`, key `U32(t % 10)` and weight `(t % 3) + 1`, then requires `query_interval(120, 149)` to return `EhUnivMonQueryResult::Exact` whose `total_count` equals the exact summed weight over `120..=149`, whose `counts.len()` equals the exact distinct-key count, and whose per-key entry equals the exact per-key weight for every key. | +| `exponential_histogram_univmon_promotes_into_the_sketch_tier_and_answers_from_it` | The map tier promotes into the sketch tier, whose interval answer keeps L1 exact and puts L2 and entropy in their measured bands. | Feeds 80,000 Zipf(1.1) draws over a 30,000-key domain (stream seed `4242`) into `ExponentialHistogramUnivMon::new(k=2, window=100_000_000, heap=32, rows=3, cols=512, layers=2)`, asserts at least two `sketch_buckets` exist, queries exactly `[sketch_buckets[0].min_time, sketch_buckets.last().max_time]` and requires an `EhUnivMonQueryResult::Sketch`; then `estimate_l1()` must equal the window's exact total weight by equality, `estimate_l2()` must lie in `[0.90, 1.10] ×` the exact L2 norm, and `estimate_entropy()` in `[0.90, 1.40] ×` the exact entropy in bits (bands measured on this configuration). | +| `exponential_histogram_univmon_sketch_tier_cardinality_is_documented_as_unrecoverable` | The sketch tier's cardinality estimate is a near-total underestimate, and that is pinned rather than hidden in a tolerance. | On the same stream and configuration (80,000 Zipf(1.1) draws over 30,000 keys, seed `4242`, `k=2`, `rows=3`, `cols=512`, `layers=2`), asserts the premise that `ceil(log2(distinct))` exceeds the `2` sampling layers, and that `estimate_cardinality()` over the promoted span is below `0.10 ×` the exact distinct count. Only that upper bound is asserted — the reported value itself is not pinned — and the assertion fails loudly if F0 recovery ever starts working. | +| `exponential_histogram_univmon_answers_a_mixed_exact_and_sketch_interval` | A span crossing the tier boundary is answered as a Sketch and still carries the window's exact L1. | Feeds 60,000 updates `t` in `0..60_000` with key `U64(t % 20_000)` and weight `1` into `ExponentialHistogramUnivMon::new(k=2, window=1_000_000, heap=32, rows=3, cols=512, layers=2)`, asserts both `sketch_buckets` and `exact_buckets` are non-empty, and requires `query_interval(sketch_buckets[0].min_time, exact_buckets.last().max_time)` to return an `EhUnivMonQueryResult::Sketch` whose `estimate_l1()` equals the exact window weight. Only L1 is asserted on the mixed span. | +| `exponential_histogram_univmon_expires_buckets_past_the_window` | Expiry drops whole buckets past the window on both tiers, and the retained span and bucket count stay bounded. | Feeds 40,000 updates with key `U64(t % 8_000)` into `ExponentialHistogramUnivMon::new(k=8, window=5_000, heap=32, rows=3, cols=512, layers=2)`, then asserts `max_time() == 39_999`, that `min_time() <= max_time()`, that every retained `sketch_buckets` and `exact_buckets` entry has `max_time >= max_time - 5_000` (a bucket may still reach back before the cutoff with its `min_time`), that `cover(min_time, max_time)` is `true` and `cover(0, max_time)` is `false`, and that 30,000 further updates leave `bucket_count()` at most double its earlier value. The span-ordering check is trivially satisfiable, and the growth bound is a `2×` loosening rather than a fixed ceiling. | +| `exponential_histogram_univmon_reuses_pooled_sketches_without_leaking_state` | A sketch recycled through the pool carries no counters from a previous window. | Feeds 80,000 updates with key `U64(t % 20_000)` into `ExponentialHistogramUnivMon::new(k=8, window=20_000, heap=32, rows=3, cols=512, layers=2)` — long enough that buckets are created, promoted, merged and expired repeatedly — and asserts `query_interval(min_time(), max_time()).estimate_l1()` equals the exact weight of the retained window, which a pooled sketch holding stale counters would inflate. L1 is the only quantity asserted. | +| `exponential_histogram_univmon_exact_tier_matches_exact_per_key_counts_on_a_skewed_stream` | The map tier's per-key counts on a weighted skewed stream are the exact reference the sketch tier is scored against. | Feeds 4,000 Zipf(1.2) draws over a 64-key domain (stream seed `9_001`) with weight `1 + (t % 3)` into `ExponentialHistogramUnivMon::with_defaults(4, 1_000_000)`, then requires `query_interval(0, 3_999)` to return `EhUnivMonQueryResult::Exact` whose `total_count` equals the exact total weight and whose `counts` entry matches the exact weight for every key in the truth. | ### Nitro Sampling @@ -2218,7 +2218,7 @@ Test file: [`tests/e2e/nitro.rs`](../tests/e2e/nitro.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | | `every_nitro_path_is_unbiased_inside_its_sampling_band_at_every_rate` | Every Nitro ingestion path lands inside the band its own rate earns and is unbiased across seeds. | Crosses the five paths (`CountMinSketch::nitro_update` and `CountSketch::nitro_update` over a `3x4096` `Vector2D`, `NitroBatch::update_batch`, `NitroBatch::update_batch_cached_step`, and `NitroBatch>::insert`) with rates `{1.0, 0.5, 0.3, 0.1, 0.07, 0.01}` and 12 trials at seeds `0x0117_0000 + 5_000*t`, feeding a single key `0xC0FF_EE01` `f = round(1000/rate)` times (`1_000`, `2_000`, `3_333`, `10_000`, `14_286`, `100_000`). Each trial's per-row statistics are checked against `SamplingConfidenceSpec::new(rate, 4.0)`, whose sigma is `sqrt(f((1-p)/p + p*r(1-r)))` for `r = frac(1/p)`; a trial passes only if all its rows do, and the trial outcomes go through `Tally::assert_independent_binomial`. Separately, the mean of the 12 first-row values must be within `4*sigma(f)/sqrt(12)` of `f`, which is what catches a `ceil(1/p)` weight that a single-trial band would pass. | -| `full_sampling_is_exact_and_the_query_reads_the_cells_the_insert_wrote` | At rate `1.0` nothing is approximate, and the estimator reads back exactly the cells the insert wrote. | For `F = 5_000` updates of key `0xC0FF_EE01` at rate `1.0` and seed `0x0117_0000`, asserts every row mass on all five paths equals exactly `5_000` — `0` would mean insert and query disagree on the hash domain, `F / rows` that a path assigned each record to one row instead of all — then that `CountMinSketch::nitro_estimate` returns exactly `5000.0` on a `3x4096` sketch, and that `NitroBatch::estimate_median` returns exactly `5000.0` after `update` and after `update_batch_cached_step`. | +| `full_sampling_is_exact_and_the_query_reads_the_cells_the_update_wrote` | At rate `1.0` nothing is approximate, and the estimator reads back exactly the cells the insert wrote. | For `F = 5_000` updates of key `0xC0FF_EE01` at rate `1.0` and seed `0x0117_0000`, asserts every row mass on all five paths equals exactly `5_000` — `0` would mean insert and query disagree on the hash domain, `F / rows` that a path assigned each record to one row instead of all — then that `CountMinSketch::nitro_estimate` returns exactly `5000.0` on a `3x4096` sketch, and that `NitroBatch::estimate_median` returns exactly `5000.0` after `update` and after `update_batch_cached_step`. | | `every_nitro_path_is_reproducible_from_its_seed` | Every path reproduces from its seed, and different seeds admit different subsets. | At rate `0.3` with `f = 3_333`, asserts each of the five paths returns the identical per-row statistics vector on two runs at seed `0x0117_0000`. The differing-seeds half asserts only that the 11 further seeds `0x0117_0000 + 5_000*t` for `t` in `1..12` produce more than one distinct first-row value; no per-seed value is pinned. | | `nitro_merge_lands_in_the_combined_band` | Merging two same-rate Nitro sketches sums their admitted mass, so a split key lands in the band for the combined work. | For `CountMinSketch::nitro_update` + `CountMinSketch::merge` and for `NitroBatch::update_batch` + `NitroBatch::merge`, at each of the rates `{1.0, 0.5, 0.3, 0.1, 0.07, 0.01}`, builds 6 disjoint seed pairs `(0x0117_0000 + 5_000*2p, ... + 5_000*(2p+1))`, feeds `f = round(1000/rate)` updates of key `0xC0FF_EE01` into each shard, merges, and scores the row-0 mass (or `estimate_median`) against `SamplingConfidenceSpec::new(rate, 4.0)` at the combined truth `2f`; the 6 outcomes go through `Tally::assert_independent_binomial`. | | `row_level_countmin_nitro_tracks_a_zipf_stream_within_the_combined_band` | Under collisions the row-level Count-Min estimate stays between its sampling floor and the sketch's own collision budget, for every key. | Feeds 200,000 Zipf(1.1) draws over a 512-key domain (stream seed `0x0117_2199`) into a `3x4096` `CountMinSketch, FastPath>` at rates `{1.0, 0.3, 0.1, 0.07}`, seed `0x0117_0000`, and for every distinct key requires `nitro_estimate` to lie in `[f - 4*sigma(f), f + CountMinSpec::new(3, 4096).simultaneous_bound(N, f, distinct, 1e-3) + 4*sigma(N)]`. Both halves are union-bounded, so `Tally::assert_none` tolerates zero violations. | @@ -2227,10 +2227,10 @@ Test file: [`tests/e2e/nitro.rs`](../tests/e2e/nitro.rs) | `a_serde_round_trip_after_every_update_reproduces_the_uninterrupted_run` | A sketch encoded and decoded after every update is cell-for-cell identical to one that was never interrupted, and its future agrees too. | At rates `0.3` and `0.07` (reciprocals that are not integers, so the stochastic-rounding stream is live), runs 600 updates of key `0xC0FF_EE01` into two `3x4096` `CountMinSketch, FastPath>` sketches at seed `0x0117_0000`, round-tripping one through `rmp_serde::to_vec_named` / `from_slice` after every single update, then asserts the two sketches' full 12,288-cell vectors are equal and their `nitro_estimate` values are equal; 200 further updates on both must again leave the cell vectors equal, which catches a decoded sketch that resumed from a reset rounding stream. | | `a_payload_written_before_the_rounding_field_existed_still_decodes` | A map-encoded payload that lacks `Nitro::rounding_state` decodes with the default stream and keeps sampling. | Serializes a hand-written mirror of the `CountMinSketch` / `Vector2D` / `Nitro` field layout without `rounding_state` (`rows=3`, `cols=4096`, `mask_bits = 12`, `mask = 4095`, `is_nitro_mode = true`, `sampling_rate = 0.3`, `to_skip = 2`, `delta = 3`, `idx = 9`, and a single counter `data[17] = 4`) with `rmp_serde::to_vec_named`, decodes it as `CountMinSketch, FastPath>`, asserts the decoded cell vector equals the 12,288-entry `data` exactly, then feeds 2,000 further `nitro_update` calls and requires the row-0 mass minus the pre-existing `4` to pass `SamplingConfidenceSpec::new(0.3, 4.0).check` against a truth of `2_000`. | | `a_context_snapshot_restores_the_rest_of_the_stream_exactly` | A `NitroContext` snapshot restored onto a fresh sketch replays the remainder of the stream exactly. | At rates `0.3` and `0.07`, feeds 500 updates of key `0xC0FF_EE01` into a `3x4096` `CountMinSketch, FastPath>` at seed `0x0117_3A98`, takes `nitro().context()`, records the cell vector, and runs 500 more updates; a fresh sketch at the same rate and seed gets `nitro_mut().restore_context(snapshot)` and the same 500 updates, and its cell vector must equal the continuous run's cell-by-cell difference over that stretch exactly. | -| `count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes` | At rate `1.0` the Nitro insert path writes the same Count-Min counters a plain insert writes, and the row median never falls below the row minimum. | Feeds 40,000 Zipf(1.1) draws over a 2,048-key domain (stream seed `0x4E17_0001`) into a plain `4x2048` `CountMinSketch, FastPath>` via `update` and into a twin with `enable_nitro(1.0)` via `nitro_update`, then asserts the two 8,192-counter vectors are equal, and per distinct key that `estimate` agrees between the two sketches and that `nitro_estimate` is at least the true count and at least the plain `estimate`. | -| `count_min_after_disable_nitro_leaves_the_sampled_insert_path_inert` | `disable_nitro` makes the sampled insert path a no-op without disabling the ordinary one. | Feeds the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) into a `4x2048` `CountMinSketch, FastPath>` with `enable_nitro(1.0)`, snapshots the 8,192 counters, calls `disable_nitro()` and replays the whole stream through `nitro_update` — the counters must be unchanged — then replays it through `update` and requires the counters to change. | +| `count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_update_writes` | At rate `1.0` the Nitro insert path writes the same Count-Min counters a plain insert writes, and the row median never falls below the row minimum. | Feeds 40,000 Zipf(1.1) draws over a 2,048-key domain (stream seed `0x4E17_0001`) into a plain `4x2048` `CountMinSketch, FastPath>` via `update` and into a twin with `enable_nitro(1.0)` via `nitro_update`, then asserts the two 8,192-counter vectors are equal, and per distinct key that `estimate` agrees between the two sketches and that `nitro_estimate` is at least the true count and at least the plain `estimate`. | +| `count_min_after_disable_nitro_leaves_the_sampled_update_path_inert` | `disable_nitro` makes the sampled insert path a no-op without disabling the ordinary one. | Feeds the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) into a `4x2048` `CountMinSketch, FastPath>` with `enable_nitro(1.0)`, snapshots the 8,192 counters, calls `disable_nitro()` and replays the whole stream through `nitro_update` — the counters must be unchanged — then replays it through `update` and requires the counters to change. | | `count_min_seeded_nitro_sampling_is_reproducible_and_admits_a_strict_subset` | Seeded sampling at rate `0.1` reproduces its admitted subset and touches strictly fewer counters than an unsampled pass. | Builds two `4x2048` `CountMinSketch, FastPath>` sketches at rate `0.1` and seed `0x5A11_0001`, feeds each the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) through `nitro_update`, and asserts their 8,192-counter vectors are equal; then counts non-zero counters against an unsampled `update`-only pass and requires the sampled count to be strictly smaller. Only the strict inequality is asserted, not how much smaller. | -| `count_sketch_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes` | At rate `1.0` the Count Sketch Nitro insert path writes the same signed counters a plain insert writes. | Feeds the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) into a plain `4x2048` `CountSketch, FastPath>` via `update` and into a twin with `enable_nitro(1.0)` via `nitro_update`, then asserts the two 8,192-counter signed vectors are equal and that `estimate` agrees between the two sketches for every distinct key. The exact per-key truth is read from `FreqTruth` but discarded, so no accuracy floor against the true count is asserted. | +| `count_sketch_at_full_nitro_sampling_writes_exactly_what_a_plain_update_writes` | At rate `1.0` the Count Sketch Nitro insert path writes the same signed counters a plain insert writes. | Feeds the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) into a plain `4x2048` `CountSketch, FastPath>` via `update` and into a twin with `enable_nitro(1.0)` via `nitro_update`, then asserts the two 8,192-counter signed vectors are equal and that `estimate` agrees between the two sketches for every distinct key. The exact per-key truth is read from `FreqTruth` but discarded, so no accuracy floor against the true count is asserted. | | `count_sketch_seeded_nitro_sampling_is_reproducible` | Two Count Sketches at the same rate and seed admit the same subset. | Builds two `4x2048` `CountSketch, FastPath>` sketches at rate `0.25` and seed `0x5A11_0002`, feeds each the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) through `nitro_update`, and asserts their 8,192-counter vectors are equal. Reproducibility is all that is asserted: there is no differing-seed check and no accuracy check. | ### ASAPv1 Envelope Round Trips @@ -2245,7 +2245,7 @@ Test file: [`tests/e2e/wire.rs`](../tests/e2e/wire.rs) | `a_hydra_envelope_round_trips_and_names_the_counter_it_carries` | A Hydra envelope's kind_id names the counter it carries, and the schema and cells survive. | Builds `Hydra::with_schema(4, 512, ["region", "user"], HydraCounter::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))`, updates each of the nine region/user pairs 40 times with `DataInput::Str("event")`, verifies the `kind_id_len` of `2` and bytes `0x07 0x01`, an unchanged `schema()`, and equal `query_key` answers under `HydraQuery::Frequency` for all nine fully specified keys. | | `a_kll_dynamic_envelope_round_trips_and_names_its_kind` | A KllDynamic envelope names kind_id `0x06 0x01` and re-encodes byte-identically. | Fills `KllDynamic::::with_k_and_seed(200, 0x5EED_9001)` from `uniform_u64(20_000, 1_000_000, 0xC0DE_0001)`, verifies the `kind_id_len` of `2` and bytes `0x06 0x01`, an unchanged `count()`, bit-identical `quantile` at q `0.0`, `0.1`, `0.5`, `0.9` and `1.0`, and that the decode re-serializes to the same bytes. | | `a_space_saving_envelope_round_trips_and_names_its_kind` | A Space-Saving envelope names kind_id `0x18 0x00` and preserves every estimate. | Fills `SpaceSaving::::with_capacity(256)` from a 20,000-draw Zipf stream (domain 1,024, exponent 1.1, seed `0xC1EA_0001`), verifies the `kind_id_len` of `2` and bytes `0x18 0x00`, and that `estimate` on the decode equals the source for all 1,024 domain keys. | -| `an_eh_sketch_list_envelope_round_trips_and_names_its_kind` | An EhBucketSketch envelope names kind_id `0x14 0x00` and keeps the payload sketch it wraps. | Fills an `EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512))` with the first 5,000 draws of the Zipf stream, verifies the `kind_id_len` of `2` and bytes `0x14 0x00`, an unchanged `sketch_type()`, and equal `query` answers for keys `0..512`. | +| `an_eh_bucket_sketch_envelope_round_trips_and_names_its_kind` | An EhBucketSketch envelope names kind_id `0x14 0x00` and keeps the payload sketch it wraps. | Fills an `EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512))` with the first 5,000 draws of the Zipf stream, verifies the `kind_id_len` of `2` and bytes `0x14 0x00`, an unchanged `sketch_type()`, and equal `query` answers for keys `0..512`. | | `an_elastic_envelope_round_trips_and_names_its_kind` | An Elastic envelope names kind_id `0x0b 0x00` and preserves every per-key answer. | Fills `ElasticSketch::::with_heavy_and_light(64, 3, 1_024)` with the keys `flow-{k}` of the 20,000-draw Zipf stream, verifies the `kind_id_len` of `2` and bytes `0x0b 0x00`, and that `query` on the decode equals the source for all 1,024 domain keys. | | `an_envelope_is_refused_by_every_decoder_but_its_own` | One sketch's envelope never decodes as another's, and malformed buffers are errors. | Verifies a `DdSketch::new(0.01)` envelope over `1..1_000` fails to decode as `Bloom` and as `SpaceSaving`, that a one-key `Bloom` envelope fails to decode as `DdSketch`, and that both an empty buffer and the first half of the DDSketch envelope are refused. | | `an_exponential_histogram_envelope_round_trips_and_names_its_kind` | An ExponentialHistogram envelope names kind_id `0x13 0x00` and answers interval merges identically. | Updates `ExponentialHistogram::new(8, 1_000_000, EhBucketSketch::CountMinSketch(CountMinSketch::, FastPath>::with_dimensions(3, 512)))` with the first 3,000 Zipf draws at timestamps `0..3_000`, verifies the `kind_id_len` of `2` and bytes `0x13 0x00`, unchanged `bucket_count()`, `min_time()` and `max_time()`, and that `query_interval` over the full time span answers identically for keys `0..256`. | @@ -2323,7 +2323,7 @@ Test file: [`tests/msgpack_compat.rs`](../tests/msgpack_compat.rs) | `count_sketch_round_trip` | A portable `CountSketch` survives a msgpack round trip with its geometry intact. | Updates `CountSketch::new(3, 64)` with `k1` `+1.0` and `k2` `+2.0`, then verifies the decode reports `rows` `3` and `cols` `64`; the signed counters and the estimates are not checked. | | `dd_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | | `dd_sketch_round_trip` | A portable `DdSketch` survives a msgpack round trip carrying its full count in the bucket array. | Updates `DdSketch::new(0.01)` with `1.0`, `10.0` and `100.0`, then verifies the decode's `total_count()` — recovered by summing the buckets, since the count is not on the wire — is `3`; `alpha`, the bucket offset and the quantiles are not checked. | -| `delta_result_round_trip` | A `SetAggregatorDelta` survives a msgpack round trip with both of its sets. | Serializes a `SetAggregatorDelta` whose `added` holds `a` and whose `removed` holds `b`, then verifies the decode's `added` contains `a` and its `removed` contains `b`. | +| `set_aggregator_delta_round_trip` | A `SetAggregatorDelta` survives a msgpack round trip with both of its sets. | Serializes a `SetAggregatorDelta` whose `added` holds `a` and whose `removed` holds `b`, then verifies the decode's `added` contains `a` and its `removed` contains `b`. | | `hll_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | | `hll_sketch_round_trip` | A portable `HllSketch` survives a msgpack round trip with its register count intact. | Updates `HllSketch::new(HllVariant::Regular, 8)` with `b"a"`, `b"b"` and `b"c"`, then verifies only that the decode's `registers` has the same length as the source's; the register values, the precision and the variant are not checked. | | `hydra_kll_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | @@ -2340,7 +2340,7 @@ Test file: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | | `core_ddsketch_passes_relative_quantile_conformance` | The core `DdSketch` holds its relative-value-error promise at three accuracy settings. | For alpha `0.001`, `0.01` and `0.05`, runs `relative_quantile_battery` on `DdSketch::new(alpha)` over 30,000 values — 20,000 Zipf draws (domain 4,096, exponent 1.1, seed `7_701`) offset by `1_000.0`, plus 10,000 draws (domain 512, exponent 1.3, seed `7_702`) mapped to `1e6 + v * 1e3` — requiring at q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` a relative error of at most `alpha` plus a few-ULP numerical slack against the exact `ceil(q*n)` order statistic. | -| `count_l2hh_passes_signed_frequency_and_merge_conformance` | `CountSketchWithL2Norm` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `CountSketchWithL2Norm::with_dimensions(5, 4_096)` over a 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`) through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0` on every key of true count at least `25`, through `turnstile_battery` on key `42` (`+500` then `-200` must estimate `300` within `1.0`, then `-300` must cancel to within `1e-6`), and through `merge_equivalence_battery`, which shards the stream by parity, merges, and requires agreement with a single-pass sketch within `25.0 + 0.06 * count`. | +| `count_sketch_with_l2_norm_passes_signed_frequency_and_merge_conformance` | `CountSketchWithL2Norm` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `CountSketchWithL2Norm::with_dimensions(5, 4_096)` over a 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`) through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0` on every key of true count at least `25`, through `turnstile_battery` on key `42` (`+500` then `-200` must estimate `300` within `1.0`, then `-300` must cancel to within `1e-6`), and through `merge_equivalence_battery`, which shards the stream by parity, merges, and requires agreement with a single-pass sketch within `25.0 + 0.06 * count`. | | `countmin_passes_frequency_and_merge_conformance` | `CountMinSketch` passes the one-sided frequency and merge-equivalence batteries. | Drives `CountMinSketch::, FastPath>::with_dimensions(4, 4096)` over the 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`) through `frequency_battery` one-sided at `rel_tol` `0.01` and `abs_tol` `4.0`, which requires the absent sentinel key `i64::MIN` to estimate within `±4.0` and each key of true count at least `25` to land in `[count, count * 1.01 + 4.0]`, then through `merge_equivalence_battery` at the same spec. | | `countsketch_passes_signed_frequency_conformance` | `CountSketch` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `CountSketch::, RegularPath>::with_dimensions(5, 4096)` over the 60,000-draw Zipf stream through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0`, `turnstile_battery` on key `42` via `update_weighted` (`+500 - 200` must read `300` within `1.0`, then cancel to within `1e-6`), and `merge_equivalence_battery` at the same spec. | | `ddsketch_passes_relative_quantile_conformance` | The portable `DdSketch` holds its relative-value-error promise under its own rank convention. | Runs `relative_quantile_battery` on `PortableDds::new(0.01)` over the positive samples of `normal_f64(40_000, 500.0, 80.0, 7001)`, requiring at q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` a relative error of at most `0.01` plus a few-ULP numerical slack against the exact `floor(q*(n-1))` order statistic. | @@ -2368,7 +2368,7 @@ Test file: [`tests/sketches_go_parity_probe.rs`](../tests/sketches_go_parity_pro | test_name | test_description | what_is_tested | | --- | --- | --- | -| `sketches_count_fastpath_matches_go_count_sketch_envelope` | A `sketches::CountSketch` fast-path matrix encodes to the checked-in Count-Sketch envelope bytes. | Fills `CountSketch, FastPath, DefaultXxHasher>` at `3x512` with 25 `update_weighted` calls over the five keys `k-a`..`k-e`, packs the row-major cells and per-row L2 sums into a `CountSketchState` with `counter_type` `INT64` inside a `SketchEnvelope { format_version: 1 }` with no producer or hash_spec, and verifies the prost encoding equals the 1,577-byte fixture `tests/cs_envelope_golden.hex`. | +| `sketches_count_sketch_fastpath_matches_go_count_sketch_envelope` | A `sketches::CountSketch` fast-path matrix encodes to the checked-in Count-Sketch envelope bytes. | Fills `CountSketch, FastPath, DefaultXxHasher>` at `3x512` with 25 `update_weighted` calls over the five keys `k-a`..`k-e`, packs the row-major cells and per-row L2 sums into a `CountSketchState` with `counter_type` `INT64` inside a `SketchEnvelope { format_version: 1 }` with no producer or hash_spec, and verifies the prost encoding equals the 1,577-byte fixture `tests/cs_envelope_golden.hex`. | | `sketches_countmin_fastpath_matches_go_count_min_envelope` | A `sketches::CountMinSketch` fast-path matrix encodes to the checked-in Count-Min envelope bytes. | Fills `CountMinSketch, FastPath, DefaultXxHasher>` at `4x2048` with 50 `update_weighted` calls over the ten keys `flow-0`..`flow-9`, packs the cells cast to `i64` plus per-row L1 and L2 sums into a `CountMinState` with `counter_type` `INT64`, and verifies the prost encoding matches the 8,275-byte fixture `src/sketches/testdata/cms_envelope_golden.hex` in length and then byte for byte. | | `sketches_ddsketch_matches_go_envelope` | A `sketches::DdSketch` encodes to the DDSketch envelope hex constant held in the test file. | Fills `DdSketch::new(0.01)` with `1..=50` as `f64`, encodes a `DdSketchState` carrying `positive_store_counts`, `positive_store_offset` and an alpha round-tripped through gamma (`gamma = (1 + a) / (1 - a)`, `alpha_wire = (gamma - 1) / (gamma + 1)`), and verifies the envelope matches the 403-byte inline hex constant in length and then byte for byte; the comment on that constant records it as this crate's own output rather than a fixture from another implementation, so no cross-language parity is pinned. | | `sketches_hll_classic_matches_go_envelope` | An Ertl P14 register set encodes to the checked-in HyperLogLog envelope bytes. | Feeds `HyperLogLogImpl` the `hash64_seeded(DEFAULT_CANONICAL_SEED_INDEX, ...)` digests of the IEEE-754 little-endian bytes of `1.0..=50.0` through `update_with_hash`, encodes a `HyperLogLogState` with `variant` `2`, `precision` `14`, the dense registers, zeroed HIP scalars and no sparse form, and verifies the envelope matches the 16,398-byte fixture `src/sketches/testdata/hll_envelope_golden.hex` in length and then byte for byte; despite the name, the variant under test is `Ertl`, not `Classic`. | @@ -2401,4 +2401,4 @@ Test file: [`tests/bug_verification.rs`](../tests/bug_verification.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | | `portable_ddsketch_respects_alpha_at_bucket_edges` | Both DDSketch implementations hold the advertised relative accuracy on a value sitting at a bucket's lower edge. | Sets `alpha = 0.05`, `gamma = (1+alpha)/(1-alpha)`, and feeds the bucket-edge value `gamma^20 * (1 + 1e-6)` 10,000 times into both `message_pack_format::portable::ddsketch::DdSketch::new(0.05)` and `DdSketch::new(0.05)`, then asserts the portable `quantile(0.5)` and the core `get_value_at_quantile(0.5)` are each within `alpha * (1 + 1e-6)` relative error of that value — a representative reported as the bucket log-midpoint `gamma^(k+0.5)` gives about `5.13%` here and fails. Exact agreement between the two is not asserted, because the core sketch clamps representatives to its observed min/max and the portable wire format does not carry them. | -| `countl2hh_f2_survives_beyond_i64_max` | `CountSketchWithL2Norm`'s hot-path L2 accumulation stays exact inside `i64` and saturates beyond it. | Builds `CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 7)` and calls `update_weighted(&DataInput::U32(1), 3_000_000_000)`, asserting `l2_squared()` matches the exact `9e18` to a relative `1e-12`; a second key `U32(2)` at the same count puts the true F2 at `1.8e19`, past `i64::MAX`, and `l2_squared()` must then equal `i64::MAX` (`9.223e18`) to a relative `1e-12` rather than wrapping to `0` or panicking in debug. | +| `count_sketch_with_l2_norm_f2_survives_beyond_i64_max` | `CountSketchWithL2Norm`'s hot-path L2 accumulation stays exact inside `i64` and saturates beyond it. | Builds `CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 7)` and calls `update_weighted(&DataInput::U32(1), 3_000_000_000)`, asserting `l2_squared()` matches the exact `9e18` to a relative `1e-12`; a second key `U32(2)` at the same count puts the true F2 at `1.8e19`, past `i64::MAX`, and `l2_squared()` must then equal `i64::MAX` (`9.223e18`) to a relative `1e-12` rather than wrapping to `0` or panicking in debug. | diff --git a/src/common/structures/vector3d.rs b/src/common/structures/vector3d.rs index f8dae3d0..87f67f53 100644 --- a/src/common/structures/vector3d.rs +++ b/src/common/structures/vector3d.rs @@ -328,7 +328,7 @@ mod tests { #[test] #[should_panic(expected = "dimensions must be non-zero")] - fn init_rejects_zero_depth() { + fn unfilled_rejects_zero_bucket_len() { let _: Vector3D = Vector3D::unfilled(2, 4, 0); } @@ -431,7 +431,7 @@ mod tests { #[test] #[should_panic(expected = "overflows usize")] - fn init_rejects_dimensions_whose_product_overflows() { + fn unfilled_rejects_dimensions_whose_product_overflows() { // Each dimension is legal on its own; the product is not. Without the // check `Vec::with_capacity` receives the wrapped product and the // container reports a shape its storage cannot hold. diff --git a/src/sketch_framework/eh.rs b/src/sketch_framework/eh.rs index 33a84a8c..a1210c83 100644 --- a/src/sketch_framework/eh.rs +++ b/src/sketch_framework/eh.rs @@ -346,7 +346,7 @@ mod tests { } #[test] - fn l2_merge_invariant_sum_l22() { + fn l2_merge_invariant_sum_l2_squared() { let mut eh = ExponentialHistogram::new( 1, 100, diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index 55e0852e..d3a26d3a 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -292,7 +292,7 @@ mod tests { use super::*; #[test] - fn insert_routes_to_countl2hh_and_univmon() { + fn update_routes_to_count_sketch_with_l2_norm_and_univmon() { let key = DataInput::I64(7); let mut count_l2hh = @@ -317,7 +317,7 @@ mod tests { } #[test] - fn count_sketch_insert_and_query_round_trip() { + fn count_sketch_update_and_query_round_trip() { let mut cs = EhBucketSketch::CountSketch(CountSketch::, FastPath>::default()); let key = DataInput::I64(11); cs.update(&key); @@ -326,7 +326,7 @@ mod tests { } #[test] - fn ddsketch_insert_and_quantile_query_round_trip() { + fn ddsketch_update_and_quantile_query_round_trip() { let mut dd = EhBucketSketch::DdSketch(DdSketch::new(0.01)); dd.update(&DataInput::F64(10.0)); dd.update(&DataInput::F64(20.0)); diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index d19e5aff..eeafda5d 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -373,7 +373,7 @@ pub(crate) mod tests { } #[test] - fn eh_sketch_list_round_trip_serialization() { + fn eh_bucket_sketch_round_trip_serialization() { let mut sketch = EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(3, 8), ); @@ -394,7 +394,7 @@ pub(crate) mod tests { /// Every variant this build carries round-trips, keeps its type, and /// re-serializes byte-identically. #[test] - fn eh_sketch_list_every_variant_round_trips() { + fn eh_bucket_sketch_every_variant_round_trips() { for sketch in populated_variants() { let encoded = sketch .to_asapv1() @@ -415,7 +415,7 @@ pub(crate) mod tests { /// The ten nested kind_ids and their registry names are the same in every /// build, and every variant emits the id the table gives it. #[test] - fn eh_sketch_list_kind_ids_are_build_independent() { + fn eh_bucket_sketch_kind_ids_are_build_independent() { let table: [(&[u8], &str); 10] = [ (kind::COUNT_MIN, "CountMinSketch"), (kind::COCO, "Coco"), @@ -447,7 +447,7 @@ pub(crate) mod tests { /// The experimental kind_id is rejected without the feature, with an error /// naming the variant. Crafted bytes, so the test runs in both builds. #[test] - fn eh_sketch_list_rejects_an_experimental_kind_id() { + fn eh_bucket_sketch_rejects_an_experimental_kind_id() { let triple = SketchState { kind_id: kind::UNIFORM_SAMPLING.to_vec(), descriptor: Vec::new(), @@ -465,7 +465,7 @@ pub(crate) mod tests { } #[test] - fn eh_sketch_list_rejects_an_unknown_kind_id() { + fn eh_bucket_sketch_rejects_an_unknown_kind_id() { let sketch = EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(3, 8), ); @@ -479,7 +479,7 @@ pub(crate) mod tests { /// A nested kind_id longer than the envelope's one-byte length field is /// rejected as unknown, before any block is assembled. #[test] - fn eh_sketch_list_rejects_an_over_long_kind_id() { + fn eh_bucket_sketch_rejects_an_over_long_kind_id() { for len in [256usize, 4096] { let triple = SketchState { kind_id: vec![0x14; len], @@ -496,7 +496,7 @@ pub(crate) mod tests { /// The nested ids are pinned to one algorithm each: HLL is Ertl-MLE and KLL /// is compact, so their siblings' ids are unknown here. #[test] - fn eh_sketch_list_rejects_sibling_algorithm_kind_ids() { + fn eh_bucket_sketch_rejects_sibling_algorithm_kind_ids() { let hll = EhBucketSketch::Hll(HyperLogLog::::default()); for sibling in [[0x01u8, 0x01], [0x01, 0x03]] { let triple = relabelled(&hll, &sibling); @@ -513,7 +513,7 @@ pub(crate) mod tests { /// A kind_id that does not match the blocks it carries is rejected by the /// variant's own decoder. #[test] - fn eh_sketch_list_rejects_a_mismatched_kind_id_and_descriptor() { + fn eh_bucket_sketch_rejects_a_mismatched_kind_id_and_descriptor() { let sketch = EhBucketSketch::Hll(HyperLogLog::::default()); let triple = relabelled(&sketch, kind::COUNT_MIN); assert!(EhBucketSketch::from_asapv1(&envelope_for(&triple)).is_err()); @@ -522,7 +522,7 @@ pub(crate) mod tests { /// A Count-Min envelope and an ExponentialHistogram envelope are not /// EhBucketSketch envelopes. #[test] - fn eh_sketch_list_rejects_foreign_kind_ids() { + fn eh_bucket_sketch_rejects_foreign_kind_ids() { let cms = CountMinSketch::, FastPath>::with_dimensions(3, 8); let cms_bytes = cms.to_asapv1().expect("serialize CMS"); assert!(EhBucketSketch::from_asapv1(&cms_bytes).is_err()); @@ -540,7 +540,7 @@ pub(crate) mod tests { /// Fail closed on an unexpected metadata key. #[test] - fn eh_sketch_list_metadata_rejects_unknown_keys() { + fn eh_bucket_sketch_metadata_rejects_unknown_keys() { #[derive(Serialize)] struct WithExtra { metadata_version: u8, @@ -556,7 +556,7 @@ pub(crate) mod tests { /// `metadata_version` is required: a map missing it does not decode. #[test] - fn eh_sketch_list_metadata_rejects_a_missing_key() { + fn eh_bucket_sketch_metadata_rejects_a_missing_key() { #[derive(Serialize)] struct Empty {} let bytes = rmp_serde::to_vec_named(&Empty {}).unwrap(); @@ -565,7 +565,7 @@ pub(crate) mod tests { /// Crafted blocks fail closed with an error, never a panic. #[test] - fn eh_sketch_list_rejects_crafted_blocks() { + fn eh_bucket_sketch_rejects_crafted_blocks() { let sketch = EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(3, 8), ); @@ -589,14 +589,14 @@ pub(crate) mod tests { /// A descriptor naming a custom hash profile is rejected: the variant's /// decoder pins the profile of the type it rebuilds. #[test] - fn eh_sketch_list_rejects_a_custom_hash_profile_descriptor() { + fn eh_bucket_sketch_rejects_a_custom_hash_profile_descriptor() { let triple = alt_profile_triple(); assert!(EhBucketSketch::from_asapv1(&envelope_for(&triple)).is_err()); } /// A decoded union answers a query the way the original did. #[test] - fn eh_sketch_list_query_agrees_after_decode() { + fn eh_bucket_sketch_query_agrees_after_decode() { for sketch in populated_variants() { let key = sample_input(sketch.sketch_type()); let encoded = sketch.to_asapv1().expect("serialize"); diff --git a/src/sketch_framework/hashlayer.rs b/src/sketch_framework/hashlayer.rs index fc39a079..8972db15 100644 --- a/src/sketch_framework/hashlayer.rs +++ b/src/sketch_framework/hashlayer.rs @@ -432,7 +432,7 @@ mod tests { } #[test] - fn test_insert_and_estimate() { + fn test_update_and_estimate() { let data = sample_zipf_u64(ZIPF_DOMAIN, ZIPF_EXPONENT, SAMPLE_SIZE, SEED); let baseline = create_baseline(&data); @@ -464,7 +464,7 @@ mod tests { } #[test] - fn test_insert_at() { + fn test_update_at_indices() { let data = sample_zipf_u64(ZIPF_DOMAIN, ZIPF_EXPONENT, SAMPLE_SIZE, SEED); let baseline = create_baseline(&data); @@ -485,7 +485,7 @@ mod tests { } #[test] - fn test_insert_with_hash_matches_insert() { + fn test_update_with_hash_matches_update() { let data = sample_zipf_u64(ZIPF_DOMAIN, ZIPF_EXPONENT, SAMPLE_SIZE, SEED); let mut layer_a = default_layer(); diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index 3d6a67ef..ec6fab2c 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -1523,7 +1523,7 @@ impl OctoWorker for CountMinSketchWithHeapOctoWorker { // keeps no key storage, so a residual cell cannot be attributed back to a // key. A key that has never promoted is absent from the aggregator's heap // entirely and estimates zero. See - // `a_key_below_the_threshold_never_reaches_the_topk_aggregator`. + // `a_key_below_the_threshold_never_reaches_the_count_min_sketch_with_heap_aggregator`. } /// Count-Min parent that also maintains the only heavy-hitter heap in the @@ -3843,7 +3843,7 @@ mod runtime_tests { } #[test] - fn the_topk_aggregator_rebuilds_the_heavy_hitter_heap_from_worker_keys() { + fn the_count_min_sketch_with_heap_aggregator_rebuilds_the_heavy_hitter_heap_from_worker_keys() { let (rows, cols, top_k) = (4usize, 2048usize, 8usize); let mut inputs: Vec> = Vec::new(); // Four unmistakable heavy hitters over a wide light tail. @@ -3906,7 +3906,7 @@ mod runtime_tests { } #[test] - fn a_key_below_the_threshold_never_reaches_the_topk_aggregator() { + fn a_key_below_the_threshold_never_reaches_the_count_min_sketch_with_heap_aggregator() { // The keyed workers keep no key storage and so cannot flush, and the // aggregator only learns a key exists from a delta. A key promotes the // first time an increment it caused takes some row to tau on its @@ -3970,7 +3970,7 @@ mod runtime_tests { } #[test] - fn the_count_topk_aggregator_also_tracks_heavy_hitters() { + fn the_count_sketch_with_heap_aggregator_also_tracks_heavy_hitters() { let (rows, cols, top_k) = (5usize, 2048usize, 8usize); let mut inputs: Vec> = Vec::new(); for i in 0..3u64 { @@ -4327,7 +4327,7 @@ mod runtime_tests { #[test] #[should_panic(expected = "cannot update after runtime has been closed")] - fn octo_runtime_insert_after_close_panics() { + fn octo_runtime_update_after_close_panics() { let mut runtime = OctoRuntime::new(&config(2), HllOctoPlan::new(), HllOctoAggregator::new); runtime.close(); runtime.update(DataInput::U64(1)); @@ -4571,7 +4571,7 @@ mod univmon_tests { } #[test] - fn a_worker_picks_the_same_layers_as_a_single_threaded_insert() { + fn a_worker_picks_the_same_layers_as_a_single_threaded_update() { let reference = UnivMon::new(HEAP, ROWS, COLS, LAYERS); for key in 0..5_000u64 { let input = DataInput::U64(key); diff --git a/src/sketch_framework/tumbling.rs b/src/sketch_framework/tumbling.rs index 92b75989..3f8deb75 100644 --- a/src/sketch_framework/tumbling.rs +++ b/src/sketch_framework/tumbling.rs @@ -483,7 +483,7 @@ mod tests { // -- Clear method tests -------------------------------------------------- #[test] - fn fold_cms_clear_resets_to_empty() { + fn count_min_sketch_fold_clear_resets_to_empty() { let mut sk: CountMinSketchFold = CountMinSketchFold::new(3, 1024, 3, 10); for i in 0..50u64 { sk.update_weighted(&DataInput::U64(i), 1); @@ -503,7 +503,7 @@ mod tests { } #[test] - fn fold_cs_clear_resets_to_empty() { + fn count_sketch_fold_clear_resets_to_empty() { let mut sk: CountSketchFold = CountSketchFold::new(3, 1024, 3, 10); for i in 0..50u64 { sk.update_weighted(&DataInput::U64(i), 1); @@ -749,7 +749,7 @@ mod tests { // -- CountMinSketchFold hierarchical merge via tumbling windows ---------------------- #[test] - fn fold_cms_tumbling_hierarchical_merge() { + fn count_min_sketch_fold_tumbling_hierarchical_merge() { let rows = 3; let full_cols = 4096; let fold_level = 4; @@ -885,7 +885,7 @@ mod tests { // -- CountSketchFold tumbling test ------------------------------------------------ #[test] - fn fold_cs_tumbling_basic() { + fn count_sketch_fold_tumbling_basic() { let config = CountSketchFoldConfig { rows: 3, full_cols: 1024, @@ -903,7 +903,7 @@ mod tests { } #[test] - fn fold_cs_tumbling_hierarchical_merge() { + fn count_sketch_fold_tumbling_hierarchical_merge() { let config = CountSketchFoldConfig { rows: 5, full_cols: 4096, @@ -938,7 +938,7 @@ mod tests { // -- Accuracy & correctness tests ---------------------------------------- #[test] - fn fold_cms_tumbling_accuracy_zipf() { + fn count_min_sketch_fold_tumbling_accuracy_zipf() { let rows = 3; let full_cols = 4096; let fold_level = 4; @@ -996,7 +996,7 @@ mod tests { let pct = within as f64 / truth.len() as f64 * 100.0; let mean_abs_error = total_abs_error as f64 / truth.len() as f64; eprintln!( - "[fold_cms_tumbling_accuracy_zipf] mean_abs_error={mean_abs_error:.2}, \ + "[count_min_sketch_fold_tumbling_accuracy_zipf] mean_abs_error={mean_abs_error:.2}, \ max_abs_error={max_abs_error}, pct_within_bound={pct:.1}% \ (required>{required_pct:.1}%), error_bound={error_bound:.2}" ); @@ -1008,7 +1008,7 @@ mod tests { } #[test] - fn fold_cms_hierarchical_vs_flat_merge() { + fn count_min_sketch_fold_hierarchical_vs_flat_merge() { let rows = 3; let full_cols = 4096; // fold_level = 3 so hierarchical merge with 8 windows (log2(8)=3 rounds) @@ -1079,7 +1079,7 @@ mod tests { } #[test] - fn fold_cs_tumbling_accuracy_zipf() { + fn count_sketch_fold_tumbling_accuracy_zipf() { let rows = 5; let full_cols = 4096; let fold_level = 4; @@ -1140,7 +1140,7 @@ mod tests { let pct = within as f64 / truth.len() as f64 * 100.0; let mean_abs_error = total_abs_error as f64 / truth.len() as f64; eprintln!( - "[fold_cs_tumbling_accuracy_zipf] mean_abs_error={mean_abs_error:.2}, \ + "[count_sketch_fold_tumbling_accuracy_zipf] mean_abs_error={mean_abs_error:.2}, \ max_abs_error={max_abs_error}, pct_within_bound={pct:.1}% \ (required>{required_pct:.1}%), error_bound={error_bound:.2}" ); @@ -1532,7 +1532,7 @@ mod tests { } #[test] - fn fold_cms_tumbling_heap_correctness() { + fn count_min_sketch_fold_tumbling_heap_correctness() { let rows = 3; let full_cols = 4096; let fold_level = 4; @@ -1626,7 +1626,7 @@ mod tests { // Require at least 80% of the true top-k to be present in the heap. let recall_pct = found_in_heap as f64 / top_k as f64 * 100.0; eprintln!( - "[fold_cms_tumbling_heap_correctness] heap_size={}, \ + "[count_min_sketch_fold_tumbling_heap_correctness] heap_size={}, \ true_top_k_recall={found_in_heap}/{top_k} ({recall_pct:.0}%)", heap_entries.len() ); @@ -1639,7 +1639,7 @@ mod tests { // -- Gap 1: CountSketchFold query_recent accuracy --------------------------------- #[test] - fn fold_cs_tumbling_query_recent_accuracy() { + fn count_sketch_fold_tumbling_query_recent_accuracy() { let rows = 5; let full_cols = 4096; let fold_level = 4; @@ -1717,7 +1717,7 @@ mod tests { } let pct = within as f64 / recent_truth.len() as f64 * 100.0; eprintln!( - "[fold_cs_tumbling_query_recent_accuracy] pct_within_bound={pct:.1}%, \ + "[count_sketch_fold_tumbling_query_recent_accuracy] pct_within_bound={pct:.1}%, \ error_bound={error_bound:.2}" ); assert!( @@ -1811,7 +1811,7 @@ mod tests { // -- Gap 3: CountSketchFold heap correctness -------------------------------------- #[test] - fn fold_cs_tumbling_heap_correctness() { + fn count_sketch_fold_tumbling_heap_correctness() { let rows = 5; let full_cols = 4096; let fold_level = 4; @@ -1896,7 +1896,7 @@ mod tests { let recall_pct = found_in_heap as f64 / top_k as f64 * 100.0; eprintln!( - "[fold_cs_tumbling_heap_correctness] heap_size={}, \ + "[count_sketch_fold_tumbling_heap_correctness] heap_size={}, \ true_top_k_recall={found_in_heap}/{top_k} ({recall_pct:.0}%)", heap_entries.len() ); @@ -1909,7 +1909,7 @@ mod tests { // -- Gap 4: Tumbling vs monolithic degradation --------------------------- #[test] - fn fold_cms_tumbling_vs_monolithic() { + fn count_min_sketch_fold_tumbling_vs_monolithic() { let rows = 3; let full_cols = 4096; let fold_level = 4; @@ -1958,7 +1958,7 @@ mod tests { let tumbling_mae = tumbling_total_err as f64 / truth.len() as f64; let mono_mae = mono_total_err as f64 / truth.len() as f64; eprintln!( - "[fold_cms_tumbling_vs_monolithic] tumbling_mae={tumbling_mae:.2}, \ + "[count_min_sketch_fold_tumbling_vs_monolithic] tumbling_mae={tumbling_mae:.2}, \ mono_mae={mono_mae:.2}, ratio={:.2}", tumbling_mae / mono_mae.max(1.0) ); @@ -1969,7 +1969,7 @@ mod tests { } #[test] - fn fold_cs_tumbling_vs_monolithic() { + fn count_sketch_fold_tumbling_vs_monolithic() { let rows = 5; let full_cols = 4096; let fold_level = 4; @@ -2017,7 +2017,7 @@ mod tests { let tumbling_mae = tumbling_total_err as f64 / truth.len() as f64; let mono_mae = mono_total_err as f64 / truth.len() as f64; eprintln!( - "[fold_cs_tumbling_vs_monolithic] tumbling_mae={tumbling_mae:.2}, \ + "[count_sketch_fold_tumbling_vs_monolithic] tumbling_mae={tumbling_mae:.2}, \ mono_mae={mono_mae:.2}, ratio={:.2}", tumbling_mae / mono_mae.max(1.0) ); diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 3653ef45..dfe51f8e 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -691,13 +691,13 @@ mod tests { /// Layer `UNIVMON_BOTTOM_LAYER_SEED_INDEX` would hash at the finder's own seed. #[test] #[should_panic(expected = "UNIVMON_MAX_LAYERS")] - fn init_univmon_rejects_a_layer_on_the_finder_seed() { + fn new_rejects_a_layer_on_the_bottom_layer_seed_index() { UnivMon::new(4, 2, 16, UNIVMON_BOTTOM_LAYER_SEED_INDEX + 1); } #[test] #[should_panic(expected = "hash bits")] - fn init_univmon_rejects_column_bits_that_reach_the_sign_bits() { + fn new_rejects_column_bits_that_reach_the_sign_bits() { UnivMon::new(4, 11, 2048, 2); } @@ -769,7 +769,7 @@ mod tests { } #[test] - fn update_populates_bucket_size_and_heavy_hitters() { + fn update_populates_total_weight_and_heavy_hitters() { // processing a single hot key should record its weight in the heavy hitter layers let mut um = UnivMon::new(16, 3, 32, 4); let key = "alpha"; @@ -988,7 +988,7 @@ mod tests { } #[test] - fn univmon_bucket_size_tracked_correctly() { + fn univmon_total_weight_tracked_correctly() { // Verify that total_weight is correctly tracked with seed configuration let mut um = UnivMon::new(20, 3, 1024, 6); diff --git a/src/sketch_framework/univmon_two_tier.rs b/src/sketch_framework/univmon_two_tier.rs index 37fe6f47..aff62c2c 100644 --- a/src/sketch_framework/univmon_two_tier.rs +++ b/src/sketch_framework/univmon_two_tier.rs @@ -428,7 +428,7 @@ mod tests { // ======================================================================= #[test] - fn pyramid_basic_insert_and_query() { + fn univmon_two_tier_basic_update_and_query() { let mut um = UnivMonTwoTier::with_defaults(); let cases: Vec<(&str, i64)> = vec![("hello", 10), ("world", 20), ("hello", 5), ("foo", 30)]; @@ -447,7 +447,7 @@ mod tests { } #[test] - fn pyramid_fast_insert_matches_standard() { + fn univmon_two_tier_update_weighted_terminal_matches_standard() { // Both insert paths should produce identical sketches. // Retain the complete 100-key support so this tests construction // equivalence rather than independent candidate-sampling variance. @@ -481,7 +481,7 @@ mod tests { } #[test] - fn pyramid_two_tier_dimensions() { + fn univmon_two_tier_dimensions() { // Verify elephant layers are larger than mouse layers. let um = UnivMonTwoTier::new(32, 4, 5, 2048, 3, 256, 8); @@ -492,7 +492,7 @@ mod tests { } #[test] - fn pyramid_free_resets_state() { + fn univmon_two_tier_clear_resets_state() { let mut um = UnivMonTwoTier::with_defaults(); for i in 0..100i64 { um.update_weighted(&DataInput::I64(i), 10); @@ -505,7 +505,7 @@ mod tests { } #[test] - fn pyramid_merge_combines_data() { + fn univmon_two_tier_merge_combines_data() { let mut one_pass = UnivMonTwoTier::new(128, 8, 3, 2048, 3, 512, 16); let mut left = UnivMonTwoTier::new(128, 8, 3, 2048, 3, 512, 16); let mut right = UnivMonTwoTier::new(128, 8, 3, 2048, 3, 512, 16); @@ -567,7 +567,7 @@ mod tests { } #[test] - fn pyramid_accuracy_zipf() { + fn univmon_two_tier_accuracy_zipf() { use std::collections::HashMap; let mut um = UnivMonTwoTier::new(64, 8, 5, 2048, 3, 512, 16); @@ -609,7 +609,7 @@ mod tests { } #[test] - fn pyramid_fast_insert_accuracy() { + fn univmon_two_tier_update_weighted_terminal_accuracy() { use std::collections::HashMap; let mut um = UnivMonTwoTier::new(64, 8, 5, 2048, 3, 512, 16); @@ -645,7 +645,7 @@ mod tests { } #[test] - fn pyramid_memory_savings_vs_uniform() { + fn univmon_two_tier_memory_savings_vs_uniform() { // Pyramid with 16 layers should use less memory than a uniform // UnivMon with the same elephant dimensions at all 16 layers. // We verify this by comparing total sketch column counts. diff --git a/src/sketch_framework/univmon_two_tier/wire.rs b/src/sketch_framework/univmon_two_tier/wire.rs index 986a5b42..bb777fb8 100644 --- a/src/sketch_framework/univmon_two_tier/wire.rs +++ b/src/sketch_framework/univmon_two_tier/wire.rs @@ -326,7 +326,7 @@ mod tests { } #[test] - fn pyramid_round_trip_serialization() { + fn univmon_two_tier_round_trip_serialization() { let pyramid = populated(); let encoded = pyramid.to_asapv1().expect("serialize"); assert!(encoded.starts_with(b"ASAPv1")); @@ -368,7 +368,7 @@ mod tests { /// The two tiers are derived from the layer's position: the elephant /// layers keep their dimensions and the mouse layers keep theirs. #[test] - fn pyramid_two_tier_geometry_survives() { + fn univmon_two_tier_geometry_survives() { let pyramid = populated(); let encoded = pyramid.to_asapv1().expect("serialize"); let decoded = UnivMonTwoTier::from_asapv1(&encoded).expect("decode"); @@ -388,7 +388,7 @@ mod tests { /// Layers hold different numbers of entries: each layer's heap contents /// survive the round trip. #[test] - fn pyramid_layers_with_different_heap_loads_round_trip() { + fn univmon_two_tier_layers_with_different_heap_loads_round_trip() { let mut pyramid = UnivMonTwoTier::new(8, 2, 3, 32, 2, 16, 4); for key in 0..40u64 { pyramid.update_weighted(&DataInput::U64(key), 1 + (key as i64 % 5)); @@ -419,7 +419,7 @@ mod tests { /// A pyramid with no mouse layers is the one-tier case and still /// round-trips. #[test] - fn pyramid_without_mouse_layers_round_trips() { + fn univmon_two_tier_without_mouse_layers_round_trips() { let mut pyramid = UnivMonTwoTier::new(4, 4, 2, 16, 2, 8, 3); pyramid.update_weighted(&DataInput::U64(7), 3); let encoded = pyramid.to_asapv1().expect("serialize"); @@ -434,7 +434,7 @@ mod tests { /// An empty pyramid has exactly one encoding. /// An empty pyramid carries `key_type` `"u64"`; any other name is rejected. #[test] - fn pyramid_empty_must_carry_the_u64_key_type() { + fn univmon_two_tier_empty_must_carry_the_u64_key_type() { use crate::sketches::countminsketch_heap::heap_wire::relabel_u64_key_type; let bytes = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4) .to_asapv1() @@ -448,7 +448,7 @@ mod tests { } #[test] - fn pyramid_empty_has_one_encoding() { + fn univmon_two_tier_empty_has_one_encoding() { let left = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); let mut right = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); right.update_weighted(&DataInput::Str("alpha"), 5); @@ -463,7 +463,7 @@ mod tests { /// `update_mode` and `candidate_complete` are state, not derived. #[test] - fn pyramid_carries_update_mode_and_candidate_flags() { + fn univmon_two_tier_carries_update_mode_and_candidate_flags() { let mut terminal = UnivMonTwoTier::new(2, 1, 2, 8, 2, 8, 3); for key in 0..20u64 { terminal.update_weighted_terminal(&DataInput::U64(key), 3); @@ -530,7 +530,7 @@ mod tests { /// profile: (a) it emits that profile, (b) a custom-profile envelope is /// different bytes, and (c) decode fails closed on it. #[test] - fn pyramid_pins_its_hash_profile() { + fn univmon_two_tier_pins_its_hash_profile() { let encoded = populated().to_asapv1().expect("serialize"); let meta = metadata_of(&encoded); assert_eq!(meta.hash_profile_id, DefaultXxHasher::PROFILE_ID); @@ -565,7 +565,7 @@ mod tests { /// Each family's envelope is rejected by the other three, and by a plain /// Count Sketch envelope. #[test] - fn pyramid_rejects_foreign_kind_ids() { + fn univmon_two_tier_rejects_foreign_kind_ids() { let count_sketch = crate::CountSketch::, RegularPath>::with_dimensions(3, 8) .to_asapv1() .expect("serialize Count Sketch"); @@ -599,7 +599,7 @@ mod tests { /// capacities, including a `layers` far larger than the payload /// carries. #[test] - fn pyramid_rejects_crafted_shapes() { + fn univmon_two_tier_rejects_crafted_shapes() { let encoded = populated().to_asapv1().expect("serialize"); let base = metadata_of(&encoded); let payload = payload_of::(&encoded); @@ -701,7 +701,7 @@ mod tests { /// a pyramid decoded with zero mouse dimensions panics on any merge /// against a normally built one. #[test] - fn pyramid_rejects_zero_dimensions_of_a_tier_it_never_materializes() { + fn univmon_two_tier_rejects_zero_dimensions_of_a_tier_it_never_materializes() { let mut one_tier = UnivMonTwoTier::new(4, 4, 2, 16, 2, 8, 3); one_tier.update_weighted(&DataInput::U64(7), 3); let encoded = one_tier.to_asapv1().expect("serialize"); @@ -721,7 +721,7 @@ mod tests { /// runs the same `check_distinct_keys` per layer, so the pyramid never /// emits it. #[test] - fn pyramid_refuses_to_serialize_a_layer_holding_a_key_twice() { + fn univmon_two_tier_refuses_to_serialize_a_layer_holding_a_key_twice() { let mut nans = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); nans.layer_heaps[2].update(&DataInput::F64(f64::NAN), 5); nans.layer_heaps[2].update(&DataInput::F64(f64::NAN), 4); @@ -746,7 +746,7 @@ mod tests { /// refuses a zero dimension, so the encoder must too. Each case below is /// built by writing the public field directly. #[test] - fn pyramid_refuses_to_serialize_a_zero_dimension() { + fn univmon_two_tier_refuses_to_serialize_a_zero_dimension() { for zero in [ "layers", "heap_size", @@ -776,7 +776,7 @@ mod tests { /// [`UNIVMON_MAX_LAYERS`] layers is the ceiling: the boundary round-trips and a /// deeper pyramid is refused at the decode door. #[test] - fn pyramid_rejects_layers_past_the_shift_bound() { + fn univmon_two_tier_rejects_layers_past_the_shift_bound() { let deepest = UnivMonTwoTier::new(4, 2, 2, 16, 2, 8, UNIVMON_MAX_LAYERS); let encoded = deepest.to_asapv1().expect("serialize"); assert!(UnivMonTwoTier::from_asapv1(&encoded).is_ok()); @@ -797,14 +797,14 @@ mod tests { #[test] #[should_panic(expected = "at most UNIVMON_MAX_LAYERS")] - fn pyramid_refuses_constructing_past_the_shift_bound() { + fn univmon_two_tier_refuses_constructing_past_the_shift_bound() { UnivMonTwoTier::new(4, 2, 2, 16, 2, 8, UNIVMON_MAX_LAYERS + 1); } /// A pyramid whose layers disagree with their declared tier or seed index /// must not serialize. #[test] - fn pyramid_rejects_serializing_an_inconsistent_layout() { + fn univmon_two_tier_rejects_serializing_an_inconsistent_layout() { let mut wrong_tier = UnivMonTwoTier::new(4, 2, 3, 16, 2, 8, 4); wrong_tier.layer_sketches[3] = CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 16, 3); @@ -824,7 +824,7 @@ mod tests { /// Fail closed on an unexpected metadata key, and on a missing required /// one. #[test] - fn pyramid_metadata_rejects_unknown_and_missing_keys() { + fn univmon_two_tier_metadata_rejects_unknown_and_missing_keys() { #[derive(Serialize)] struct WithExtra { metadata_version: u8, diff --git a/src/sketches/bloom.rs b/src/sketches/bloom.rs index 0345cb97..805a6db1 100644 --- a/src/sketches/bloom.rs +++ b/src/sketches/bloom.rs @@ -624,7 +624,7 @@ mod tests { /// escaped its slice, or a slice width that is not a power of two folding /// wrong, shows up here as a bit count that is not the row count. #[test] - fn a_single_insert_sets_exactly_one_bit_in_every_slice() { + fn a_single_update_sets_exactly_one_bit_in_every_slice() { for cols in [1usize, 2, 7, 64, 100, WIDE_COLS] { for rows in [1usize, 5, BLOOM_MAX_SLICES] { let key = DataInput::Str("solo"); diff --git a/src/sketches/coco.rs b/src/sketches/coco.rs index 15694b14..ac21d877 100644 --- a/src/sketches/coco.rs +++ b/src/sketches/coco.rs @@ -322,7 +322,7 @@ mod tests { } #[test] - fn insert_then_estimate_matches_full_value_for_partial_key() { + fn update_then_estimate_matches_full_value_for_partial_key() { // cover end-to-end flow of inserting a key and querying with a substring let mut coco: CocoSketch = CocoSketch::with_dimensions(TEST_D, TEST_W); let key = "user:1234"; @@ -522,7 +522,7 @@ mod tests { } #[test] - fn estimate_key_never_exceeds_the_inserted_mass() { + fn estimate_never_exceeds_the_inserted_mass() { // biased replacement is unbiased in expectation but never invents mass // beyond what the whole table holds let mut coco: CocoSketch = CocoSketch::with_dimensions(2, 8); diff --git a/src/sketches/count_min_fold.rs b/src/sketches/count_min_fold.rs index fe627a1d..50f884ef 100644 --- a/src/sketches/count_min_fold.rs +++ b/src/sketches/count_min_fold.rs @@ -620,7 +620,7 @@ mod tests { } #[test] - fn cell_single_insert() { + fn cell_single_add() { let mut cell = FoldCell::Empty; cell.add(10, 5); assert_eq!(cell.entry_count(), 1); @@ -734,7 +734,7 @@ mod tests { // -- CountMinSketchFold basic tests ------------------------------------------------ #[test] - fn fold_cms_dimensions() { + fn count_min_sketch_fold_dimensions() { let sketch: CountMinSketchFold = CountMinSketchFold::new(3, 4096, 4, 10); assert_eq!(sketch.rows(), 3); assert_eq!(sketch.full_cols(), 4096); @@ -743,7 +743,7 @@ mod tests { } #[test] - fn fold_cms_level_zero_is_full() { + fn count_min_sketch_fold_level_zero_is_full() { let sketch: CountMinSketchFold = CountMinSketchFold::unfolded(3, 1024, 10); assert_eq!(sketch.fold_cols(), 1024); assert_eq!(sketch.fold_level(), 0); @@ -751,18 +751,18 @@ mod tests { #[test] #[should_panic(expected = "full_cols must be a power of two")] - fn fold_cms_rejects_non_power_of_two() { + fn count_min_sketch_fold_rejects_non_power_of_two() { let _: CountMinSketchFold = CountMinSketchFold::new(3, 1000, 0, 10); } #[test] #[should_panic(expected = "fold_level")] - fn fold_cms_rejects_excessive_fold_level() { + fn count_min_sketch_fold_rejects_excessive_fold_level() { let _: CountMinSketchFold = CountMinSketchFold::new(3, 256, 9, 10); // 256 = 2^8, fold_level 9 is too big } #[test] - fn fold_cms_insert_query_single_key() { + fn count_min_sketch_fold_update_estimate_single_key() { let mut sketch: CountMinSketchFold = CountMinSketchFold::new(3, 1024, 4, 10); let key = DataInput::Str("hello"); sketch.update_weighted(&key, 7); @@ -770,7 +770,7 @@ mod tests { } #[test] - fn fold_cms_insert_accumulates() { + fn count_min_sketch_fold_update_accumulates() { let mut sketch: CountMinSketchFold = CountMinSketchFold::new(3, 1024, 4, 10); let key = DataInput::Str("hello"); sketch.update_weighted(&key, 3); @@ -779,14 +779,14 @@ mod tests { } #[test] - fn fold_cms_absent_key_returns_zero() { + fn count_min_sketch_fold_absent_key_returns_zero() { let mut sketch: CountMinSketchFold = CountMinSketchFold::new(3, 1024, 4, 10); sketch.update_weighted(&DataInput::Str("present"), 10); assert_eq!(sketch.estimate(&DataInput::Str("absent")), 0); } #[test] - fn fold_cms_multiple_keys() { + fn count_min_sketch_fold_multiple_keys() { let mut sketch: CountMinSketchFold = CountMinSketchFold::new(3, 4096, 4, 10); for i in 0..100u64 { sketch.update_weighted(&DataInput::U64(i), i as i64); @@ -804,7 +804,7 @@ mod tests { // -- Exact match with standard CountMinSketch --------------------------------- #[test] - fn fold_cms_matches_standard_cms_exact() { + fn count_min_sketch_fold_matches_standard_cms_exact() { let rows = 3; let cols = 256; // small for deterministic testing let fold_level = 3; // 256/8 = 32 physical columns @@ -842,7 +842,7 @@ mod tests { } #[test] - fn fold_cms_matches_standard_cms_insert_many() { + fn count_min_sketch_fold_matches_standard_cms_update_weighted() { let rows = 3; let cols = 512; let fold_level = 4; @@ -1245,7 +1245,7 @@ mod tests { // -- Error bound (statistical) ------------------------------------------ #[test] - fn fold_cms_error_bound_zipf() { + fn count_min_sketch_fold_error_bound_zipf() { let rows = 3; let cols = 4096; let fold_level = 4; diff --git a/src/sketches/count_sketch_fold.rs b/src/sketches/count_sketch_fold.rs index 57a5e723..0afadc09 100644 --- a/src/sketches/count_sketch_fold.rs +++ b/src/sketches/count_sketch_fold.rs @@ -443,7 +443,7 @@ mod tests { // -- CountSketchFold basic tests ------------------------------------------------- #[test] - fn fold_cs_dimensions() { + fn count_sketch_fold_dimensions() { let sketch: CountSketchFold = CountSketchFold::new(3, 4096, 4, 10); assert_eq!(sketch.rows(), 3); assert_eq!(sketch.full_cols(), 4096); @@ -452,7 +452,7 @@ mod tests { } #[test] - fn fold_cs_level_zero_is_full() { + fn count_sketch_fold_level_zero_is_full() { let sketch: CountSketchFold = CountSketchFold::unfolded(3, 1024, 10); assert_eq!(sketch.fold_cols(), 1024); assert_eq!(sketch.fold_level(), 0); @@ -460,18 +460,18 @@ mod tests { #[test] #[should_panic(expected = "full_cols must be a power of two")] - fn fold_cs_rejects_non_power_of_two() { + fn count_sketch_fold_rejects_non_power_of_two() { let _: CountSketchFold = CountSketchFold::new(3, 1000, 0, 10); } #[test] #[should_panic(expected = "fold_level")] - fn fold_cs_rejects_excessive_fold_level() { + fn count_sketch_fold_rejects_excessive_fold_level() { let _: CountSketchFold = CountSketchFold::new(3, 256, 9, 10); // 256 = 2^8, fold_level 9 is too big } #[test] - fn fold_cs_insert_query_single_key() { + fn count_sketch_fold_update_estimate_single_key() { let mut sketch: CountSketchFold = CountSketchFold::new(3, 1024, 4, 10); let key = DataInput::Str("hello"); sketch.update_weighted(&key, 7); @@ -479,7 +479,7 @@ mod tests { } #[test] - fn fold_cs_insert_accumulates() { + fn count_sketch_fold_update_accumulates() { let mut sketch: CountSketchFold = CountSketchFold::new(3, 1024, 4, 10); let key = DataInput::Str("hello"); sketch.update_weighted(&key, 3); @@ -488,14 +488,14 @@ mod tests { } #[test] - fn fold_cs_absent_key_returns_zero() { + fn count_sketch_fold_absent_key_returns_zero() { let mut sketch: CountSketchFold = CountSketchFold::new(3, 1024, 4, 10); sketch.update_weighted(&DataInput::Str("present"), 10); assert_eq!(sketch.estimate(&DataInput::Str("absent")), 0); } #[test] - fn fold_cs_multiple_keys() { + fn count_sketch_fold_multiple_keys() { let mut sketch: CountSketchFold = CountSketchFold::new(3, 4096, 4, 10); for i in 0..100u64 { sketch.update_weighted(&DataInput::U64(i), i as i64); @@ -515,7 +515,7 @@ mod tests { // -- Sign application check --------------------------------------------- #[test] - fn fold_cs_sign_application() { + fn count_sketch_fold_sign_application() { // Verify that raw cell values include both positive and negative entries, // confirming sign is being applied on insert. let mut sketch: CountSketchFold = CountSketchFold::new(5, 1024, 4, 10); @@ -544,7 +544,7 @@ mod tests { // -- Exact match with standard Count Sketch ----------------------------- #[test] - fn fold_cs_matches_standard_cs_exact() { + fn count_sketch_fold_matches_standard_cs_exact() { let rows = 3; let cols = 256; // small for deterministic testing let fold_level = 3; // 256/8 = 32 physical columns @@ -570,7 +570,7 @@ mod tests { } #[test] - fn fold_cs_matches_standard_cs_flat_counters() { + fn count_sketch_fold_matches_standard_cs_flat_counters() { let rows = 3; let cols = 256; let fold_level = 3; @@ -597,7 +597,7 @@ mod tests { } #[test] - fn fold_cs_matches_standard_cs_insert_many() { + fn count_sketch_fold_matches_standard_cs_update_weighted() { let rows = 3; let cols = 512; let fold_level = 4; @@ -988,7 +988,7 @@ mod tests { // -- Error bound (statistical) ------------------------------------------ #[test] - fn fold_cs_error_bound_zipf() { + fn count_sketch_fold_error_bound_zipf() { let rows = 3; let cols = 4096; let fold_level = 4; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index bc0b7183..8038640e 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -708,7 +708,7 @@ mod tests { use crate::{DataInput, hash64_seeded}; #[test] - fn countmin_insert_emit_delta_emits_at_threshold_and_resets_period() { + fn countmin_update_emit_delta_emits_at_threshold_and_resets_period() { let mut sketch = CountMinSketch::, RegularPath>::with_dimensions(3, 64); let key = DataInput::U64(42); let mut deltas: Vec = Vec::new(); @@ -776,7 +776,7 @@ mod tests { } #[test] - fn fast_insert_same_estimate() { + fn fast_path_update_same_estimate() { let mut slow = CountMinSketch::, RegularPath>::with_dimensions(3, 64); let mut fast = CountMinSketch::, FastPath>::with_dimensions(3, 64); diff --git a/src/sketches/countminsketch_heap.rs b/src/sketches/countminsketch_heap.rs index 987fe82c..7de2a51d 100644 --- a/src/sketches/countminsketch_heap.rs +++ b/src/sketches/countminsketch_heap.rs @@ -404,7 +404,7 @@ mod tests { } #[test] - fn insert_and_estimate() { + fn update_and_estimate() { // Verifies single-key inserts update both CMS estimate and visible frequency. let mut sh = CountMinSketchWithHeap::, RegularPath>::new(3, 64, 10); let key = DataInput::Str("hello"); @@ -487,7 +487,7 @@ mod tests { } #[test] - fn insert_many_updates_estimate_and_heap() { + fn update_weighted_updates_estimate_and_heap() { // Verifies update_weighted updates both estimate and heap entry count. let mut sh = CountMinSketchWithHeap::, RegularPath>::new(3, 1024, 4); let key = DataInput::Str("many"); @@ -498,7 +498,7 @@ mod tests { } #[test] - fn bulk_insert_updates_multiple_keys() { + fn update_batch_updates_multiple_keys() { // Verifies update_batch processes streams correctly across multiple keys. let mut sh = CountMinSketchWithHeap::, RegularPath>::new(3, 1024, 4); let values = vec![ @@ -564,7 +564,7 @@ mod tests { } #[test] - fn fast_path_insert_and_estimate() { + fn fast_path_update_and_estimate() { // Verifies FastPath insert and estimate agree on repeated updates. let mut sh = CountMinSketchWithHeap::, FastPath>::new(3, 64, 10); let key = DataInput::Str("fast"); @@ -575,7 +575,7 @@ mod tests { } #[test] - fn fast_path_insert_many_and_bulk_insert() { + fn fast_path_update_weighted_and_update_batch() { // Verifies FastPath batched APIs keep estimate and heap counts in sync. let mut sh = CountMinSketchWithHeap::, FastPath>::new(3, 1024, 4); let key = DataInput::Str("fast-many"); @@ -738,7 +738,7 @@ mod tests { } #[test] - fn bulk_insert_equivalent_to_repeated_insert() { + fn update_batch_equivalent_to_repeated_update() { // Verifies update_batch yields same estimates and heap counts as repeated insert. let values = vec![ DataInput::U64(1), diff --git a/src/sketches/countminsketch_heap/wire.rs b/src/sketches/countminsketch_heap/wire.rs index 768d0eec..c97e3132 100644 --- a/src/sketches/countminsketch_heap/wire.rs +++ b/src/sketches/countminsketch_heap/wire.rs @@ -162,7 +162,7 @@ mod tests { } #[test] - fn cms_heap_round_trip_serialization() { + fn count_min_sketch_with_heap_round_trip_serialization() { let sketch = populated(); let encoded = sketch .to_asapv1() @@ -203,7 +203,7 @@ mod tests { /// The `f64` base counter is a Count-Min wire type, so a float-counter /// CountMinSketchWithHeap serializes as well. #[test] - fn cms_heap_f64_counters_round_trip() { + fn count_min_sketch_with_heap_f64_counters_round_trip() { let mut sketch = CountMinSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |r, c| (r * 4 + c) as f64 + 0.5), 4, @@ -228,7 +228,7 @@ mod tests { /// so a key that is not UTF-8 round trips and still answers its own /// `DataInput::Bytes`. #[test] - fn cms_heap_byte_keys_round_trip() { + fn count_min_sketch_with_heap_byte_keys_round_trip() { let raw: &[u8] = &[0xff, 0x00, 0xfe]; let mut sketch = CountMinSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |r, c| (r * 4 + c) as i64), @@ -252,7 +252,7 @@ mod tests { /// The base counter type is pinned by the target: i64 bytes must not decode /// into an f64 sketch or the reverse. #[test] - fn cms_heap_counter_type_is_pinned_by_the_target() { + fn count_min_sketch_with_heap_counter_type_is_pinned_by_the_target() { let wide = CountMinSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |r, c| (r * 4 + c) as i64), 4, @@ -276,7 +276,7 @@ mod tests { } #[test] - fn cms_heap_mode_in_metadata_round_trips() { + fn count_min_sketch_with_heap_mode_in_metadata_round_trips() { let mut sketch = CountMinSketchWithHeap::, FastPath>::new(4, 16, 8); sketch.update_weighted(&DataInput::U64(1), 5); sketch.update_weighted(&DataInput::U64(2), 3); @@ -300,7 +300,7 @@ mod tests { /// CountMinSketchWithHeap, CountSketchWithHeap and the two base sketches all carry different kind_ids; /// a foreign envelope must not decode as a CountMinSketchWithHeap. #[test] - fn cms_heap_rejects_foreign_kind_ids() { + fn count_min_sketch_with_heap_rejects_foreign_kind_ids() { let cs_heap = crate::CountSketchWithHeap::, RegularPath>::new(3, 8, 4); let cms = crate::CountMinSketch::, RegularPath>::with_dimensions(3, 8); let count = crate::CountSketch::, RegularPath>::with_dimensions(3, 8); @@ -340,7 +340,7 @@ mod tests { /// `Vector2D::from_fn` derives its mask via `cols.ilog2()`, which panics on /// `cols == 0`. #[test] - fn cms_heap_rejects_zero_dimension_payload() { + fn count_min_sketch_with_heap_rejects_zero_dimension_payload() { let bytes = crafted(4, 0, 4, Vec::new()); let problem = decode_error(&bytes); assert!(problem.contains("must be non-zero"), "got {problem}"); @@ -349,7 +349,7 @@ mod tests { /// Crafted dimensions must not drive a huge allocation: the length check /// runs before `Vector2D::from_fn`. #[test] - fn cms_heap_rejects_dimension_length_mismatch() { + fn count_min_sketch_with_heap_rejects_dimension_length_mismatch() { let bytes = crafted(MATRIX_MAX_ROWS as u32, 1 << 24, 4, vec![1, 2, 3]); let problem = decode_error(&bytes); assert!(problem.contains("!= rows*cols"), "got {problem}"); @@ -358,7 +358,7 @@ mod tests { /// More rows than the seed list has seeds is outside the wire-eligible /// subset, on both sides. #[test] - fn cms_heap_rejects_too_many_rows() { + fn count_min_sketch_with_heap_rejects_too_many_rows() { let rows = MATRIX_MAX_ROWS + 1; assert!( CountMinSketchWithHeap::, RegularPath>::new(rows, 8, 4) @@ -381,7 +381,7 @@ mod tests { /// A `rows == 0` matrix satisfies the length check trivially but is refused /// on decode, so serializing it must fail. #[test] - fn cms_heap_rejects_serializing_zero_rows() { + fn count_min_sketch_with_heap_rejects_serializing_zero_rows() { let sketch = CountMinSketchWithHeap::, RegularPath>::new(0, 4, 4); let problem = sketch .to_asapv1() @@ -393,7 +393,7 @@ mod tests { /// An unpopulated matrix carries dimensions its cells do not match. /// Serializing it must fail rather than emit bytes the decoder refuses. #[test] - fn cms_heap_rejects_serializing_an_unfilled_matrix() { + fn count_min_sketch_with_heap_rejects_serializing_an_unfilled_matrix() { let sketch = CountMinSketchWithHeap::, RegularPath>::from_storage( Vector2D::::unfilled(2, 4), 4, @@ -406,7 +406,7 @@ mod tests { /// Fail closed on an unexpected metadata key. #[test] - fn cms_heap_metadata_rejects_unknown_keys() { + fn count_min_sketch_with_heap_metadata_rejects_unknown_keys() { #[derive(Serialize)] struct WithExtra { metadata_version: u8, @@ -451,7 +451,7 @@ mod tests { /// `heap_size` is required: a metadata map missing it does not decode, so /// the heap capacity can never be silently defaulted. #[test] - fn cms_heap_metadata_rejects_a_missing_heap_size_key() { + fn count_min_sketch_with_heap_metadata_rejects_a_missing_heap_size_key() { #[derive(Serialize)] struct WithoutHeapSize { metadata_version: u8, @@ -494,7 +494,7 @@ mod tests { /// but it names a counter this `i64` sketch does not hold, and must not /// decode into it. #[test] - fn cms_heap_rejects_a_foreign_counter_type_name() { + fn count_min_sketch_with_heap_rejects_a_foreign_counter_type_name() { let metadata = rmp_serde::to_vec_named(&heap_metadata::( 2, 4, "i32", "regular", 4, "u64", )) @@ -558,7 +558,7 @@ mod tests { } #[test] - fn cms_heap_custom_hasher_profile_round_trips_and_is_self_describing() { + fn count_min_sketch_with_heap_custom_hasher_profile_round_trips_and_is_self_describing() { // (a) A CountMinSketchWithHeap built with a custom-profile hasher round-trips. let mut alt = CountMinSketchWithHeap::, RegularPath, AltHasher>::new(3, 8, 4); let mut std = CountMinSketchWithHeap::, RegularPath>::new(3, 8, 4); @@ -593,7 +593,7 @@ mod tests { /// A `heap_size` past the metadata's `u32` field fails the encode rather than /// emitting a truncated capacity. #[test] - fn cms_heap_refuses_a_heap_size_the_metadata_cannot_carry() { + fn count_min_sketch_with_heap_refuses_a_heap_size_the_metadata_cannot_carry() { let sketch = CountMinSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |_, _| 0i64), 1usize << 40, diff --git a/src/sketches/countminsketch_hll.rs b/src/sketches/countminsketch_hll.rs index b7bd2a10..3088e610 100644 --- a/src/sketches/countminsketch_hll.rs +++ b/src/sketches/countminsketch_hll.rs @@ -526,7 +526,7 @@ mod tests { } #[test] - fn insert_many_matches_sequential_inserts() { + fn update_batch_matches_sequential_updates() { let k = key("user_A"); let vals: Vec> = (0..500u64).map(val).collect(); let pairs: Vec<(&DataInput, &DataInput)> = vals.iter().map(|v| (&k, v)).collect(); diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 2cbf536f..036ff942 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -706,7 +706,7 @@ mod tests { use std::collections::HashMap; #[test] - fn count_child_insert_emits_at_threshold() { + fn count_sketch_child_update_emits_at_threshold() { let mut child = CountSketch::, RegularPath>::with_dimensions(3, 64); let key = DataInput::U64(99); let mut deltas: Vec = Vec::new(); @@ -771,7 +771,7 @@ mod tests { } #[test] - fn insert_updates_signed_counters_per_row() { + fn update_writes_signed_counters_per_row() { let mut sketch = CountSketch::, RegularPath>::with_dimensions(3, 64); let key = DataInput::Str("alpha"); @@ -789,7 +789,7 @@ mod tests { } #[test] - fn fast_insert_produces_consistent_estimates() { + fn fast_path_update_produces_consistent_estimates() { let mut fast = CountSketch::, FastPath>::with_dimensions(4, 128); let keys = vec![ @@ -814,7 +814,7 @@ mod tests { } #[test] - fn insert_produces_consistent_estimates() { + fn update_produces_consistent_estimates() { let mut sketch = CountSketch::, RegularPath>::with_dimensions(3, 64); let keys = vec![ diff --git a/src/sketches/countsketch_heap.rs b/src/sketches/countsketch_heap.rs index 2f1d8330..a5574a81 100644 --- a/src/sketches/countsketch_heap.rs +++ b/src/sketches/countsketch_heap.rs @@ -462,7 +462,7 @@ mod tests { } #[test] - fn insert_and_estimate() { + fn update_and_estimate() { // Verifies single-key inserts update both CS estimate and wrapper behavior. let mut sh = CountSketchWithHeap::, RegularPath>::new(5, 256, 10); let key = DataInput::Str("hello"); @@ -560,7 +560,7 @@ mod tests { } #[test] - fn insert_many_updates_estimate_and_heap() { + fn update_weighted_updates_estimate_and_heap() { // Verifies update_weighted updates estimate and heap count consistently. let mut sh = CountSketchWithHeap::, RegularPath>::new(5, 2048, 4); let key = DataInput::Str("many"); @@ -572,7 +572,7 @@ mod tests { } #[test] - fn bulk_insert_updates_multiple_keys() { + fn update_batch_updates_multiple_keys() { // Verifies bulk stream ingestion updates multiple keys and heap tracking. let mut sh = CountSketchWithHeap::, RegularPath>::new(5, 2048, 4); let values = vec![ @@ -643,7 +643,7 @@ mod tests { } #[test] - fn fast_path_insert_and_estimate() { + fn fast_path_update_and_estimate() { // Verifies FastPath insert and estimate stay coherent for repeated keys. let mut sh = CountSketchWithHeap::, FastPath>::new(5, 256, 10); let key = DataInput::Str("fast"); @@ -654,7 +654,7 @@ mod tests { } #[test] - fn fast_path_insert_many_and_bulk_insert() { + fn fast_path_update_weighted_and_update_batch() { // Verifies FastPath batched APIs maintain estimate/heap consistency. let mut sh = CountSketchWithHeap::, FastPath>::new(5, 2048, 4); let key = DataInput::Str("fast-many"); @@ -818,7 +818,7 @@ mod tests { } #[test] - fn bulk_insert_equivalent_to_repeated_insert() { + fn update_batch_equivalent_to_repeated_update() { // Verifies update_batch behavior matches repeated insert behavior. let values = vec![ DataInput::U64(1), diff --git a/src/sketches/countsketch_heap/wire.rs b/src/sketches/countsketch_heap/wire.rs index 7f62a7d5..e7e48dc7 100644 --- a/src/sketches/countsketch_heap/wire.rs +++ b/src/sketches/countsketch_heap/wire.rs @@ -166,7 +166,7 @@ mod tests { } #[test] - fn cs_heap_round_trip_serialization() { + fn count_sketch_with_heap_round_trip_serialization() { let sketch = populated(); let encoded = sketch.to_asapv1().expect("serialize CountSketchWithHeap"); assert!(encoded.starts_with(b"ASAPv1")); @@ -205,7 +205,7 @@ mod tests { /// Count Sketch cells are signed — the payload must carry negatives through /// unchanged. #[test] - fn cs_heap_negative_counters_round_trip() { + fn count_sketch_with_heap_negative_counters_round_trip() { let mut sketch = CountSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |r, c| { let v = (r * 4 + c) as i64; @@ -238,7 +238,7 @@ mod tests { /// The `i32` wire config round-trips, and the counter type is pinned by the /// target: i32 bytes must not decode into an i64 sketch or the reverse. #[test] - fn cs_heap_i32_round_trips_and_is_pinned_by_counter_type() { + fn count_sketch_with_heap_i32_round_trips_and_is_pinned_by_counter_type() { let cells = |r: usize, c: usize| { let v = (r * 4 + c) as i32; if v % 2 == 0 { v } else { -v } @@ -276,7 +276,7 @@ mod tests { } #[test] - fn cs_heap_mode_in_metadata_round_trips() { + fn count_sketch_with_heap_mode_in_metadata_round_trips() { let mut sketch = CountSketchWithHeap::, FastPath>::new(4, 16, 8); sketch.update_weighted(&DataInput::U64(1), 5); sketch.update_weighted(&DataInput::U64(2), 3); @@ -298,7 +298,7 @@ mod tests { /// CountSketchWithHeap, CountMinSketchWithHeap and the two base sketches all carry different kind_ids; /// a foreign envelope must not decode as a CountSketchWithHeap. #[test] - fn cs_heap_rejects_foreign_kind_ids() { + fn count_sketch_with_heap_rejects_foreign_kind_ids() { let cms_heap = crate::CountMinSketchWithHeap::, RegularPath>::new(3, 8, 4); let cms = crate::CountMinSketch::, RegularPath>::with_dimensions(3, 8); let count = crate::CountSketch::, RegularPath>::with_dimensions(3, 8); @@ -338,7 +338,7 @@ mod tests { /// `Vector2D::from_fn` derives its mask via `cols.ilog2()`, which panics on /// `cols == 0`. #[test] - fn cs_heap_rejects_zero_dimension_payload() { + fn count_sketch_with_heap_rejects_zero_dimension_payload() { let bytes = crafted(4, 0, 4, Vec::new()); let problem = decode_error(&bytes); assert!(problem.contains("must be non-zero"), "got {problem}"); @@ -347,7 +347,7 @@ mod tests { /// Crafted dimensions must not drive a huge allocation: the length check /// runs before `Vector2D::from_fn`. #[test] - fn cs_heap_rejects_dimension_length_mismatch() { + fn count_sketch_with_heap_rejects_dimension_length_mismatch() { let bytes = crafted(MATRIX_MAX_ROWS as u32, 1 << 24, 4, vec![1, 2, 3]); let problem = decode_error(&bytes); assert!(problem.contains("!= rows*cols"), "got {problem}"); @@ -356,7 +356,7 @@ mod tests { /// More rows than the seed list has seeds is outside the wire-eligible /// subset, on both sides. #[test] - fn cs_heap_rejects_too_many_rows() { + fn count_sketch_with_heap_rejects_too_many_rows() { let rows = MATRIX_MAX_ROWS + 1; assert!( CountSketchWithHeap::, RegularPath>::new(rows, 8, 4) @@ -379,7 +379,7 @@ mod tests { /// An unpopulated matrix carries dimensions its cells do not match. /// Serializing it must fail rather than emit bytes the decoder refuses. #[test] - fn cs_heap_rejects_serializing_an_unfilled_matrix() { + fn count_sketch_with_heap_rejects_serializing_an_unfilled_matrix() { let sketch = CountSketchWithHeap::, RegularPath>::from_storage( Vector2D::::unfilled(2, 4), 4, @@ -392,7 +392,7 @@ mod tests { /// Fail closed on an unexpected metadata key. #[test] - fn cs_heap_metadata_rejects_unknown_keys() { + fn count_sketch_with_heap_metadata_rejects_unknown_keys() { #[derive(Serialize)] struct WithExtra { metadata_version: u8, @@ -437,7 +437,7 @@ mod tests { /// `key_type` is required: a metadata map missing it does not decode, so the /// heap's key variant can never be silently defaulted. #[test] - fn cs_heap_metadata_rejects_a_missing_key_type_key() { + fn count_sketch_with_heap_metadata_rejects_a_missing_key_type_key() { #[derive(Serialize)] struct WithoutKeyType { metadata_version: u8, @@ -477,7 +477,7 @@ mod tests { /// An `f64`-labelled envelope decodes into neither integer sketch. #[test] - fn cs_heap_rejects_f64_into_an_integer_sketch() { + fn count_sketch_with_heap_rejects_f64_into_an_integer_sketch() { let metadata = rmp_serde::to_vec_named(&heap_metadata::( 2, 4, "f64", "regular", 4, "u64", )) @@ -545,7 +545,7 @@ mod tests { } #[test] - fn cs_heap_custom_hasher_profile_round_trips_and_is_self_describing() { + fn count_sketch_with_heap_custom_hasher_profile_round_trips_and_is_self_describing() { // (a) A CountSketchWithHeap built with a custom-profile hasher round-trips. let mut alt = CountSketchWithHeap::, RegularPath, AltHasher>::new(3, 8, 4); let mut std = CountSketchWithHeap::, RegularPath>::new(3, 8, 4); @@ -579,7 +579,7 @@ mod tests { /// A `heap_size` past the metadata's `u32` field fails the encode rather than /// emitting a truncated capacity. #[test] - fn cs_heap_refuses_a_heap_size_the_metadata_cannot_carry() { + fn count_sketch_with_heap_refuses_a_heap_size_the_metadata_cannot_carry() { let sketch = CountSketchWithHeap::, RegularPath>::from_storage( Vector2D::from_fn(2, 4, |_, _| 0i64), 1usize << 40, diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs index 0d4ee7bb..1e98bc42 100644 --- a/src/sketches/countsketch_l2_norm.rs +++ b/src/sketches/countsketch_l2_norm.rs @@ -284,21 +284,21 @@ mod tests_count_l2_hh { /// hash: the deep rows' columns would reuse the shallow rows' signs. #[test] #[should_panic(expected = "hash bits")] - fn countl2hh_rejects_column_bits_that_reach_the_sign_bits() { + fn count_sketch_with_l2_norm_rejects_column_bits_that_reach_the_sign_bits() { let _: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions_and_seed_index(11, 2048, 0); } /// 10 x 2048 spends 120 bits and is accepted. #[test] - fn countl2hh_accepts_the_widest_disjoint_geometry() { + fn count_sketch_with_l2_norm_accepts_the_widest_disjoint_geometry() { let sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions_and_seed_index(10, 2048, 0); assert_eq!((sketch.rows(), sketch.cols()), (10, 2048)); } #[test] - fn countl2hh_estimates_and_l2_are_consistent() { + fn count_sketch_with_l2_norm_estimates_and_l2_are_consistent() { let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); let key = DataInput::Str("gamma"); @@ -313,7 +313,7 @@ mod tests_count_l2_hh { } #[test] - fn countl2hh_merge_combines_frequency_vectors() { + fn count_sketch_with_l2_norm_merge_combines_frequency_vectors() { let mut left: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); let mut right: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); let key = DataInput::U32(42); @@ -329,7 +329,7 @@ mod tests_count_l2_hh { } #[test] - fn countl2hh_round_trip_serialization() { + fn count_sketch_with_l2_norm_round_trip_preserves_estimate_and_l2() { let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions_and_seed_index(3, 32, 7); let key = DataInput::Str("serialize"); diff --git a/src/sketches/countsketch_l2_norm/wire.rs b/src/sketches/countsketch_l2_norm/wire.rs index 17981d3f..919f2c65 100644 --- a/src/sketches/countsketch_l2_norm/wire.rs +++ b/src/sketches/countsketch_l2_norm/wire.rs @@ -299,7 +299,7 @@ mod tests { /// More rows than the seed list has seeds is outside the wire-eligible /// subset, on both sides. #[test] - fn countl2hh_rejects_too_many_rows() { + fn count_sketch_with_l2_norm_rejects_too_many_rows() { let rows = MATRIX_MAX_ROWS + 1; assert!( Std::with_dimensions(rows, 32).to_asapv1().is_err(), @@ -325,7 +325,7 @@ mod tests { } #[test] - fn count_l2hh_round_trip_serialization() { + fn count_sketch_with_l2_norm_round_trip_serialization() { let sketch = populated(); let encoded = sketch.to_asapv1().expect("serialize CountSketchWithL2Norm"); assert!(encoded.starts_with(b"ASAPv1")); @@ -352,7 +352,7 @@ mod tests { /// A decoded sketch re-serializes byte-identically. #[test] - fn count_l2hh_decoded_re_serializes_identically() { + fn count_sketch_with_l2_norm_decoded_re_serializes_identically() { let sketch = populated(); let encoded = sketch.to_asapv1().expect("serialize"); let decoded = Std::from_asapv1(&encoded).expect("decode"); @@ -362,7 +362,7 @@ mod tests { /// Count Sketch cells are signed, so a CountSketchWithL2Norm carries negatives through /// unchanged. #[test] - fn count_l2hh_negative_counters_round_trip() { + fn count_sketch_with_l2_norm_negative_counters_round_trip() { let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(2, 8); sketch.update_weighted(&DataInput::U64(11), -25); sketch.update_weighted(&DataInput::U64(12), -7); @@ -381,7 +381,7 @@ mod tests { /// The seed index is state, not a profile constant: two sketches that /// differ only in it do not share bytes and do not decode into each other. #[test] - fn count_l2hh_seed_index_travels_with_the_sketch() { + fn count_sketch_with_l2_norm_seed_index_travels_with_the_sketch() { let zero: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions_and_seed_index(2, 8, 0); let nine: CountSketchWithL2Norm = @@ -438,7 +438,7 @@ mod tests { } #[test] - fn count_l2hh_custom_hasher_profile_round_trips_and_is_self_describing() { + fn count_sketch_with_l2_norm_custom_hasher_profile_round_trips_and_is_self_describing() { // (a) A CountSketchWithL2Norm built with a custom-profile hasher round-trips. let mut alt = CountSketchWithL2Norm::::with_dimensions(3, 32); let mut std: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); @@ -464,7 +464,7 @@ mod tests { /// Each family's envelope is rejected by the other three, and by a plain /// Count Sketch envelope. #[test] - fn count_l2hh_rejects_foreign_kind_ids() { + fn count_sketch_with_l2_norm_rejects_foreign_kind_ids() { let count_sketch = crate::CountSketch::, RegularPath>::with_dimensions(3, 8) .to_asapv1() .expect("serialize Count Sketch"); @@ -498,7 +498,7 @@ mod tests { /// dimension, and on one whose declared geometry the payload does not /// carry. Both checks precede any allocation. #[test] - fn count_l2hh_rejects_crafted_geometry() { + fn count_sketch_with_l2_norm_rejects_crafted_geometry() { let meta = count_sketch_with_l2_norm_metadata::(0, 4, 0); assert!(Std::from_asapv1(&crafted(&meta, Vec::new(), vec![0; 4])).is_err()); @@ -522,7 +522,7 @@ mod tests { /// An unpopulated matrix carries dimensions its cells do not match. /// Serializing it must fail rather than emit bytes the decoder refuses. #[test] - fn count_l2hh_rejects_serializing_an_unfilled_matrix() { + fn count_sketch_with_l2_norm_rejects_serializing_an_unfilled_matrix() { let mut sketch: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(2, 4); sketch.counts = Vector2D::unfilled(2, 4); assert!( @@ -534,7 +534,7 @@ mod tests { /// Fail closed on an unexpected metadata key, and on a missing required /// one. #[test] - fn l2hh_metadata_rejects_unknown_and_missing_keys() { + fn count_sketch_with_l2_norm_metadata_rejects_unknown_and_missing_keys() { #[derive(Serialize)] struct WithExtra { metadata_version: u8, @@ -614,7 +614,7 @@ mod tests { /// `rows = 12, cols = 4096` and `rows = 13, cols = 2048` both need 156: /// their last row shifts the hash by 132, past the 127-bit ceiling. #[test] - fn count_l2hh_rejects_a_geometry_past_the_hash_budget() { + fn count_sketch_with_l2_norm_rejects_a_geometry_past_the_hash_budget() { for (rows, cols) in [(12usize, 4096usize), (13, 2048)] { let problem = unchecked(rows, cols) .to_asapv1() @@ -638,7 +638,7 @@ mod tests { /// columns fill bits 0..119 with column fields while the sign bits start at /// 118, so two rows would read a column out of bits that are already signs. #[test] - fn count_l2hh_rejects_column_fields_overlapping_the_sign_bits() { + fn count_sketch_with_l2_norm_rejects_column_fields_overlapping_the_sign_bits() { assert!( unchecked(10, 4096).to_asapv1().is_err(), "column fields must not reach into the sign bits" @@ -656,7 +656,7 @@ mod tests { /// encoding of a state an in-range index already encodes — and two such /// sketches would refuse to merge. Neither door accepts it. #[test] - fn count_l2hh_rejects_a_seed_index_past_the_seed_list() { + fn count_sketch_with_l2_norm_rejects_a_seed_index_past_the_seed_list() { let seeds = ::seed_list().len(); let wrapped = seeds + 5; @@ -691,7 +691,7 @@ mod tests { /// An empty sketch has exactly one encoding. #[test] - fn count_l2hh_empty_has_one_encoding() { + fn count_sketch_with_l2_norm_empty_has_one_encoding() { let left: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); let mut right: CountSketchWithL2Norm = CountSketchWithL2Norm::with_dimensions(3, 32); right.update_weighted(&DataInput::U64(5), 4); diff --git a/src/sketches/elastic.rs b/src/sketches/elastic.rs index 5e40aefe..d117b7e8 100644 --- a/src/sketches/elastic.rs +++ b/src/sketches/elastic.rs @@ -642,7 +642,7 @@ mod tests { } #[test] - fn init_with_dimensions_sizes_both_parts() { + fn with_heavy_and_light_sizes_both_parts() { let sketch: ElasticSketch = ElasticSketch::with_heavy_and_light(12, 2, 256); assert_eq!(sketch.heavy.len(), 12); @@ -652,7 +652,7 @@ mod tests { } #[test] - fn init_with_length_keeps_the_default_light_layer() { + fn with_heavy_buckets_keeps_the_default_light_layer() { let sketch: ElasticSketch = ElasticSketch::with_heavy_buckets(8); assert_eq!(sketch.heavy.len(), 8); @@ -1106,7 +1106,7 @@ mod tests { } #[test] - fn heavy_only_insert_never_touches_the_light_layer() { + fn heavy_only_update_never_touches_the_light_layer() { // seed the light layer through the normal path so the comparison is // against real counters rather than an all-zero table let mut sketch: ElasticSketch = ElasticSketch::with_heavy_and_light(8, 2, 64); @@ -1222,7 +1222,7 @@ mod tests { } #[test] - fn heavy_only_matches_insert_while_buckets_seat_and_match() { + fn heavy_only_matches_update_while_buckets_seat_and_match() { let mut normal: ElasticSketch = ElasticSketch::with_heavy_buckets(16); let mut overload: ElasticSketch = ElasticSketch::with_heavy_buckets(16); @@ -1305,7 +1305,7 @@ mod tests { } #[test] - fn an_insert_onto_a_stale_copy_replaces_it() { + fn an_update_onto_a_stale_copy_replaces_it() { let (mut sk, _) = seeded_sketch(8, 24); sk.expand_heavy(); diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 47b74b57..5f25b46a 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -548,7 +548,7 @@ mod tests { const P12_ERROR_TOLERANCE: f64 = 0.03; #[test] - fn hll_child_insert_emits_on_improvement() { + fn hll_child_update_emits_on_improvement() { let mut child = HyperLogLog::::default(); let mut deltas: Vec = Vec::new(); diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 56916afc..e72de1b8 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -1710,7 +1710,7 @@ mod tests { } #[test] - fn bulk_update_equivalent_to_loop_and_empty_is_noop() { + fn update_batch_equivalent_to_loop_and_empty_is_noop() { // Empty must be no-op and keep cache/coin. let mut sk = Kll::::with_seed(200, 8, 42); for &v in &[1.0, 2.0, 3.0] { diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 61990951..d4c93a15 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -971,7 +971,7 @@ mod tests { } #[test] - fn bulk_update_equivalent_to_loop() { + fn update_batch_equivalent_to_loop() { let vals = sample_uniform_f64(0.0, 1000.0, 10_000, 0xCAFE_1234); let mut a = KllDynamic::::with_k(200); let mut b = KllDynamic::::with_k(200); diff --git a/src/sketches/microscope.rs b/src/sketches/microscope.rs index 833703bd..a98d7311 100644 --- a/src/sketches/microscope.rs +++ b/src/sketches/microscope.rs @@ -912,7 +912,7 @@ mod tests { #[test] #[should_panic(expected = "use update_at()")] - fn insert_refuses_a_time_based_clock() { + fn update_refuses_a_time_based_clock() { let mut sk: MicroCm = MicroCm::with_dimensions( 1, 16, @@ -925,7 +925,7 @@ mod tests { #[test] #[should_panic(expected = "use update()")] - fn insert_at_refuses_a_count_based_clock() { + fn update_at_refuses_a_count_based_clock() { let mut sk = sketch(1, 16, 4, 10); sk.update_at(&key(1), 5); } diff --git a/tests/bug_verification.rs b/tests/bug_verification.rs index a8bfbbf3..fa54cb58 100644 --- a/tests/bug_verification.rs +++ b/tests/bug_verification.rs @@ -6,7 +6,7 @@ //! //! The two Nitro defects the probe found — the insert and query paths reading //! different hash domains, and each sampled record reaching only one row — are -//! pinned by `full_sampling_is_exact_and_the_query_reads_the_cells_the_insert_wrote` +//! pinned by `full_sampling_is_exact_and_the_query_reads_the_cells_the_update_wrote` //! in `tests/e2e/nitro.rs`, which names both failure modes on the same run //! that already exercises every ingestion path. @@ -66,7 +66,7 @@ fn portable_ddsketch_respects_alpha_at_bucket_edges() { // Pre-fix behavior: 0 in release, overflow panic in debug. Post-fix: saturates. // --------------------------------------------------------------------------- #[test] -fn countl2hh_f2_survives_beyond_i64_max() { +fn count_sketch_with_l2_norm_f2_survives_beyond_i64_max() { let c = 3_000_000_000i64; let mut sk = CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 7); diff --git a/tests/conformance_kit.rs b/tests/conformance_kit.rs index 99ede9c3..b82dd286 100644 --- a/tests/conformance_kit.rs +++ b/tests/conformance_kit.rs @@ -707,7 +707,7 @@ fn heap_backed_frequency_sketches_pass_frequency_and_merge_conformance() { } #[test] -fn count_l2hh_passes_signed_frequency_and_merge_conformance() { +fn count_sketch_with_l2_norm_passes_signed_frequency_and_merge_conformance() { let stream = zipf_stream(); let truth = stream_truth(&stream); let spec = conformance::FrequencySpec { diff --git a/tests/e2e/experimental.rs b/tests/e2e/experimental.rs index 9fdae8eb..e6ea498b 100644 --- a/tests/e2e/experimental.rs +++ b/tests/e2e/experimental.rs @@ -535,7 +535,7 @@ const EH_LAYERS: usize = 2; const EH_K: usize = 2; #[test] -fn eh_univ_optimized_map_tier_exact_windows() { +fn exponential_histogram_univmon_exact_tier_windows() { let window = 100u64; let mut eh = ExponentialHistogramUnivMon::with_defaults(2, window); @@ -591,7 +591,7 @@ fn eh_univ_optimized_map_tier_exact_windows() { /// over 30k keys, n=200k, k=2, rows=3, cols=512, layers=2, stream seed 4242), /// where L2 lands 3.0% below exact and entropy 20.4% above it. #[test] -fn eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it() { +fn exponential_histogram_univmon_promotes_into_the_sketch_tier_and_answers_from_it() { const N: usize = 80_000; const DOMAIN: usize = 30_000; const STREAM_SEED: u64 = 4242; @@ -685,7 +685,7 @@ fn eh_univ_optimized_promotes_into_the_sketch_tier_and_answers_from_it() { /// The lower guard makes a future fix fail this test loudly instead of leaving /// a stale claim in the suite. #[test] -fn eh_univ_optimized_sketch_tier_cardinality_is_documented_as_unrecoverable() { +fn exponential_histogram_univmon_sketch_tier_cardinality_is_documented_as_unrecoverable() { const N: usize = 80_000; const DOMAIN: usize = 30_000; @@ -725,7 +725,7 @@ fn eh_univ_optimized_sketch_tier_cardinality_is_documented_as_unrecoverable() { /// inside the interval are replayed into a merged UnivMon. The result must /// still carry the whole window's L1 exactly. #[test] -fn eh_univ_optimized_answers_a_mixed_map_and_sketch_interval() { +fn exponential_histogram_univmon_answers_a_mixed_exact_and_sketch_interval() { const N: u64 = 60_000; const DOMAIN: u64 = 20_000; @@ -761,7 +761,7 @@ fn eh_univ_optimized_answers_a_mixed_map_and_sketch_interval() { /// span shrinks accordingly. Bucket bookkeeping is arithmetic, so these are /// equalities rather than bands. #[test] -fn eh_univ_optimized_expires_buckets_past_the_window() { +fn exponential_histogram_univmon_expires_buckets_past_the_window() { const WINDOW: u64 = 5_000; let mut eh = promoting_eh(8, WINDOW); for t in 0..40_000u64 { @@ -821,7 +821,7 @@ fn eh_univ_optimized_expires_buckets_past_the_window() { /// Recycling through the sketch pool must not leak state between windows: a /// sketch taken from the pool has to behave like a fresh one. #[test] -fn eh_univ_optimized_reuses_pooled_sketches_without_leaking_state() { +fn exponential_histogram_univmon_reuses_pooled_sketches_without_leaking_state() { const DOMAIN: u64 = 20_000; // Long run with a short window, so buckets are created, promoted, merged // and expired repeatedly and the pool is exercised. @@ -852,7 +852,7 @@ fn eh_univ_optimized_reuses_pooled_sketches_without_leaking_state() { /// counts, which is the reference the sketch tier's estimates are compared to /// elsewhere in this file. #[test] -fn eh_univ_optimized_map_tier_matches_exact_per_key_counts_on_a_skewed_stream() { +fn exponential_histogram_univmon_exact_tier_matches_exact_per_key_counts_on_a_skewed_stream() { const N: u64 = 4_000; let mut eh = ExponentialHistogramUnivMon::with_defaults(4, 1_000_000); let keys = zipf_u64(N as usize, 64, 1.2, 9_001); diff --git a/tests/e2e/frameworks.rs b/tests/e2e/frameworks.rs index 4f877a2d..67ea89a9 100644 --- a/tests/e2e/frameworks.rs +++ b/tests/e2e/frameworks.rs @@ -1277,7 +1277,7 @@ fn hydra_univmon_head_subpopulation_metrics() { if l1 != exact.total() as f64 { failures.push(format!("{key:?}: L1 {l1} != exact {}", exact.total())); } - // Bands from `univmon_weighted_metrics_and_fast_insert_parity`. + // Bands from `univmon_weighted_metrics_and_update_weighted_terminal_parity`. for (label, est, want, tol) in [ ( "L2", @@ -1482,7 +1482,7 @@ fn hydra_shard_merge_preserves_answers_for_every_counter() { // ----------------------------------------------------------------- UnivMon #[test] -fn univmon_weighted_metrics_and_fast_insert_parity() { +fn univmon_weighted_metrics_and_update_weighted_terminal_parity() { let build = || UnivMon::new(32, 5, 2048, 8); let mut um = build(); let mut fast = build(); @@ -1533,7 +1533,7 @@ fn univmon_weighted_metrics_and_fast_insert_parity() { } #[test] -fn univmon_pyramid_weighted_metrics() { +fn univmon_two_tier_weighted_metrics() { let mut up = UnivMonTwoTier::with_defaults(); let mut truth = common::FreqTruth::default(); let stream = zipf_u64(15_000, 800, 1.3, 4004); @@ -1699,7 +1699,7 @@ fn hydra_query_frequency_is_the_frequency_query_it_wraps() { } #[test] -fn hydra_query_quantile_is_the_cumulative_query_it_wraps() { +fn hydra_query_cdf_is_the_cumulative_query_it_wraps() { let mut hydra = Hydra::with_schema( 4, 512, diff --git a/tests/e2e/frequency.rs b/tests/e2e/frequency.rs index 7b6fc1fe..a7391978 100644 --- a/tests/e2e/frequency.rs +++ b/tests/e2e/frequency.rs @@ -558,7 +558,7 @@ fn countsketch_error_stays_rank_independent_within_the_documented_empirical_band /// CountSketchWithL2Norm is a Count Sketch carrying a running F2, so its point estimates /// obey the Count Sketch L2 bound and its `l2_squared` tracks the exact F2. #[test] -fn countl2hh_weighted_turnstile_satisfies_the_l2_median_bound() { +fn count_sketch_with_l2_norm_weighted_turnstile_satisfies_the_l2_median_bound() { const ROWS: usize = 4; const COLS: usize = 2048; const HASH_SEED_IDX: usize = 11; @@ -604,7 +604,7 @@ fn countl2hh_weighted_turnstile_satisfies_the_l2_median_bound() { // ------------------------------------------- CountMinSketchFold / CountSketchFold counts + merge #[test] -fn fold_cms_foldcs_counts_and_hierarchical_merge() { +fn count_min_sketch_fold_and_count_sketch_fold_counts_and_hierarchical_merge() { // Weighted per-key counts stay exact on sparse dims. let mut a = CountMinSketchFold::::new(3, 2048, 0, 32); let mut b = CountMinSketchFold::::new(3, 2048, 0, 32); diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index 9f6f17f6..1b2dee15 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -444,7 +444,7 @@ fn a_weighted_arrival_matches_repeating_it_under_eviction() { /// `update_batch` is the loop it replaces, and a zero weight records nothing. #[test] -fn bulk_insert_matches_repeated_inserts_and_a_zero_weight_is_inert() { +fn update_batch_matches_repeated_updates_and_a_zero_weight_is_inert() { let stream = stream(); let values: Vec = stream.iter().map(|key| DataInput::I64(*key)).collect(); @@ -1437,7 +1437,7 @@ mod partial_key_and_heavy_maintenance { } #[test] - fn elastic_insert_heavy_only_never_writes_the_light_layer() { + fn elastic_update_heavy_only_never_writes_the_light_layer() { let mut sk = ElasticSketch::::with_heavy_and_light(8, 3, 256); for e in 0..64 { for _ in 0..20 { diff --git a/tests/e2e/matrix_instances.rs b/tests/e2e/matrix_instances.rs index 66f07fdc..a942dce5 100644 --- a/tests/e2e/matrix_instances.rs +++ b/tests/e2e/matrix_instances.rs @@ -609,7 +609,7 @@ macro_rules! heap_matrix_test { // (`i64`, or `f64` for an `f64` counter). `i128` has no `Into`, so those // instances construct but have no `update`, `estimate` or `merge` at all. // They are enumerated and pinned as inert by -// `cmsheap_inert_instances_construct_and_report_their_geometry`. +// `count_min_sketch_with_heap_inert_instances_construct_and_report_their_geometry`. // - `CountSketchWithHeap`'s insert path is bounded on `S::Counter: CountSketchHeapCounter`, // which `i128` *does* satisfy. All six `i128` instances are fully // operational and are covered below; the previous revision of this file and @@ -687,7 +687,7 @@ heap_matrix_test!( /// `i128` would change the heap wire payload for every family that shares it. /// This test fails if any of that silently changes. #[test] -fn csheap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { +fn count_sketch_with_heap_i128_counters_saturate_into_the_heap_instead_of_wrapping() { macro_rules! probe { ($storage:ty, $path:ty, $ctor:expr) => {{ let label = concat!( @@ -830,7 +830,7 @@ fn heap_count(items: &[asap_sketchlib::HeapEntry], key: &DataInput) -> Option {{ let label = concat!( @@ -1372,7 +1372,7 @@ fn countmin_precomputed_hash_entry_points_match_the_value_entry_points() { } #[test] -fn countmin_weighted_batch_entry_points_match_a_loop_of_single_inserts() { +fn countmin_weighted_batch_entry_points_match_a_loop_of_single_updates() { const ROWS: usize = 4; const COLS: usize = 2_048; let (stream, truth) = stream_and_truth(); diff --git a/tests/e2e/membership.rs b/tests/e2e/membership.rs index dabfcbe5..6047aa80 100644 --- a/tests/e2e/membership.rs +++ b/tests/e2e/membership.rs @@ -178,7 +178,7 @@ fn the_fill_based_estimate_tracks_the_measured_rate() { /// Inserting a key twice sets no bit the first insert did not, so the filter's /// fill is a function of the distinct keys alone. #[test] -fn repeated_inserts_leave_the_bits_unchanged() { +fn repeated_updates_leave_the_bits_unchanged() { let mut once = Bloom::::with_capacity(1_000, 0.01); let mut twice = Bloom::::with_capacity(1_000, 0.01); for key in 0..1_000i64 { @@ -681,7 +681,7 @@ fn merging_filters_of_different_slice_counts_panics() { /// `update_batch` is the loop, not a different filter. #[test] -fn bulk_insert_matches_inserting_one_at_a_time() { +fn update_batch_matches_updating_one_at_a_time() { let batch: Vec = members().into_iter().map(DataInput::I64).collect(); let mut one_by_one = Bloom::::with_capacity(MEMBERS as usize, 0.01); diff --git a/tests/e2e/nitro.rs b/tests/e2e/nitro.rs index 71d39724..ca9123d7 100644 --- a/tests/e2e/nitro.rs +++ b/tests/e2e/nitro.rs @@ -361,7 +361,7 @@ fn every_nitro_path_is_unbiased_inside_its_sampling_band_at_every_rate() { /// one row instead of all of them, which divides every per-row counter by /// the depth. #[test] -fn full_sampling_is_exact_and_the_query_reads_the_cells_the_insert_wrote() { +fn full_sampling_is_exact_and_the_query_reads_the_cells_the_update_wrote() { const F: usize = 5_000; let key = DataInput::U64(SOLO_KEY); @@ -920,7 +920,7 @@ fn counters(storage: &Vector2D) -> Vec { } #[test] -fn count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes() { +fn count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_update_writes() { let (stream, truth) = integration_stream(); let mut plain = CountMinSketch::, FastPath>::with_dimensions( INTEGRATION_ROWS, @@ -959,7 +959,7 @@ fn count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes() } #[test] -fn count_min_after_disable_nitro_leaves_the_sampled_insert_path_inert() { +fn count_min_after_disable_nitro_leaves_the_sampled_update_path_inert() { let (stream, _) = integration_stream(); let mut sketch = CountMinSketch::, FastPath>::with_dimensions( INTEGRATION_ROWS, @@ -1039,7 +1039,7 @@ fn count_min_seeded_nitro_sampling_is_reproducible_and_admits_a_strict_subset() } #[test] -fn count_sketch_at_full_nitro_sampling_writes_exactly_what_a_plain_insert_writes() { +fn count_sketch_at_full_nitro_sampling_writes_exactly_what_a_plain_update_writes() { let (stream, truth) = integration_stream(); let mut plain = CountSketch::, FastPath>::with_dimensions(INTEGRATION_ROWS, INTEGRATION_COLS); diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index cf26c390..23dc8bb3 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -155,7 +155,7 @@ fn cm_promotion_is_lossless_modulo_the_child_residual() { } #[test] -fn cm_deltas_are_well_formed_and_carry_exactly_one_promotion() { +fn count_min_sketch_cell_deltas_are_well_formed_and_carry_exactly_one_promotion() { let stream = keys(20_000, 512, 9_102); let (_, deltas) = cm_child_run(&stream, ROWS, COLS); @@ -204,7 +204,7 @@ fn cm_parent_holds_every_completed_promotion() { } #[test] -fn cm_octo_estimate_trails_the_single_thread_estimate_by_under_one_promotion() { +fn count_min_sketch_octo_estimate_trails_the_single_thread_estimate_by_under_one_promotion() { let stream = keys(40_000, 256, 9_104); let (_, deltas) = cm_child_run(&stream, ROWS, COLS); @@ -272,7 +272,7 @@ fn cm_fast_path_promotes_on_the_same_schedule_as_the_regular_path() { } #[test] -fn cm_delta_addressing_survives_columns_past_the_u16_ceiling() { +fn count_min_sketch_cell_delta_addressing_survives_columns_past_the_u16_ceiling() { // `CountMinSketchCellDelta`'s row and col fields are u32, so a geometry wider than // 65_536 columns addresses every cell exactly. let rows = 3; @@ -302,7 +302,7 @@ fn cm_delta_addressing_survives_columns_past_the_u16_ceiling() { } #[test] -fn cm_delta_application_is_order_independent() { +fn count_min_sketch_cell_delta_application_is_order_independent() { let stream = keys(15_000, 512, 9_106); let (_, deltas) = cm_child_run(&stream, ROWS, COLS); @@ -373,7 +373,7 @@ fn cm_sharded_children_conserve_counts_against_a_single_pass() { // --------------------------------------------------------------------------- #[test] -fn count_deltas_carry_exactly_the_signed_threshold() { +fn count_sketch_cell_deltas_carry_exactly_the_signed_threshold() { let stream = keys(30_000, 256, 9_201); let (_, deltas) = cs_child_run(&stream, ROWS, COLS); @@ -425,7 +425,7 @@ fn count_promotion_is_lossless_modulo_the_child_residual() { } #[test] -fn count_octo_estimate_stays_within_one_residual_of_the_single_thread_estimate() { +fn count_sketch_octo_estimate_stays_within_one_residual_of_the_single_thread_estimate() { let stream = keys(40_000, 256, 9_203); let (_, deltas) = cs_child_run(&stream, ROWS, COLS); @@ -451,7 +451,7 @@ fn count_octo_estimate_stays_within_one_residual_of_the_single_thread_estimate() } #[test] -fn count_fast_path_conserves_counts_like_the_regular_path() { +fn count_sketch_fast_path_conserves_counts_like_the_regular_path() { let stream = keys(20_000, 256, 9_204); let mut child = CsFast::with_dimensions(ROWS, COLS); @@ -481,7 +481,7 @@ fn count_fast_path_conserves_counts_like_the_regular_path() { } #[test] -fn count_delta_application_is_order_independent() { +fn count_sketch_cell_delta_application_is_order_independent() { let stream = keys(15_000, 256, 9_205); let (_, deltas) = cs_child_run(&stream, ROWS, COLS); @@ -613,7 +613,7 @@ fn hll_promotion_is_exact_for_every_variant_and_precision() { } #[test] -fn hll_deltas_are_strictly_increasing_per_register_and_never_repeat() { +fn hll_register_deltas_are_strictly_increasing_per_register_and_never_repeat() { let stream: Vec = (0..30_000u64).collect(); let (child, deltas) = hll_child_run(&stream); @@ -647,7 +647,7 @@ fn hll_duplicate_keys_promote_nothing_after_the_first_improvement() { } #[test] -fn hll_delta_application_is_order_independent() { +fn hll_register_delta_application_is_order_independent() { let stream: Vec = (0..30_000u64).collect(); let (_, deltas) = hll_child_run(&stream); @@ -915,7 +915,7 @@ mod runtime { } #[test] - fn insert_batch_matches_element_wise_inserts() { + fn update_batch_matches_element_wise_updates() { let inputs = inputs_from(&keys(20_000, 512, 9_405)); let mut one_by_one = @@ -991,7 +991,7 @@ mod runtime { } #[test] - fn a_borrowed_key_may_be_dropped_the_moment_insert_returns() { + fn a_borrowed_key_may_be_dropped_the_moment_update_returns() { // The point of a borrowed key is that its owner does not have to // outlive the sketch. `update` hashes for partitioning and prepares the // payload on this thread, so nothing borrowed crosses to a worker and @@ -1230,7 +1230,7 @@ mod runtime { #[test] #[should_panic(expected = "cannot update after runtime has been closed")] - fn inserting_after_close_panics() { + fn updating_after_close_panics() { let workers = 2; let mut runtime = OctoRuntime::new(&config(workers), SumPlan, || SumAggregator { per_worker: vec![0; workers], @@ -4422,7 +4422,8 @@ mod top_k_plans { /// bound, widened by exactly that same `k*τ` promotion residual and by /// nothing else. #[test] - fn cm_top_k_point_estimates_trail_the_single_thread_reference_by_at_most_k_tau() { + fn count_min_sketch_with_heap_point_estimates_trail_the_single_thread_reference_by_at_most_k_tau() + { let stream = stream(); let truth = truth_of(&stream); let reference = reference_cm(&stream); @@ -4512,7 +4513,7 @@ mod top_k_plans { /// two-sided `k*τ` gap against the single-threaded reference (signed /// counters can be withheld in either direction). #[test] - fn count_top_k_point_estimates_satisfy_the_l2_bound_within_the_promotion_residual() { + fn count_sketch_with_heap_point_estimates_satisfy_the_l2_bound_within_the_promotion_residual() { let stream = stream(); let truth = truth_of(&stream); let reference = reference_cs(&stream); @@ -4711,7 +4712,8 @@ mod top_k_plans { /// aggregator at all and estimates zero — which is what makes the `k*τ` /// deficit above a *deficit* rather than an unbounded miss. #[test] - fn a_key_below_the_promotion_threshold_never_reaches_the_top_k_aggregator() { + fn a_key_below_the_promotion_threshold_never_reaches_the_count_min_sketch_with_heap_aggregator() + { let plan = CountMinSketchWithHeapOctoPlan::new(ROWS, COLS); let mut worker = plan.worker(0); let mut parent = plan.aggregator(TOP_K); diff --git a/tests/e2e/quantiles.rs b/tests/e2e/quantiles.rs index 9355b65b..ea411c4d 100644 --- a/tests/e2e/quantiles.rs +++ b/tests/e2e/quantiles.rs @@ -392,7 +392,7 @@ fn bulk_cases(trial: usize, n: usize) -> Vec<(&'static str, Vec)> { /// still legal — sketch, so what is asserted here is equality: serialized /// bytes, retained count, and quantile bits over `RANK_QS`. #[test] -fn kll_bulk_update_is_byte_identical_to_the_update_loop() { +fn kll_update_batch_is_byte_identical_to_the_update_loop() { const N: usize = 20_000; for (case, (shape, values)) in bulk_cases(0, N).iter().enumerate() { let seed = kll_bulk_seed(case as u64); @@ -426,7 +426,7 @@ fn kll_bulk_update_is_byte_identical_to_the_update_loop() { /// `KllDynamic` under the same equality, on its own wire form. #[test] -fn kll_dynamic_bulk_update_is_byte_identical_to_the_update_loop() { +fn kll_dynamic_update_batch_is_byte_identical_to_the_update_loop() { const N: usize = 10_000; for (case, (shape, values)) in bulk_cases(1, N).iter().enumerate() { let seed = kll_bulk_seed(0x0100 + case as u64); @@ -462,7 +462,7 @@ fn kll_dynamic_bulk_update_is_byte_identical_to_the_update_loop() { /// byte-for-byte untouched, and a one-element `update_batch` equals a single /// `update`. #[test] -fn kll_bulk_update_on_a_degenerate_slice_matches_the_loop() { +fn kll_update_batch_on_a_degenerate_slice_matches_the_loop() { let seed = kll_bulk_seed(0x0200); let mut sk: Kll = Kll::with_k_and_seed(200, seed); for v in &[1.0f64, 2.0, 3.0] { @@ -499,7 +499,7 @@ fn kll_bulk_update_on_a_degenerate_slice_matches_the_loop() { /// The `DataInput` batch path matches the loop, and a non-numeric element /// stops it at that element with the preceding prefix already applied. #[test] -fn kll_bulk_update_data_input_matches_the_loop_and_stops_at_the_first_non_numeric() { +fn kll_update_batch_data_input_matches_the_loop_and_stops_at_the_first_non_numeric() { let seed = kll_bulk_seed(0x0300); let values = normal_f64(5_000, 100.0, 20.0, 8001); let inputs: Vec = values.iter().map(|v| DataInput::F64(*v)).collect(); diff --git a/tests/e2e/windows.rs b/tests/e2e/windows.rs index d283211e..d459b8ff 100644 --- a/tests/e2e/windows.rs +++ b/tests/e2e/windows.rs @@ -206,7 +206,7 @@ fn eh_count_sketch_variant_conforms_to_the_l2_model_over_the_retained_window() { } #[test] -fn eh_countl2hh_variant_satisfies_the_l2_bound_over_the_retained_window() { +fn eh_count_sketch_with_l2_norm_variant_satisfies_the_l2_bound_over_the_retained_window() { let (merged, truth, ctx) = run_keyed_variant(asap_sketchlib::EhBucketSketch::CountSketchWithL2Norm( CountSketchWithL2Norm::with_dimensions(ROWS, COLS), @@ -540,7 +540,7 @@ fn eh_expires_buckets_past_the_window_and_reports_its_retained_span() { /// `TumblingWindow`: window bookkeeping is exact, and the folded Count /// Sketch inside each window carries the L2 bound at its *folded* width. #[test] -fn tumbling_fold_cs_windows_are_exact_and_answers_satisfy_the_l2_bound() { +fn tumbling_count_sketch_fold_windows_are_exact_and_answers_satisfy_the_l2_bound() { const FULL_COLS: usize = 4_096; const FOLD_LEVEL: u32 = 2; const WINDOW: u64 = 500; @@ -823,7 +823,7 @@ fn eh_uniform_sampling_variant_reports_exact_retention_bookkeeping() { } #[test] -fn tumbling_fold_cms_hierarchical_merge_covers_every_observation() { +fn tumbling_count_min_sketch_fold_hierarchical_merge_covers_every_observation() { const FULL_COLS: usize = 4_096; const FOLD_LEVEL: u32 = 2; const WINDOW: u64 = 500; @@ -1011,7 +1011,7 @@ fn an_exponential_histogram_expires_against_the_window_length_it_was_last_given( } #[test] -fn an_exponential_histogram_custom_bucket_update_matches_repeated_inserts() { +fn an_exponential_histogram_custom_bucket_update_matches_repeated_updates() { let proto = EhBucketSketch::CountMinSketch( CountMinSketch::, FastPath>::with_dimensions(ROWS, COLS), ); diff --git a/tests/e2e/wire.rs b/tests/e2e/wire.rs index 8673525d..00adb208 100644 --- a/tests/e2e/wire.rs +++ b/tests/e2e/wire.rs @@ -249,7 +249,7 @@ fn an_exponential_histogram_envelope_round_trips_and_names_its_kind() { } #[test] -fn an_eh_sketch_list_envelope_round_trips_and_names_its_kind() { +fn an_eh_bucket_sketch_envelope_round_trips_and_names_its_kind() { let mut payload = eh_prototype(); for k in frequency_stream().iter().take(5_000) { payload.update(&DataInput::U64(*k)); diff --git a/tests/msgpack_compat.rs b/tests/msgpack_compat.rs index 4100d11e..45f646ce 100644 --- a/tests/msgpack_compat.rs +++ b/tests/msgpack_compat.rs @@ -136,7 +136,7 @@ fn set_aggregator_round_trip() { } #[test] -fn delta_result_round_trip() { +fn set_aggregator_delta_round_trip() { let mut added = HashSet::new(); added.insert("a".to_string()); let mut removed = HashSet::new(); diff --git a/tests/pbt/coco.rs b/tests/pbt/coco.rs index 89e2b04b..23482137 100644 --- a/tests/pbt/coco.rs +++ b/tests/pbt/coco.rs @@ -360,7 +360,7 @@ proptest! { } #[test] - fn estimate_key_reports_the_mass_of_the_bucket_that_holds_the_key( + fn estimate_reports_the_mass_of_the_bucket_that_holds_the_key( (w, d, seed, stream) in crowded_stream(300), ) { let sketch = filled(w, d, seed, &stream); diff --git a/tests/pbt/count_min.rs b/tests/pbt/count_min.rs index 2921c3d2..4ba71f32 100644 --- a/tests/pbt/count_min.rs +++ b/tests/pbt/count_min.rs @@ -272,7 +272,7 @@ proptest! { // ===== A weighted arrival must match repeating it ===== #[test] - fn count_min_weighted_matches_repeating_the_insert( + fn count_min_weighted_matches_repeating_the_update( rows in 1usize..6, cols in 1usize..64, updates in weighted(120), diff --git a/tests/pbt/count_sketch.rs b/tests/pbt/count_sketch.rs index cf142a0e..168651d7 100644 --- a/tests/pbt/count_sketch.rs +++ b/tests/pbt/count_sketch.rs @@ -117,7 +117,7 @@ proptest! { // ===== A weighted arrival must match repeating it ===== #[test] - fn count_sketch_weighted_matches_repeating_the_insert( + fn count_sketch_weighted_matches_repeating_the_update( rows in 1usize..6, cols in 1usize..64, updates in weighted(120), @@ -218,7 +218,7 @@ proptest! { } #[test] - fn a_weight_of_minus_one_undoes_a_unit_insert( + fn a_weight_of_minus_one_undoes_a_unit_update( rows in 1usize..6, cols in 1usize..8, background in keys(150), diff --git a/tests/pbt/elastic.rs b/tests/pbt/elastic.rs index befba3d0..b1f9e3f3 100644 --- a/tests/pbt/elastic.rs +++ b/tests/pbt/elastic.rs @@ -247,7 +247,7 @@ proptest! { // ===== Bucket invariants along the insert path ===== #[test] - fn the_eviction_flag_is_never_cleared_by_an_insert( + fn the_eviction_flag_is_never_cleared_by_an_update( (buckets, light_rows, light_cols, stream) in crowded(300), ) { let mut sketch = E::with_heavy_and_light(buckets, light_rows, light_cols); diff --git a/tests/pbt/ensemble.rs b/tests/pbt/ensemble.rs index 961519bd..21229612 100644 --- a/tests/pbt/ensemble.rs +++ b/tests/pbt/ensemble.rs @@ -331,7 +331,7 @@ proptest! { // ===== An insert reaches the members it names and no others ===== #[test] - fn an_insert_at_reaches_the_named_members_and_leaves_the_rest_empty( + fn an_update_at_reaches_the_named_members_and_leaves_the_rest_empty( (rows, pool) in geometry(), members in mixed_members(3), picks in prop::collection::vec(0usize..8, 0..8), diff --git a/tests/pbt/exponential_histogram.rs b/tests/pbt/exponential_histogram.rs index a448cc20..28e1dce1 100644 --- a/tests/pbt/exponential_histogram.rs +++ b/tests/pbt/exponential_histogram.rs @@ -416,7 +416,7 @@ proptest! { } #[test] - fn tumbling_fold_cms_windows_merge_back_into_the_single_pass_sketch( + fn tumbling_count_min_sketch_fold_windows_merge_back_into_the_single_pass_sketch( size in 2u64..16, max_windows in 1usize..6, stream in timed_stream(200), @@ -441,7 +441,7 @@ proptest! { } #[test] - fn tumbling_fold_cs_windows_merge_back_into_the_single_pass_sketch( + fn tumbling_count_sketch_fold_windows_merge_back_into_the_single_pass_sketch( size in 2u64..16, max_windows in 1usize..6, stream in timed_stream(200), @@ -589,7 +589,7 @@ proptest! { // answers the next window gives, not through its empty state. #[test] - fn sketch_pool_hands_back_a_fold_cms_matching_a_fresh_one( + fn sketch_pool_hands_back_a_count_min_sketch_fold_matching_a_fresh_one( stream in timed_stream(120), ) { let (first, second) = stream.split_at(stream.len() / 2); @@ -618,7 +618,7 @@ proptest! { } #[test] - fn sketch_pool_hands_back_a_fold_cs_matching_a_fresh_one( + fn sketch_pool_hands_back_a_count_sketch_fold_matching_a_fresh_one( stream in timed_stream(120), ) { let (first, second) = stream.split_at(stream.len() / 2); diff --git a/tests/pbt/octo_delta.rs b/tests/pbt/octo_delta.rs index b435f8ab..a4952082 100644 --- a/tests/pbt/octo_delta.rs +++ b/tests/pbt/octo_delta.rs @@ -507,7 +507,7 @@ proptest! { /// At a threshold of 0 a worker holds nothing back, so the shards need no /// flush and the parent is the register-wise maximum outright. #[test] - fn hll_deltas_merge_to_the_register_wise_maximum( + fn hll_register_deltas_merge_to_the_register_wise_maximum( shards in 1usize..5, keys in stream(400), ) { @@ -545,7 +545,7 @@ proptest! { } #[test] - fn hll_delta_application_is_order_independent_and_idempotent( + fn hll_register_delta_application_is_order_independent_and_idempotent( tau in 0u8..=6, keys in stream(400), seed in any::(), @@ -999,7 +999,7 @@ proptest! { // ===== The default thresholds ===== #[test] - fn every_promask_is_the_promotion_step_of_its_own_delta_type( + fn every_default_threshold_is_the_promotion_step_of_its_own_delta_type( seed in any::(), key in 0u64..1024, n in 600usize..800, @@ -1108,7 +1108,7 @@ proptest! { } #[test] - fn max_promask_is_the_widest_threshold_a_one_byte_counter_survives( + fn max_promotion_threshold_is_the_widest_a_one_byte_counter_survives( key in 0u64..1024, n in 300usize..500, ) { diff --git a/tests/pbt/space_saving.rs b/tests/pbt/space_saving.rs index e11a2fc8..75e6bb3b 100644 --- a/tests/pbt/space_saving.rs +++ b/tests/pbt/space_saving.rs @@ -349,7 +349,7 @@ proptest! { } #[test] - fn a_bulk_insert_matches_the_same_values_one_at_a_time( + fn an_update_batch_matches_the_same_values_one_at_a_time( capacity in 1usize..40, stream in keys(400), ) { diff --git a/tests/pbt/topk_wrappers.rs b/tests/pbt/topk_wrappers.rs index 9d306ecc..eec656bc 100644 --- a/tests/pbt/topk_wrappers.rs +++ b/tests/pbt/topk_wrappers.rs @@ -150,7 +150,7 @@ proptest! { // ===== A top-k wrapper must not disturb its inner matrix ===== #[test] - fn cms_heap_leaves_the_matrix_a_bare_count_min_would_build( + fn count_min_sketch_with_heap_leaves_the_matrix_a_bare_count_min_would_build( rows in 1usize..6, cols in 1usize..64, top_k in 1usize..16, @@ -167,7 +167,7 @@ proptest! { } #[test] - fn cs_heap_leaves_the_matrix_a_bare_count_sketch_would_build( + fn count_sketch_with_heap_leaves_the_matrix_a_bare_count_sketch_would_build( rows in 1usize..6, cols in 1usize..64, top_k in 1usize..16, @@ -186,7 +186,7 @@ proptest! { // ===== Wire ===== #[test] - fn cms_heap_round_trips_at_edge_geometries( + fn count_min_sketch_with_heap_round_trips_at_edge_geometries( rows in edge_rows(), cols in edge_dimension(), top_k in 0usize..16, @@ -208,7 +208,7 @@ proptest! { } #[test] - fn cs_heap_round_trips_at_edge_geometries( + fn count_sketch_with_heap_round_trips_at_edge_geometries( rows in edge_rows(), cols in edge_dimension(), top_k in 0usize..16, @@ -260,7 +260,7 @@ proptest! { /// A Count Sketch estimate is a signed median and moves either way, so the /// resident's count is pinned to the reading its own last insert took. #[test] - fn a_cs_resident_carries_the_estimate_its_last_insert_read( + fn a_cs_resident_carries_the_estimate_its_last_update_read( rows in 1usize..5, cols in 1usize..32, top_k in 1usize..12, @@ -282,7 +282,7 @@ proptest! { } #[test] - fn neither_wrapper_lets_its_heap_outgrow_top_k( + fn neither_wrapper_lets_its_heap_outgrow_heap_size( rows in 1usize..5, cols in 1usize..32, top_k in 0usize..12, @@ -468,7 +468,7 @@ proptest! { /// An OctoSketch worker addresses cells with `CountSketchWithL2Norm::cell_for_row` and its /// parent with the sketch's own insert, so the two must name one cell. #[test] - fn count_l2hh_writes_the_cells_the_shared_map_names( + fn count_sketch_with_l2_norm_writes_the_cells_the_shared_map_names( rows in 1usize..6, cols in edge_dimension(), updates in hashed_updates(64), @@ -494,7 +494,7 @@ proptest! { /// The read path resolves its own cells and signs; a key inserted alone /// comes back exactly, whatever cells the map chose for it. #[test] - fn count_l2hh_reads_back_the_cells_the_shared_map_names( + fn count_sketch_with_l2_norm_reads_back_the_cells_the_shared_map_names( rows in 1usize..6, cols in edge_dimension(), updates in hashed_updates(64), @@ -539,7 +539,7 @@ proptest! { } #[test] - fn merging_count_l2hh_adds_the_cells_and_re_derives_the_l2( + fn merging_count_sketch_with_l2_norm_adds_the_cells_and_re_derives_the_l2( rows in 1usize..6, cols in edge_dimension(), left in hashed_updates(64), diff --git a/tests/sketches_go_parity_probe.rs b/tests/sketches_go_parity_probe.rs index 67ee0a1f..6848d7fc 100644 --- a/tests/sketches_go_parity_probe.rs +++ b/tests/sketches_go_parity_probe.rs @@ -47,7 +47,7 @@ const HLL_GOLDEN_HEX: &str = include_str!("../src/sketches/testdata/hll_envelope const DDSKETCH_GOLDEN_HEX: &str = "0801728e03096214ae47e17a843f128003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000100000000000000000000000000000100000000000000000000010000000000000000010000000000000001000000000001000000000001000000000001000000010000000001000000010000010000000100000100000100000100000100010000010001000100010001000100010001000100010100010100010100010101000101010100010101010101010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000187f"; #[test] -fn sketches_count_fastpath_matches_go_count_sketch_envelope() { +fn sketches_count_sketch_fastpath_matches_go_count_sketch_envelope() { let rows = 3usize; let cols = 512usize; From 0690e34e69238ef36cde3ff6832608578485ca8b Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:02:25 +0000 Subject: [PATCH 11/11] refactor!: Coco Octo constructors take (rows, cols) CocoOctoWorker, CocoOctoAggregator and CocoOctoPlan constructors take (rows, cols), as CocoSketch::with_dimensions does; CocoSketchWorker::width()/depth() -> cols()/rows(). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- docs/api/api_coco_sketch.md | 2 +- docs/api/api_octo.md | 4 +- docs/tests.md | 12 +++--- src/sketch_framework/octo.rs | 84 +++++++++++++++++++----------------- tests/e2e/octo.rs | 12 +++--- tests/pbt/octo_delta.rs | 2 +- 7 files changed, 65 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e728db8d..0cbd1fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -177,7 +177,9 @@ signals a backwards-compatible change. constructors take `(rows, cols)`: `init_with_size(w, d)` -> `with_dimensions(rows, cols)`, `init_with_size_and_seed(w, d, seed)` -> `with_dimensions_and_seed(rows, cols, seed)`, and `CocoSketchWorker::new` / - `with_seed` swap their first two arguments the same way. + `with_seed` and the `CocoOctoWorker`, `CocoOctoAggregator` and + `CocoOctoPlan` constructors swap their first two arguments the same way. + `CocoSketchWorker::width()`/`depth()` -> `cols()`/`rows()`. - **KLL, CDF and DDSketch** (breaking). KLL `init`/`init_with_seed`/ `init_kll`/`init_kll_with_seed` -> `new`/`with_seed`/`with_k`/ `with_k_and_seed`; `wire_*` accessors -> `level_bounds_top_down()`, diff --git a/docs/api/api_coco_sketch.md b/docs/api/api_coco_sketch.md index c31b51d5..fe22065a 100644 --- a/docs/api/api_coco_sketch.md +++ b/docs/api/api_coco_sketch.md @@ -146,7 +146,7 @@ let _ = sk.estimate_projected("region=us", |full| { > **Feature gate:** requires `octo-runtime` for the runtime itself. ```rust -let plan = CocoOctoPlan::new(1024, 2); // or CocoOctoPlan::with_seed(1024, 2, seed) +let plan = CocoOctoPlan::new(2, 1024); // or CocoOctoPlan::with_seed(2, 1024, seed) let config = OctoConfig { threshold: plan.threshold().clone(), ..OctoConfig::default() }; let out = run_octo(&inputs, &config, plan.clone(), || plan.aggregator()); out.parent.sketch.estimate("flow::7"); diff --git a/docs/api/api_octo.md b/docs/api/api_octo.md index dc4aadfb..dc331927 100644 --- a/docs/api/api_octo.md +++ b/docs/api/api_octo.md @@ -542,7 +542,9 @@ heap, which is Algorithm 2. The two keyed-bucket pairs instead replay the key, per §4.4. `CocoOctoAggregator` calls the weighted `CocoSketch::update_weighted`, so the promoted mass contests the parent's own buckets and runs the parent's own `v/val` -election. Both halves of the Coco pair draw from the thread generator; +election. `CocoOctoWorker`, `CocoOctoAggregator` and `CocoOctoPlan` take +`(rows, cols)`, as `CocoSketch::with_dimensions` does. Both halves of the Coco +pair draw from the thread generator; `CocoSketchWorker::with_seed`, `CocoOctoWorker::with_threshold_and_seed`, `CocoOctoAggregator::with_seed` and `CocoOctoPlan::with_seed` / `with_threshold_and_seed` draw from a seeded one instead. A seeded plan diff --git a/docs/tests.md b/docs/tests.md index 244245c2..5468b62d 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1160,9 +1160,9 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `a_count_worker_refuses_hashes_built_for_a_different_geometry` | Hashes built for another row count panic rather than partly landing. | Verifies a `4x16` `CountSketchWorker` fed two-row hashes panics with a "one hash per row" message. | | `a_worker_refuses_hashes_built_for_a_different_geometry` | The Count-Min worker refuses the same mismatch. | Verifies a `4x16` `CountMinSketchWorker` fed two-row hashes panics with a "one hash per row" message. | | `coco_worker_promotes_the_key_its_bucket_holds_and_clears_it` | A Coco bucket promotes its `` pair and clears. | Inserts `"flow::coco"` into a `CocoSketchWorker::new(2, 64)` at `OCTO_COCO_DEFAULT_THRESHOLD`, and verifies nothing promotes over the first `OCTO_COCO_DEFAULT_THRESHOLD - 1` inserts, that the next insert emits exactly one delta carrying that key and a value of `OCTO_COCO_DEFAULT_THRESHOLD`, and that every residual counter is `0`. | -| `coco_promotion_conserves_the_inserted_mass` | Promoted mass plus residual is the whole stream, and a flush hands over the rest. | Drives 5,000 inserts over the 200 keys `flow::{i % 200}` through a `CocoSketchWorker::new(2, 16)` into a `CocoOctoAggregator::new(16, 2)` at `OCTO_COCO_DEFAULT_THRESHOLD`, verifies the parent's recorded mass plus the worker's residual equals `5_000`, and that after `flush` the parent alone holds `5_000`. | -| `a_seeded_coco_plan_replays_identically` | A seeded `CocoOctoPlan` replays to the same deltas and parent table. | Routes 20,000 inserts of `flow::{(i*i + 7i) % 512}` by parity through two workers of `CocoOctoPlan::with_seed(64, 2, 7)` into its aggregator, and verifies a second run emits the same per-worker deltas and leaves the same parent table bucket for bucket, while seed `9` emits different deltas. | -| `a_seeded_coco_plan_gives_each_worker_its_own_stream` | Each worker of a seeded plan draws from its own generator. | Feeds the same 20,000 inserts to worker `0` twice and to worker `1` of `CocoOctoPlan::with_seed(64, 2, 7)`, and verifies worker `0` reproduces its deltas and residual while worker `1` differs. | +| `coco_promotion_conserves_the_inserted_mass` | Promoted mass plus residual is the whole stream, and a flush hands over the rest. | Drives 5,000 inserts over the 200 keys `flow::{i % 200}` through a `CocoSketchWorker::new(2, 16)` into a `CocoOctoAggregator::new(2, 16)` at `OCTO_COCO_DEFAULT_THRESHOLD`, verifies the parent's recorded mass plus the worker's residual equals `5_000`, and that after `flush` the parent alone holds `5_000`. | +| `a_seeded_coco_plan_replays_identically` | A seeded `CocoOctoPlan` replays to the same deltas and parent table. | Routes 20,000 inserts of `flow::{(i*i + 7i) % 512}` by parity through two workers of `CocoOctoPlan::with_seed(2, 64, 7)` into its aggregator, and verifies a second run emits the same per-worker deltas and leaves the same parent table bucket for bucket, while seed `9` emits different deltas. | +| `a_seeded_coco_plan_gives_each_worker_its_own_stream` | Each worker of a seeded plan draws from its own generator. | Feeds the same 20,000 inserts to worker `0` twice and to worker `1` of `CocoOctoPlan::with_seed(2, 64, 7)`, and verifies worker `0` reproduces its deltas and residual while worker `1` differs. | | `elastic_worker_promotes_each_half_on_its_own_terms` | The heavy half, the light half and the keyed eviction spill all promote. | Alternates `"flow::a"` and `"flow::b"` over 1,000 inserts into an `ElasticSketchWorker::new(1, 2, 64)` at `OCTO_ELASTIC_DEFAULT_THRESHOLD` and verifies at least one `Heavy`, at least one `Light` and at least one `Evicted` delta; the counts and the delta contents are not checked. | | `threshold_for_error_follows_equation_four` | `threshold_for_error` is Equation 4, with a floor of one and a one-byte ceiling. | Verifies `threshold_for_error` returns `100` for (`0.0004`, `1_000_000.0`, `4`) and for (`0.0001`, `1_000_000.0`, `1`), `1` for (`1e-9`, `1_000.0`, `1`), and `OCTO_MAX_PROMOTION_THRESHOLD` for (`0.001`, `1_000_000.0`, `1`). | | `shared_threshold_is_visible_to_every_holder` | Every clone of an `OctoThreshold` reads and writes one value, clamped to `1..=OCTO_MAX_PROMOTION_THRESHOLD`. | For an `OctoThreshold::new(31)` and a clone of it, verifies `increase(4)` reads back as `35` on the clone, `decrease(34)` floors at `1` on the original, and `set(u32::MAX)` reads back as `OCTO_MAX_PROMOTION_THRESHOLD`. | @@ -1259,7 +1259,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `univmon_online_accuracy_beats_sketch_merge` | UnivMon delta promotion tracks the single-core answer far more closely than sketch-merge. | Runs 60,000 zipf(1.1) keys, domain `4_096`, seed `12_201`, through `UnivMonOctoPlan` at `5x1024`, `12` layers, heap `64`, tau `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` on `4` workers, against an ideal `UnivMon` and merge baselines sized to `1x` and `10x` Octo's counters, scoring the mean of the relative cardinality and entropy gaps to the ideal sketch at `8` off-boundary checkpoints, and verifies the baselines were funded, that Octo's gap is below half the parity baseline's, that the `10x` baseline beats the parity one, and that the `10x` spend exceeds `10x` Octo's counters. | | `univmon_heavy_hitter_recall_beats_sketch_merge` | The UnivMon aggregator's layer-0 heap recalls the hot keys at least as well as sketch-merge. | Runs 60,000 zipf(1.1) keys, domain `4_096`, seed `12_203`, through `UnivMonOctoPlan` at `5x1024`, `12` layers, heap `64`, tau `OCTO_COUNT_SKETCH_DEFAULT_THRESHOLD` on `4` workers against a merge baseline funded at parity, measuring at `8` off-boundary checkpoints the share of the truth's top `16` keys that `layer_heaps[0].find()` returns, and verifies Octo's mean recall is at least the baseline's and above `0.9`. | | `a_flat_threshold_starves_the_deep_univmon_layers` | One flat threshold empties UnivMon's deep layers while the scaled one keeps them exact. | Over 60,000 zipf(1.1) keys, domain `4_096`, seed `12_201`, drives a `12`-layer `5x1024` heap-`64` pyramid of `CountSketchWithL2NormWorker` workers twice - once at a flat base of `31` for every layer, once at `univmon_layer_threshold(31, layer)` - and verifies the ideal sketch has non-zero cells on layer `11`, the flat run has exactly `0` there, the scaled run has exactly as many as the ideal, and that the flat run's relative cardinality error against the ideal exceeds `0.5` while the scaled run's is below `0.1`. | -| `coco_deltas_carry_exactly_one_promotion_window_and_a_key_the_stream_used` | Every Coco delta carries exactly tau and names a key the stream contained. | Drives 40,000 zipf(1.1) keys, domain `2_048`, seed `21_001`, rendered through `flow_key_string`, into `CocoOctoPlan::with_seed(512, 2, COCO_SEED).worker(0)`, and verifies the promotion list is non-empty and that every `CocoDelta` has `value == OCTO_COCO_DEFAULT_THRESHOLD` (`31`) and a `key` present in the set of rendered stream keys. | +| `coco_deltas_carry_exactly_one_promotion_window_and_a_key_the_stream_used` | Every Coco delta carries exactly tau and names a key the stream contained. | Drives 40,000 zipf(1.1) keys, domain `2_048`, seed `21_001`, rendered through `flow_key_string`, into `CocoOctoPlan::with_seed(2, 512, COCO_SEED).worker(0)`, and verifies the promotion list is non-empty and that every `CocoDelta` has `value == OCTO_COCO_DEFAULT_THRESHOLD` (`31`) and a `key` present in the set of rendered stream keys. | | `coco_promotion_conserves_the_stream_mass_exactly` | Promoted plus held-back Coco mass is the stream, and no residual has reached tau. | Runs 60,000 zipf(1.1) keys, domain `2_048`, seed `21_002`, through `4` `CocoOctoWorker`s over a `512x2` table under `HashByKey`, and verifies the parent table's summed `val` plus the workers' one-byte residuals equals `60_000` exactly, that the residual is strictly positive, that every worker bucket holds strictly less than `31`, and that the unflushed parent's mass is strictly below `60_000`. | | `coco_flush_hands_over_every_residual_bucket` | A Coco flush ships exactly the partial counts and empties the workers. | Runs 60,000 zipf(1.1) keys, domain `2_048`, seed `21_003`, through `4` workers over a `512x2` table, then flushes and verifies the shipped deltas' values sum to exactly the pre-flush residual, that each flushed value is in `1..31`, that the workers' residual is then `0`, and that the parent's mass is exactly `60_000`. | | `coco_promotes_the_bucket_incumbent_when_the_arrival_loses_the_election` | A promotion may name the bucket's incumbent rather than the arriving key. | Over four 20,000-key zipf(1.1) streams, domain `2_048`, seeds `21_100..21_104`, through a `16x2` `CocoOctoWorker`, counts promotions whose `key` differs from the arriving key and verifies more than `1_000` promotions fired and that the incumbent share exceeds `0.05`; the assertion only rules out a worker that always names the arrival - it does not check that the incumbent is named on every losing election, and the exact share is not pinned. | @@ -1272,7 +1272,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `elastic_octo_never_underestimates_any_flow_after_a_flush` | A flushed Elastic parent never reads any flow below its true size. | Runs 60,000 zipf(1.1) keys, domain `4_096`, seed `22_004`, through `1, 2, 4, 8` workers over `128` buckets and a `3x2048` light layer under `HashByKey`, flushes, and verifies `query` is at or above the exact count for every one of the distinct rendered keys, and that at least one true flow is absent from the parent's heavy table (so the light-layer read-through path is actually exercised). | | `a_heavy_flow_that_never_spilled_is_estimated_exactly` | A heavy flow no worker ever evicted reads back exactly, not through the light layer. | Builds a `4`-bucket, `1x1` `ElasticOctoPlan`, picks a `lone` key hashing to bucket `0` and two contenders hashing to bucket `1`, sends `200` lone arrivals and `400` alternating contested ones through one worker into the aggregator, flushes, and verifies the single light cell holds non-zero noise from the contenders, that the lone flow's parent bucket has `eviction == false`, and that `query` returns exactly `200`. | | `a_resident_that_spilled_at_a_worker_reads_through_to_the_light_layer` | A flow evicted at a worker is flagged at the parent and read through the light layer. | With a `1`-bucket, `1x64` `ElasticOctoPlan`, sends `OCTO_ELASTIC_DEFAULT_THRESHOLD = 31` arrivals of `flow::resident`, one arrival of a challenger chosen to land on a different light cell, then `LAMBDA - 2 = 6` more resident arrivals, and verifies the worker shipped exactly `[Heavy { resident, 31, eviction: false }, Evicted { resident, votes: 0 }]`, and after the flush that the parent's bucket has `eviction == true` and `vote_pos == 31`, and that `query` returns exactly `31 + 6 = 37`. | -| `run_octo_coco_conserves_the_stream_mass_at_every_worker_count` | The threaded Coco pipeline conserves and partitions the stream at any worker count. | Under `--features octo-runtime`, runs 60,000 zipf(1.1) keys, domain `2_048`, seed `23_001`, through `run_octo` with `CocoOctoPlan::with_seed(512, 2, COCO_SEED)` at `1, 2, 3, 4, 8` workers and verifies per count that the parent table's mass is exactly `60_000` and that `estimate` summed over every distinct rendered key is exactly `60_000`. | +| `run_octo_coco_conserves_the_stream_mass_at_every_worker_count` | The threaded Coco pipeline conserves and partitions the stream at any worker count. | Under `--features octo-runtime`, runs 60,000 zipf(1.1) keys, domain `2_048`, seed `23_001`, through `run_octo` with `CocoOctoPlan::with_seed(2, 512, COCO_SEED)` at `1, 2, 3, 4, 8` workers and verifies per count that the parent table's mass is exactly `60_000` and that `estimate` summed over every distinct rendered key is exactly `60_000`. | | `run_octo_elastic_conserves_the_stream_mass_whatever_the_interleaving` | The threaded Elastic pipeline conserves per row and stays one-sided. | Under `--features octo-runtime`, runs 60,000 zipf(1.1) keys, domain `4_096`, seed `23_002`, through `run_octo` with `ElasticOctoPlan::new(128, 3, 2048)` at `1, 2, 3, 4, 8` workers and verifies per count that parent heavy votes plus each of the `3` light rows is exactly `60_000` and that `query` is at or above the exact count for every distinct rendered key. | | `run_octo_elastic_matches_a_single_threaded_replay_at_one_worker` | At one worker the Elastic runtime is bit-exact against a sequential replay. | Under `--features octo-runtime`, runs 40,000 zipf(1.1) keys, domain `2_048`, seed `23_003`, through `run_octo` with `ElasticOctoPlan::new(128, 3, 1024)` on `1` worker, repeated `3` times, and verifies every heavy bucket's `(flow_id, vote_pos, vote_neg, eviction)` and all `3 * 1024` light counters equal those of a flushed single-threaded `HashByKey` replay. | | `coco_online_accuracy_beats_sketch_merge` | Coco's online relative error and heavy-hitter F1 both favour delta promotion. | Over `3` streams of 200,000 zipf(1.1) keys, domain `5_000`, seeds `24_001..24_004`, drives a `4`-worker `512x2` Octo Coco replay, an ideal single-pass `CocoSketch` and a sketch-merge baseline sized to Octo's counter spend, scoring at `8` off-boundary checkpoints the mean relative error over the truth's top `32` keys and the F1 of the reported set against the `HH_ALPHA = 0.0002` heavy-hitter threshold, and verifies the baseline was not starved, that Octo's relative error is below half the baseline's, that Octo's F1 is above the baseline's, that the ideal pass leads Octo, and that Octo's relative error is below `0.10`. | @@ -1284,7 +1284,7 @@ Runtime feature: the `runtime_tests` module is `#[cfg(all(test, feature = "octo- | `hll_octo_plan_is_register_exact_and_in_band_under_both_partitions` | `HllOctoPlan` is register-exact and in cardinality band under either partition. | Drives 60,000 zipf(1.1) keys, domain `1_024`, seed `0x0C70_ACC0`, through `HllOctoPlan` on `4` workers under `HashByKey` and `RoundRobin`, flushes each worker, and per route verifies the parent's `registers_as_slice()` equals a single-threaded `HyperLogLog`'s exactly and that `estimate()` passes `CardinalityConfidenceSpec::hll(14, 4.0)` against the exact distinct count. | | `dd_octo_plan_is_bucket_exact_after_flush_and_holds_alpha_under_both_partitions` | `DdOctoPlan` is bucket-exact after a flush and holds alpha under either partition. | Turns the same 60,000-key stream into values `1.0 + key`, drives them through `DdOctoPlan::new(0.01)` on `4` workers under both routes, flushes, and per route verifies `get_count()` equals a single-pass `DdSketch`'s, that every bucket over the union of both stores' index ranges matches the reference (offset differences allowed for), and that the quantiles `0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0` all satisfy `RelativeQuantileSpec::core(0.01)` against the exact order statistics, with zero violations tolerated. | | `univmon_octo_plan_reports_the_delivered_l1_and_holds_its_f2_bound_under_both_partitions` | The UnivMon parent's L1 is exactly the delivered weight, and its F2 holds the AMS bound. | Drives the same 60,000-key stream through `UnivMonOctoPlan::new(5, 4096, 8)` with heap `64` on `4` workers under both routes, recording the largest `total_weight` each worker stamped on a delta, and per route verifies the delivered sum never exceeds `N`, that `parent.estimate_l1()` equals that sum *exactly* (`UnivMonOctoWorker` has no flush, so mass after a worker's last delta stays in flight and the printed context names how much), and that `estimate_l2().powi(2)` passes `SecondMomentSpec::new(5, 4096)`'s `sqrt(2*kappa/w)` relative band against the exact F2. | -| `coco_octo_plan_stays_unbiased_on_heavy_keys_under_both_partitions` | The Coco Octo plan conserves the stream mass and stays unbiased on the heavy keys under either partition. | Per route, drives the 60,000-key stream through `CocoOctoPlan::with_seed(128, 2, COCO_SEED + trial)` on `4` workers with a per-worker flush for 32 seeds, verifies every flushed parent holds exactly `60_000`, and requires the mean over seeds of the truth's top-32 summed `estimate` to lie within 6.5 standard errors of their exact sum, with that band under 2% of it. | +| `coco_octo_plan_stays_unbiased_on_heavy_keys_under_both_partitions` | The Coco Octo plan conserves the stream mass and stays unbiased on the heavy keys under either partition. | Per route, drives the 60,000-key stream through `CocoOctoPlan::with_seed(2, 128, COCO_SEED + trial)` on `4` workers with a per-worker flush for 32 seeds, verifies every flushed parent holds exactly `60_000`, and requires the mean over seeds of the truth's top-32 summed `estimate` to lie within 6.5 standard errors of their exact sum, with that band under 2% of it. | | `elastic_octo_plan_stays_inside_the_stream_mass_under_both_partitions` | The Elastic Octo plan keeps the heavy keys inside the stream mass under either partition. | Drives the same stream through `ElasticOctoPlan::new(512, 3, 4096)` on `4` workers under both routes with a per-worker flush, and per route verifies for the truth's top `32` keys that `query` stays inside `[0, stream mass]` for the plan and for a single-threaded reference. | ### TumblingWindow diff --git a/src/sketch_framework/octo.rs b/src/sketch_framework/octo.rs index ec6fab2c..2a8cf0df 100644 --- a/src/sketch_framework/octo.rs +++ b/src/sketch_framework/octo.rs @@ -549,12 +549,12 @@ impl CocoSketchWorker { } /// Buckets per array. - pub fn width(&self) -> usize { + pub fn cols(&self) -> usize { self.cols } /// Number of arrays. - pub fn depth(&self) -> usize { + pub fn rows(&self) -> usize { self.rows } @@ -2238,28 +2238,29 @@ pub struct CocoOctoWorker { } impl CocoOctoWorker { - /// Creates a worker with a private threshold fixed at `OCTO_COCO_DEFAULT_THRESHOLD`. - pub fn new(w: usize, d: usize) -> Self { - Self::with_threshold(w, d, OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD)) + /// Creates a `rows` x `cols` worker with a private threshold fixed at + /// `OCTO_COCO_DEFAULT_THRESHOLD`. + pub fn new(rows: usize, cols: usize) -> Self { + Self::with_threshold(rows, cols, OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD)) } /// Creates a worker sharing `threshold` with its peers and the aggregator. - pub fn with_threshold(w: usize, d: usize, threshold: OctoThreshold) -> Self { + pub fn with_threshold(rows: usize, cols: usize, threshold: OctoThreshold) -> Self { Self { - sketch: CocoSketchWorker::new(d, w), + sketch: CocoSketchWorker::new(rows, cols), threshold, } } /// As [`Self::with_threshold`], with the worker table drawing from `seed`. pub fn with_threshold_and_seed( - w: usize, - d: usize, + rows: usize, + cols: usize, threshold: OctoThreshold, seed: u64, ) -> Self { Self { - sketch: CocoSketchWorker::with_seed(d, w, seed), + sketch: CocoSketchWorker::with_seed(rows, cols, seed), threshold, } } @@ -2307,17 +2308,17 @@ pub struct CocoOctoAggregator { } impl CocoOctoAggregator { - /// Creates an aggregator with a `d` x `w` parent table. - pub fn new(w: usize, d: usize) -> Self { + /// Creates an aggregator with a `rows` x `cols` parent table. + pub fn new(rows: usize, cols: usize) -> Self { Self { - sketch: CocoSketch::with_dimensions(d, w), + sketch: CocoSketch::with_dimensions(rows, cols), } } /// As [`Self::new`], with the parent table drawing from `seed`. - pub fn with_seed(w: usize, d: usize, seed: u64) -> Self { + pub fn with_seed(rows: usize, cols: usize, seed: u64) -> Self { Self { - sketch: CocoSketch::with_dimensions_and_seed(d, w, seed), + sketch: CocoSketch::with_dimensions_and_seed(rows, cols, seed), } } } @@ -2731,23 +2732,23 @@ impl OctoPlan for DdOctoPlan { /// Builds `CocoOctoWorker`s; payloads are the rendered flow key. #[derive(Clone, Debug)] pub struct CocoOctoPlan { - w: usize, - d: usize, + rows: usize, + cols: usize, threshold: OctoThreshold, seed: Option, } impl CocoOctoPlan { - /// Creates a plan at the CocoSketch default threshold. - pub fn new(w: usize, d: usize) -> Self { - Self::with_threshold(w, d, OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD)) + /// Creates a plan for `rows` x `cols` tables at the CocoSketch default threshold. + pub fn new(rows: usize, cols: usize) -> Self { + Self::with_threshold(rows, cols, OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD)) } /// Creates a plan whose workers share `threshold`. - pub fn with_threshold(w: usize, d: usize, threshold: OctoThreshold) -> Self { + pub fn with_threshold(rows: usize, cols: usize, threshold: OctoThreshold) -> Self { Self { - w, - d, + rows, + cols, threshold, seed: None, } @@ -2755,20 +2756,25 @@ impl CocoOctoPlan { /// As [`Self::new`], with every worker and the aggregator drawing from its /// own generator derived from `seed` and its worker id. - pub fn with_seed(w: usize, d: usize, seed: u64) -> Self { - Self::with_threshold_and_seed(w, d, OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD), seed) + pub fn with_seed(rows: usize, cols: usize, seed: u64) -> Self { + Self::with_threshold_and_seed( + rows, + cols, + OctoThreshold::new(OCTO_COCO_DEFAULT_THRESHOLD), + seed, + ) } /// As [`Self::with_threshold`], seeded as [`Self::with_seed`]. pub fn with_threshold_and_seed( - w: usize, - d: usize, + rows: usize, + cols: usize, threshold: OctoThreshold, seed: u64, ) -> Self { Self { - w, - d, + rows, + cols, threshold, seed: Some(seed), } @@ -2782,8 +2788,8 @@ impl CocoOctoPlan { /// Builds the parent this plan's workers feed. See `CountMinSketchOctoPlan::aggregator`. pub fn aggregator(&self) -> CocoOctoAggregator { match self.seed { - Some(seed) => CocoOctoAggregator::with_seed(self.w, self.d, derive_seed(seed, 0)), - None => CocoOctoAggregator::new(self.w, self.d), + Some(seed) => CocoOctoAggregator::with_seed(self.rows, self.cols, derive_seed(seed, 0)), + None => CocoOctoAggregator::new(self.rows, self.cols), } } } @@ -2801,12 +2807,12 @@ impl OctoPlan for CocoOctoPlan { let threshold = self.threshold.clone(); match self.seed { Some(seed) => CocoOctoWorker::with_threshold_and_seed( - self.w, - self.d, + self.rows, + self.cols, threshold, derive_seed(seed, worker_id as u64 + 1), ), - None => CocoOctoWorker::with_threshold(self.w, self.d, threshold), + None => CocoOctoWorker::with_threshold(self.rows, self.cols, threshold), } } @@ -3343,17 +3349,17 @@ mod worker_tests { #[test] fn a_seeded_coco_plan_replays_identically() { - let (sent, table) = coco_plan_run(&CocoOctoPlan::with_seed(64, 2, 7)); + let (sent, table) = coco_plan_run(&CocoOctoPlan::with_seed(2, 64, 7)); assert_eq!( (sent.clone(), table), - coco_plan_run(&CocoOctoPlan::with_seed(64, 2, 7)) + coco_plan_run(&CocoOctoPlan::with_seed(2, 64, 7)) ); - assert_ne!(sent, coco_plan_run(&CocoOctoPlan::with_seed(64, 2, 9)).0); + assert_ne!(sent, coco_plan_run(&CocoOctoPlan::with_seed(2, 64, 9)).0); } #[test] fn a_seeded_coco_plan_gives_each_worker_its_own_stream() { - let plan = CocoOctoPlan::with_seed(64, 2, 7); + let plan = CocoOctoPlan::with_seed(2, 64, 7); let run = |id: usize| { let mut worker = plan.worker(id); let mut sent: Vec = Vec::new(); @@ -3393,7 +3399,7 @@ mod worker_tests { // Every insert lands in exactly one bucket, so what the parent received // plus what the worker still holds is the whole stream. let mut worker = CocoSketchWorker::new(2, 16); - let mut parent = CocoOctoAggregator::new(16, 2); + let mut parent = CocoOctoAggregator::new(2, 16); let inserted = 5_000u64; for i in 0..inserted { worker.update_emit_delta( diff --git a/tests/e2e/octo.rs b/tests/e2e/octo.rs index 23dc8bb3..870ff0ff 100644 --- a/tests/e2e/octo.rs +++ b/tests/e2e/octo.rs @@ -2964,7 +2964,7 @@ mod heavy_hitters { impl OctoCoco { fn new(workers: usize, w: usize, d: usize, route: Route, seed: u64) -> Self { - let plan = CocoOctoPlan::with_seed(w, d, seed); + let plan = CocoOctoPlan::with_seed(d, w, seed); Self { children: (0..workers).map(|id| plan.worker(id)).collect(), parent: plan.aggregator(), @@ -3255,7 +3255,7 @@ mod heavy_hitters { let stream = flow_stream(40_000, 2_048, 21_001); let seen: HashSet = stream.iter().map(|raw| key_of(*raw)).collect(); - let mut worker = CocoOctoPlan::with_seed(512, 2, COCO_SEED).worker(0); + let mut worker = CocoOctoPlan::with_seed(2, 512, COCO_SEED).worker(0); let mut promoted = Vec::new(); for raw in &stream { worker.process(&key_of(*raw), &mut |d: CocoDelta| promoted.push(d)); @@ -3364,7 +3364,7 @@ mod heavy_hitters { let mut incumbent = 0usize; for seed in 0..4u64 { let stream = flow_stream(20_000, 2_048, 21_100 + seed); - let mut worker = CocoOctoPlan::with_seed(16, 2, COCO_SEED + seed).worker(0); + let mut worker = CocoOctoPlan::with_seed(2, 16, COCO_SEED + seed).worker(0); for raw in &stream { let arrival = key_of(*raw); worker.process(&arrival, &mut |d: CocoDelta| { @@ -3941,9 +3941,9 @@ mod heavy_hitters { let distinct: HashSet = stream.iter().map(|raw| key_of(*raw)).collect(); for workers in [1usize, 2, 3, 4, 8] { - let plan = CocoOctoPlan::with_seed(512, 2, COCO_SEED); + let plan = CocoOctoPlan::with_seed(2, 512, COCO_SEED); let parent = run_octo(&inputs, &config(workers), plan, || { - CocoOctoAggregator::with_seed(512, 2, COCO_SEED) + CocoOctoAggregator::with_seed(2, 512, COCO_SEED) }) .parent .sketch; @@ -5088,7 +5088,7 @@ mod partition_accuracy { for route in routes() { let mut samples = Vec::with_capacity(TRIALS as usize); for trial in 0..TRIALS { - let plan = CocoOctoPlan::with_seed(COCO_W, COCO_D, COCO_SEED + trial); + let plan = CocoOctoPlan::with_seed(COCO_D, COCO_W, COCO_SEED + trial); let mut workers: Vec<_> = (0..WORKERS).map(|id| plan.worker(id)).collect(); let mut parent = plan.aggregator(); for (i, k) in stream.iter().enumerate() { diff --git a/tests/pbt/octo_delta.rs b/tests/pbt/octo_delta.rs index a4952082..90918b6f 100644 --- a/tests/pbt/octo_delta.rs +++ b/tests/pbt/octo_delta.rs @@ -621,7 +621,7 @@ proptest! { let mut workers: Vec = (0..shards) .map(|i| CocoSketchWorker::with_seed(d, w, seed.wrapping_add(i as u64 + 1))) .collect(); - let mut parent = CocoOctoAggregator::with_seed(w, d, seed); + let mut parent = CocoOctoAggregator::with_seed(d, w, seed); for (i, k) in keys.iter().enumerate() { workers[i % shards].update_emit_delta(&flow(*k), tau, &mut |delta| parent.apply(delta)); }