From 2b2678673b56dd4bd054b745377db24720bb1610 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:40:28 -0600 Subject: [PATCH 1/2] refactor(kll)!: drop the proto and portable formats KLL and KLLDynamic serialize only as ASAPv1 (kind_ids 0x06 0x00 / 0x06 0x01). - proto: reserve SketchEnvelope field 13 ("kll"); KLLState and CoinState stay in kll.proto because hydra.proto's HydraCell still embeds them. - portable: remove KllSketch/KllSketchData from the crate root, the sketchlib_kll_* helpers, SketchlibKll, the value-offset codec and KllProtoItems; portable::kll becomes a crate-private module holding only the per-cell type HydraKllSketch uses. - native: remove the MessagePackCodec shims for KLL and KLLDynamic. - KLL::from_portable_state is removed. - Tests: the portable KLL PBT laws and the e2e merge-and-wire rank test now run on KLL through serialize_to_bytes; proto/portable-only tests go. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 22 + docs/asapv1_wire_format.md | 4 +- docs/e2e_testing_harness.md | 3 +- docs/features.md | 2 +- docs/message_pack_format.md | 4 +- docs/tests.md | 33 +- examples/accuracy_probe.rs | 17 - proto/sketchlib.proto | 6 +- src/lib.rs | 7 +- src/message_pack_format/native/kll.rs | 20 - src/message_pack_format/native/kll_dynamic.rs | 21 - src/message_pack_format/native/mod.rs | 2 - src/message_pack_format/portable/kll.rs | 739 +----------------- src/message_pack_format/portable/mod.rs | 8 +- src/proto/generated/sketchlib.v1.rs | 6 +- src/sketches/kll.rs | 109 +-- src/sketches/kll/wire.rs | 7 +- tests/PBT_COVERAGE.md | 2 +- tests/asapv1_golden.rs | 4 +- tests/e2e/composition.rs | 47 +- tests/msgpack_compat.rs | 37 +- tests/pbt/kll.rs | 29 +- tests/xtest_consumer.rs | 111 --- 23 files changed, 99 insertions(+), 1141 deletions(-) delete mode 100644 src/message_pack_format/native/kll.rs delete mode 100644 src/message_pack_format/native/kll_dynamic.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 80d85f98..bc0a2911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,28 @@ signals a backwards-compatible change. as an empty window like any other, and that window takes a place in `max_windows`. +### Removed + +- **KLL's proto and portable formats.** `KLL` and `KLLDynamic` serialize only + as ASAPv1 (`serialize_to_bytes` / `deserialize_from_bytes`, kind_ids + `0x06 0x00` / `0x06 0x01`). Removed: + - the `kll` field (13) of `SketchEnvelope` (now reserved) and with it + `sketch_envelope::SketchState::Kll`; `KLLState` stays in + `proto/kll/kll.proto` only as `HydraCell.kll`; + - the crate-root `KllSketch` and `KllSketchData`, and the public module + `message_pack_format::portable::kll`: `SketchlibKll`, `new_sketchlib_kll`, + `new_sketchlib_kll_with_seed`, `sketchlib_kll_update`, + `sketchlib_kll_quantile`, `sketchlib_kll_merge`, + `bytes_from_sketchlib_kll`, `sketchlib_kll_from_bytes`, + `KllSketch::{merge_refs, aggregate_kll, from_portable_state}`, the + `MessagePackCodec` impls for `KllSketch` and `KllSketchData`, the + value-offset codec (`KLL_SCALE_SWEEP`, `encode_value_offset`, + `decode_value_offset`, `KllProtoItems`), and the `unsafe impl Send/Sync + for KllSketch`. `HydraKllSketch` keeps its per-cell type crate-internally; + - `message_pack_format::native::{kll, kll_dynamic}`, the `MessagePackCodec` + impls for `KLL` and `KLLDynamic`; + - `KLL::from_portable_state`. + ### Fixed - `KLLDynamic` panicked on every quantile query once a NaN had entered the diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index fd14e70a..c4586bb9 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -458,7 +458,7 @@ Both KLL variants (the compact fixed-buffer `KLL` and the growable `KLLDynamic`) `num_levels` is `levels.len() - 1` (derived, so not stored). The `coin` is the randomized-compaction RNG; it is carried so a decoded sketch can keep compacting deterministically (a query-only consumer may ignore it). It is a nested 3-element array mirroring `sketchlib-go`'s `CoinState`. -**Item order (cross-language contract).** `levels` / `items` use the **top-most-level-first** layout, byte-for-byte matching `sketchlib-go`'s `KLLState`: index `i` in `levels` maps to compactor level `num_levels - 1 - i`, and level 0's run is in **input order**. The compact `KLL` grows its buffer leftward and stores level 0 reverse-input, so its encoder reverses level 0 back to input order (and its decoder reverses it in); `KLLDynamic` already stores this layout natively. Within a level, order past the first compaction is not guaranteed byte-identical across the two Rust variants (or across languages), but the retained set and quantiles agree — see the caveat on `KLL::wire_items`. +**Item order (cross-language contract).** `levels` / `items` use the **top-most-level-first** layout, byte-for-byte matching `sketchlib-go`: index `i` in `levels` maps to compactor level `num_levels - 1 - i`, and level 0's run is in **input order**. The compact `KLL` grows its buffer leftward and stores level 0 reverse-input, so its encoder reverses level 0 back to input order (and its decoder reverses it in); `KLLDynamic` already stores this layout natively. Within a level, order past the first compaction is not guaranteed byte-identical across the two Rust variants (or across languages), but the retained set and quantiles agree — see the caveat on `KLL::wire_items`. **Wire widths.** `levels` is `u32` on the wire, widened to `usize` in memory. The coin's `remaining_bits` is `u32` on the wire but `u8` in memory, so rule 8 below is what makes the narrowing safe; `state` and `bit_cache` are `u64` on both sides. @@ -1360,7 +1360,7 @@ Retire `portable` once the fixtures cover the rest. - **Q-SEEDS**: `seed_list` is **inlined** in v1 so the bytes self-describe the hash. Resolving seeds from `hash_profile_id` alone is a v2 space optimization. Each sketch still carries only the seed *index* it uses. - **Q-PROFILE**: the hash-spec metadata is **derived from the hasher's `HashProfile`** (`hll_metadata::` / `cms_metadata::`) and never hardcoded, so it is always truthful to the hasher. Custom hash profiles are **supported and self-describing**. Merge compatibility is hash-spec equality, so a custom-profile sketch is not mergeable with a standard one. - **Q-CMS**: Count-Min is one `kind_id` (`0x02 0x00`); counter type and mode live in the metadata, so the id stays single. The counter-type domain is `"i32"`, `"i64"` and `"f64"`, with no `i128` (no msgpack integer form); `i32` is recorded at its own width and pinned on decode. -- **Q-KLL**: KLL metadata carries **no hash-spec group** — KLL is comparison-based and never hashes, so those fields have no truthful value. Its metadata is structural-only (`metadata_version`, `k`, `m`, `item_type`, optional `seed`) and is *not* `HashProfile`-derived. The two KLL variants (compact `0x06 0x00`, dynamic `0x06 0x01`) share one payload `[levels, items, coin]` and differ only by `kind_id`. `item_type` (`"f64"`/`"i64"`) is a metadata param, not a separate `kind_id` (mirrors Q-CMS's `counter_type`). Retained samples use the top-most-level-first layout that matches `sketchlib-go`'s `KLLState`. +- **Q-KLL**: KLL metadata carries **no hash-spec group** — KLL is comparison-based and never hashes, so those fields have no truthful value. Its metadata is structural-only (`metadata_version`, `k`, `m`, `item_type`, optional `seed`) and is *not* `HashProfile`-derived. The two KLL variants (compact `0x06 0x00`, dynamic `0x06 0x01`) share one payload `[levels, items, coin]` and differ only by `kind_id`. `item_type` (`"f64"`/`"i64"`) is a metadata param, not a separate `kind_id` (mirrors Q-CMS's `counter_type`). Retained samples use the top-most-level-first layout that matches `sketchlib-go`. - **Q-KLL-SEED**: KLL records its reproducible compaction `seed` as an **optional** metadata key. It is construction config (so metadata, not payload, per the config→metadata rule), and it is the first optional key in v1: present only when the sketch carries a seed, omitted otherwise. Rationale: the payload's `coin` already carries the RNG's *current* position (enough to resume compaction), but `seed` is what a later `clear()` re-seeds from — so serializing it lets a decoded sketch keep `clear()`-reproducibility instead of falling back to wall-clock. Cost of omitting it is bounded (only a decoded-then-`clear()`ed sketch loses cross-run byte reproducibility — never correctness), but it is cheap to carry and future-proofs the checkpoint/restore path. `KLLDynamic` has no seed concept and never emits the key; the two variants are deliberately **not** forced to be symmetric here. Go carries and preserves the key without interpreting it. - **Q-CS**: Count Sketch is one `kind_id` (`0x04 0x00`) and mirrors Count-Min's metadata and `[counts]` payload, including `counter_type`. Its type domain differs: `"i32"` and `"i64"`, with no `"f64"` — not because a float fails a bound, but because **no `CountSketchCounter` impl exists for it**: the trait covers `i32`, `i64` and `i128` and nothing else, so Count-Min's `f64` has no Count Sketch counterpart to serialize — and no `i128` (no msgpack integer form), which leaves the two `CsWireCounter` covers. Structural-param order matches Q-CMS: `... matrix_seed_index, rows, cols, counter_type, mode`. **`i32` is not widened to `i64`** — unlike Count-Min, whose counters are plain numbers, a Count Sketch appears *nested* inside `HydraCounter` and `EHSketchList` as `Vector2D` and must decode back into that variant, so the width is identity and is recorded exactly. Cells are signed, so a decoder must not assume monotonicity. - **Q-CMS-DIMS**: Count-Min `rows`/`cols` are **metadata** and the payload omits them. They are configuration that shapes the payload (like HLL's `precision`), so per the config-to-metadata rule they belong in the descriptor. The payload is then just `[counts]`. Canonical structural-param order: `... matrix_seed_index, rows, cols, counter_type, mode`. diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index 373f6040..e7848ca8 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -326,8 +326,7 @@ error against its bound. - No assertion depends on unseeded randomness — no `rand::rng()`, no wall-clock seeding, no implicit RNG inside a constructor. An unseeded sketch appears only where the assertion holds for every draw. `KLL::init_kll_with_seed`, - `KLLDynamic::init_kll_with_seed`, `KllSketch::with_seed`, - `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, + `KLLDynamic::init_kll_with_seed`, `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, `Coco::init_with_size_and_seed` and `CocoOctoPlan::with_seed` exist for this. - An estimator whose guarantee is unbiasedness gets no per-key band: its accuracy is the mean over independent seeds, within standard errors of the diff --git a/docs/features.md b/docs/features.md index 64f296a3..d3e38a8b 100644 --- a/docs/features.md +++ b/docs/features.md @@ -112,7 +112,7 @@ MessagePack (`rmp-serde`) support. **serde support** means the type derives `Ser | EHSketchList | Yes | Yes | | ExponentialHistogram | No | Yes | -Protobuf (prost): `.proto` definitions exist for CountMin, Count, HLL, DDSketch, KLL, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. +Protobuf (prost): `.proto` definitions exist for CountMin, Count, HLL, DDSketch, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. ### API Stability diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 12a2b91c..56e2da00 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -150,8 +150,8 @@ being **phased out** in favor of the per-sketch `wire.rs` + the shared `envelope.rs`. - `portable/` holds per-sketch wire types that predate the envelope. They - carry their own byte-parity goldens against `sketchlib-go` — HLL, Count-Min, - Count Sketch and KLL (both directions) — separate from `asapv1_golden/`. + carry their own byte-parity goldens against `sketchlib-go` — HLL, Count-Min + and Count Sketch — separate from `asapv1_golden/`. - `native/` is a set of thin `MessagePackCodec` shims over the sketches' `serialize_to_bytes` / `deserialize_from_bytes`. Every type it wraps emits the ASAPv1 envelope, so each shim is a pass-through. diff --git a/docs/tests.md b/docs/tests.md index ccb1571c..ed4984b7 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -149,7 +149,6 @@ Wire tests: [`src/sketches/kll/wire.rs`](../src/sketches/kll/wire.rs) | `cdf_cached_matches_uncached_across_lifecycle` | The memoized CDF agrees with the from-scratch rebuild at every lifecycle stage. | For `KLL::::init_with_seed(200, 8, 42)`, verifies `quantile_cached` equals `quantile` while empty at `{0.1, 0.5, 0.9}`, at `{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}` after 20,000 uniform samples in `[0, 1000)` (seed `0xACAC_0001`), and at `{0.1, 0.5, 0.9}` after merging a seed-43 sketch over 5,000 samples in `[-5000, -4000)` (seed `0xACAC_0002`); after `clear`, `quantile_cached(0.5)` is `0.0` and `count` is `0`. | | `cdf_cache_invalidated_by_subsequent_updates` | Updates after a cached query invalidate the cache instead of serving a stale CDF. | Takes `quantile_cached(0.5)` on a `init_with_seed(200, 8, 42)` sketch over 10,000 uniform samples in `[0, 100)` (seed `0xACAC_0003`), then inserts `999_999.0` 10,000 times and verifies the cached median rises by more than `50.0`. | | `kllwire_serde_rejects_crafted_dimensions` | The nested serde decoder refuses an out-of-range `k` before it sizes anything. | Encodes a `KLLWire` with no items, `levels` of `[0, 0]`, `k` of `usize::MAX`, `m` of `8`, `num_levels` `1`, and `Coin::from_seed(1)`, and verifies it does not decode as a `KLL` rather than overflowing `compute_max_capacity`. | -| `from_portable_state_reproduces_source_exactly` | Reconstruction from portable state is bit-exact, not a lossy replay. | Extracts the compacted `items` and `levels` from a `KLL::::init_kll(200)` sketch over 200,000 updates of `i * 0.0007 + 3.0`, and verifies `from_portable_state` reproduces the source's quantiles at `{0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1}` exactly; an empty reconstruction is verified to report `num_levels` `1` with `levels[0]` at `max_capacity`. | | `seeded_sketches_are_byte_identical` | Same seed and same input give byte-identical sketches. | Feeds 5,000 uniform samples in `[0, 1_000_000)` (seed `7`) to two `init_kll_with_seed(200, 42)` sketches and verifies their serialized bytes are equal. | | `different_seeds_produce_different_bytes` | The seed reaches `toss` rather than sitting unused on the struct. | Feeds the same 5,000 uniform samples in `[0, 1_000_000)` (seed `11`) to `init_kll_with_seed(200, 1)` and `init_kll_with_seed(200, 2)` and verifies their serialized bytes differ. | | `clear_preserves_seed_determinism` | `clear` re-seeds from the stored seed, so determinism survives a window rotation. | Burns 1,500 unrelated samples (seed `99`) through a `init_kll_with_seed(200, 1234)` sketch, clears it, then feeds it and a fresh seed-1234 twin the same 3,000 samples (seed `13`) and verifies their serialized bytes are equal. | @@ -1371,28 +1370,6 @@ Test file: [`src/message_pack_format/portable/hydra_kll.rs`](../src/message_pack | `test_aggregate_hydrakll` | `aggregate_hydrakll` builds a decodable matrix from parallel key and value arrays. | For `aggregate_hydrakll(2, 5, 200, ["a", "b", "a"], [1.0, 2.0, 3.0])`, verifies the returned bytes decode and report `rows` `2` and `cols` `5`; no quantile is checked. | | `test_aggregate_hydrakll_empty` | An empty stream produces nothing. | Verifies `aggregate_hydrakll(2, 5, 200, &[], &[])` returns `None`. | -### Portable KllSketch - -Test file: [`src/message_pack_format/portable/kll.rs`](../src/message_pack_format/portable/kll.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `kll_data_msgpack_matches_kll_sketch` | The DTO codec is byte-identical to the facade's. | For a `k` `200` sketch fed the 500 values `0.0..=499.0`, verifies `KllSketch::to_msgpack` equals `KllSketchData { k, sketch_bytes }::to_msgpack` and that the DTO round-trips with `k` and `sketch_bytes` unchanged. | -| `test_kll_creation` | A fresh sketch is empty and keeps its `k`. | For `KllSketch::new(200)`, verifies `count()` is `0` and `k` is `200`. | -| `test_kll_update` | Every insert is counted. | Feeds `10.0`, `20.0`, and `15.0` into a `k` `200` sketch and verifies `count()` is `3`. | -| `test_kll_quantile` | The extremes are exact and the median brackets the true one. | For a `k` `200` sketch fed `1.0..=10.0`, verifies `quantile(0.0)` is `1.0`, `quantile(1.0)` is `10.0`, and the median falls in `5.0..=6.0`. | -| `test_kll_merge` | Merge conserves the count and the extremes. | Merges a `k` `200` sketch fed `6.0..=10.0` into one fed `1.0..=5.0` and verifies `count()` is `10`, `quantile(0.0)` is `1.0`, and `quantile(1.0)` is `10.0`. | -| `test_msgpack_round_trip` | The round trip preserves `k`, the count, and the extremes. | Round-trips a `k` `200` sketch fed `1.0..=5.0` and verifies the decode reports `k` `200`, `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | -| `test_aggregate_kll` | `aggregate_kll` builds a decodable sketch from a value array. | For `aggregate_kll(200, [1.0, 2.0, 3.0, 4.0, 5.0])`, verifies the returned bytes decode to a sketch with `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | -| `test_aggregate_kll_empty` | An empty value array produces nothing. | Verifies `aggregate_kll(200, &[])` returns `None`. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The raw-`f64` proto envelope is byte-identical to the golden. | For `KLL::init_kll_with_seed(200, 42)` fed `1.0..=50.0`, builds a `KllState` from the `wire_k`/`wire_m`/`wire_num_levels`/`wire_levels`/`wire_items` accessors plus the `CoinState` from `wire_coin()`, leaving `offset` `0.0`, `value_scale` `0`, and `residuals` empty, wraps it in a `SketchEnvelope` with `format_version` `1`, and verifies the prost encoding equals a 423-byte inline golden. | -| `value_offset_round_trip_exact_fixed_point` | The fixed-point form round-trips integer and fixed-decimal series bit-exactly. | Verifies the 200 values `1_000_000 + 7i` encode at `value_scale` `0` and decode back equal, and that the 128 values `100.0 + 0.001i` encode at a scale in `-3..=0` and decode back equal. | -| `value_offset_guard_rejects_irrational` | A value with no exact decimal scale falls back to raw `f64`. | Verifies `encode_value_offset` returns `None` for `[0.0, 1.0, PI, 3.0]`, for the empty slice, for `[1.0, NAN]`, and for `[1.0, INFINITY]`. | -| `proto_dual_read_round_trip_both_forms` | The decoder reads the raw-`f64` and the value-offset form to the same answer. | For `KLL::init_kll_with_seed(200, 7)` fed `1.0..=5000.0`, builds one `KllState` carrying `wire_items()` and one carrying the `encode_value_offset` triple with `items` empty, and verifies `KllProtoItems::from_state` yields identical items and identical quantiles at `0.0`, `0.1`, `0.25`, `0.5`, `0.75`, `0.9`, and `1.0`. | -| `proto_rejects_both_forms_populated` | A state carrying both forms at once is refused. | Verifies a `KllState` with `k` `200`, `m` `8`, `num_levels` `1`, `levels` `[0, 2]`, `items` `[1.0, 2.0]`, and `residuals` `[1, 2]` fails to decode. | -| `golden_value_offset_envelope_from_go` | The value-offset envelope the Go side emits decodes to the right samples. | Decodes an 80-byte golden and verifies the `KllState` leaves `items` empty, carries `value_scale` `0`, `offset` `1.0`, and 50 residuals, and that the dual-read recovers `1.0..=50.0` with `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | -| `golden_value_offset_envelope_for_go` | The value-offset envelope this encoder emits is pinned byte for byte. | For `1.0..=50.0`, verifies `encode_value_offset` gives `offset` `1.0` and `value_scale` `0`, that the `SketchEnvelope` around a `KllState` with `k` `200`, `m` `8`, `num_levels` `1`, `levels` `[0, 50]`, and no coin encodes to the 76-byte `RUST_GOLDEN_HEX`, and that the dual-read recovers `1.0..=50.0`. | - ### Portable Sampling Rescale Test file: [`src/message_pack_format/portable/sampling.rs`](../src/message_pack_format/portable/sampling.rs) @@ -1950,7 +1927,7 @@ Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) | `univmonq_with_window_bound_chooses_a_hierarchy_that_satisfies_its_own_inequality` | `with_window_bound` picks the smallest level count whose deepest stratum still fits the candidate table. | For `max_updates` in `10_000`, `1_000_000` and `100_000_000` at `delta = 1e-3`, calls `UnivMonQConfig::default().with_window_bound` and re-derives the Bernstein upper tail `mean + sqrt(2*mean*ln(1/delta)) + (2/3)*ln(1/delta)` with `mean = max_updates / 2^(levels-1)`, asserting it is strictly below `cfg.candidates`; asserts minimality by requiring the same bound at `levels - 1` to be at or above `candidates` whenever `levels > 2`; asserts `levels` is non-decreasing across the three windows; builds each chosen config and asserts `count() == 1_000` after `1_000` updates; and asserts `with_window_bound(1_000, 1.5)` is an error. | | `univmonq_multi_shard_merge_with_distinct_source_ids_covers_the_union` | Shards with distinct source ids merge into a sketch whose exact aggregates cover the union. | Builds `4` shards with `UnivMonQ::with_hasher_and_source_id(UnivMonQConfig::default(), i + 1)`, deals `40_000` uniform values over `1_000_000` (seed `0xC090_5101`) round robin by `i % 4`, asserts each shard's `source_id()` is `i + 1`, merges the other three into the first with `merge` returning `Ok`, and asserts the merged `count()` is `40_000` and `min()`/`max()` equal the whole stream's exact extremes. Only the distinct-id path is exercised: no assertion covers a merge of shards that share a source id, and no estimator is checked. | -### Composition: Portable Sketch-With-Heap Facades +### Composition: Merge and Wire Round Trips Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) @@ -1958,7 +1935,7 @@ Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) | --- | --- | --- | | `portable_count_min_with_heap_satisfies_the_count_min_bound_through_merge_and_wire` | The portable Count-Min-plus-heap holds Count-Min's bound and an agreeing heap through a merge and a MessagePack round trip. | Feeds `40_000` Zipf(1.1) keys over a `2_048` domain (seed `0xC090_5101`) as `"k{key}"` strings into `CountMinSketchWithHeap::new(3, 4_096, 32)`, plus even/odd halves into two more. Runs `CountMinSpec::new(3, 4_096).assert_contract` on the single sketch, asserts every `topk_heap_items()` entry's `value` equals `estimate(&item.key)` with zero violations, re-runs the contract on `CountMinSketchWithHeap::merge_refs(&[&left, &right])` against the full truth, and after `to_msgpack`/`from_msgpack` asserts the decoded sketch's `estimate` is exactly equal to the original's for every observed key and still satisfies the contract. | | `portable_count_sketch_with_heap_satisfies_the_l2_bound_through_merge_and_wire` | The portable Count-Sketch-plus-heap holds the L2 bound and an agreeing heap through a merge and a MessagePack round trip. | Feeds the same `40_000`-key Zipf(1.1) stream (domain `2_048`, seed `0xC090_5101`) as `"k{key}"` strings into `CountSketchWithHeap::new(5, 4_096, 32)`, plus even/odd halves into two more, and runs `CountSketchSpec::new(5, 4_096).assert_contract` (simultaneous L2 band at `delta = 1e-3` with zero violations, marginal `sqrt(3/4096)*\|\|f_-i\|\|_2` rate pinned at `P[Bin(5, 1/3) >= 3] = 51/243`) on the single sketch, on `merge_refs(&[&left, &right])` and on the `from_msgpack` decode; every `topk_heap_items()` entry's `value` must equal `estimate(&item.key)`. Unlike the Count-Min twin, the wire round trip is only re-checked against the band - no per-key equality across the encode/decode is asserted. | -| `portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_and_wire` | The portable KLL facade holds the rank-error characterization single-pass and after a two-shard merge, and its wire round trip is bit-for-bit. | Over `12` compaction seeds `0x5EED_0400 + t * 0x9E37_79B9_7F4A_7C15`, builds `KllSketch::with_seed(200, seed)` plus even/odd shards at `seed ^ 0xAAAA` and `seed ^ 0x5555`, feeds all of them `40_000` uniform values over `1_000_000` (seed `0xC090_5101`), and merges the shards with `merge`. Each seed records two trials - the single-pass sketch and the merged pair - of the worst normalized rank error over the grid `[0.1, 0.25, 0.5, 0.75, 0.9]` against `KllRankSpec::datasketches(200)`'s `eps = 2.446/200^0.9433 = 0.0165`, and the `24` outcomes are accepted by `assert_independent_binomial` at `p = 0.01`. Per seed it also asserts `to_msgpack`/`from_msgpack` preserves `k() == 200`, `count()`, and every grid quantile exactly, and that merging a `k = 400` sketch into the single-pass one errs. | +| `kll_satisfies_the_rank_error_characterization_through_merge_and_wire` | KLL holds the rank-error characterization single-pass and after a two-shard merge, and its ASAPv1 round trip is bit-for-bit. | Over `12` compaction seeds `0x5EED_0400 + t * 0x9E37_79B9_7F4A_7C15`, builds `KLL::::init_kll_with_seed(200, seed)` plus even/odd shards at `seed ^ 0xAAAA` and `seed ^ 0x5555`, feeds all of them `40_000` uniform values over `1_000_000` (seed `0xC090_5101`), and merges the shards with `merge`. Each seed records two trials - the single-pass sketch and the merged pair - of the worst normalized rank error over the grid `[0.1, 0.25, 0.5, 0.75, 0.9]` against `KllRankSpec::datasketches(200)`'s `eps = 2.446/200^0.9433 = 0.0165`, and the `24` outcomes are accepted by `assert_independent_binomial` at `p = 0.01`. Per seed it also asserts `serialize_to_bytes`/`deserialize_from_bytes` preserves `k() == 200`, `count()`, and every grid quantile exactly. | ### Composition: Documented Input Matrix @@ -2151,9 +2128,7 @@ Test file: [`tests/msgpack_compat.rs`](../tests/msgpack_compat.rs) | `hll_sketch_round_trip` | A portable `HllSketch` survives a msgpack round trip with its register count intact. | Updates `HllSketch::new(HllVariant::Regular, 8)` with `b"a"`, `b"b"` and `b"c"`, then verifies only that the decode's `registers` has the same length as the source's; the register values, the precision and the variant are not checked. | | `hydra_kll_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | | `hydra_kll_sketch_round_trip` | A portable `HydraKllSketch` survives a msgpack round trip with its grid shape intact. | Updates `HydraKllSketch::with_seed(2, 4, 200, 0x5EED_0900)` with `("a", 1.0)`, `("a", 2.0)` and `("b", 3.0)`, then verifies the decode reports `rows` `2` and `cols` `4`; the nested KLL payloads are not checked. | -| `hydra_kll_wire_shape` | The `HydraKllSketchWire` DTO round-trips through `rmp_serde` with its two levels of nesting intact. | Serializes a wire struct of `rows: 2`, `cols: 3` and a `2x3` grid of `KllSketchData { k: 200, sketch_bytes: vec![] }`, then verifies the `rmp_serde` decode reports `rows` `2`, `cols` `3`, an outer length of `2` and an inner length of `3`. | -| `kll_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `kll_sketch_round_trip` | A portable `KllSketch` survives a msgpack round trip with its `k` and retained mass intact. | Feeds `KllSketch::with_seed(200, 0x5EED_0800)` the values `0..100` as `f64`, then verifies the decode reports `k` `200` and `count()` `100`; the retained items and the quantile answers are not checked. | +| `hydra_kll_wire_shape` | The `HydraKllSketchWire` DTO decodes a `HydraKllSketch`'s msgpack with its two levels of nesting intact. | Encodes `HydraKllSketch::new(2, 3, 200)` with `to_msgpack`, decodes the bytes as a `HydraKllSketchWire` through `rmp_serde`, and verifies it reports `rows` `2`, `cols` `3`, an outer length of `2` and an inner length of `3`. | | `set_aggregator_round_trip` | A `SetAggregator` survives a msgpack round trip with its members. | Updates a `SetAggregator::new()` with `web` and `api`, then verifies the decode holds two values, one of which is `web`. | ### Conformance Kit @@ -2202,7 +2177,7 @@ Test file: [`tests/xtest_consumer.rs`](../tests/xtest_consumer.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes nine `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `hll.pb` an ErtlMLE cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | +| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes eight `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `ddsketch.pb` p50 within 2% of `5000` and p99 within 2% of `9900`, `hll.pb` an ErtlMLE cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | ### HLL Custom Precision diff --git a/examples/accuracy_probe.rs b/examples/accuracy_probe.rs index 643bfc3e..62b3249b 100644 --- a/examples/accuracy_probe.rs +++ b/examples/accuracy_probe.rs @@ -11,7 +11,6 @@ use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketc use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; -use asap_sketchlib::message_pack_format::portable::kll::KllSketch as PortableKll; use asap_sketchlib::{ CMSHeap, CSHeap, Count as CoreCount, CountL2HH, CountMin, DDSketch, DataInput, Hydra, HyperLogLog, KLL, UnivMon, @@ -1058,22 +1057,6 @@ fn probe_portable_wire_types() { rel_err(est, 100_000.0) < 0.03, ); - // Portable KLL. - let mut pk = PortableKll::new(200); - let mut rng = StdRng::seed_from_u64(55); - let mut vals: Vec = (0..50_000).map(|_| rng.random::() * 1e6).collect(); - vals.sort_by(|a, b| a.partial_cmp(b).unwrap()); - for v in &vals { - pk.update(*v); - } - let med = pk.quantile(0.5); - let t = vals[25_000]; - p.check( - "portable KLL median", - format!("expected ~{t:.0}, got {med:.0}"), - (med - t).abs() / t < 0.05, - ); - // HydraKllSketch: per-key quantiles, median across rows. let mut hk = HydraKllSketch::new(3, 256, 200); let mut per_key: HashMap<&str, Vec> = HashMap::new(); diff --git a/proto/sketchlib.proto b/proto/sketchlib.proto index 0a6d8193..6c618327 100644 --- a/proto/sketchlib.proto +++ b/proto/sketchlib.proto @@ -32,7 +32,6 @@ import "common/common.proto"; import "countminsketch/countminsketch.proto"; import "countsketch/countsketch.proto"; import "hll/hll.proto"; -import "kll/kll.proto"; import "ddsketch/ddsketch.proto"; import "univmon/univmon.proto"; import "hydra/hydra.proto"; @@ -69,7 +68,6 @@ message SketchEnvelope { CountMinState count_min = 10; CountSketchState count_sketch = 11; HyperLogLogState hll = 12; - KLLState kll = 13; DDSketchState ddsketch = 14; UnivMonState univmon = 15; HydraState hydra = 16; @@ -77,6 +75,8 @@ message SketchEnvelope { ElasticState elastic = 18; } + reserved 13; + reserved "kll"; reserved 19 to 31; // Reserved for future sketch types. // Geometric / NitroSketch sampling probability for the producing sketch. @@ -86,7 +86,7 @@ message SketchEnvelope { // and stores the RAW SAMPLED sketch state — never the rescaled state. The // consumer applies the `× 1/sample_p` rescale at QUERY time on the count-like // estimators (HLL cardinality, CountMin / CountSketch frequency, SUM / COUNT); - // quantile estimators (KLL, DDSketch) are scale-invariant under uniform + // quantile estimators (DDSketch) are scale-invariant under uniform // sampling so they carry `sample_p` but need no rescale. // // Lives on the envelope (not per-sketch state) so downstream consumers that diff --git a/src/lib.rs b/src/lib.rs index b062b8b9..06303512 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,9 +10,9 @@ //! UnivMon, tumbling windows, and batch/parallel execution helpers. //! - [`message_pack_format`]: MessagePack/proto wire format shared with //! `sketchlib-go`. Owns the wire-DTO sketch types consumed by the ASAP -//! query engine (`CountMinSketch`, `CountSketch`, `KllSketch`, `HllSketch`, -//! `DdSketch`, `HydraKllSketch`, `CountMinSketchWithHeap`, -//! `SetAggregator`, `DeltaResult`). +//! query engine (`CountMinSketch`, `CountSketch`, `HllSketch`, `DdSketch`, +//! `HydraKllSketch`, `CountMinSketchWithHeap`, `SetAggregator`, +//! `DeltaResult`). //! //! Most users can start with the crate-root re-exports such as [`DataInput`], //! [`CountMin`], [`HyperLogLog`], [`KLL`], and [`DDSketch`]. Reach for the @@ -51,7 +51,6 @@ pub use message_pack_format::portable::ddsketch::{DDSKETCH_GROW_CHUNK, DdSketch, pub use message_pack_format::portable::delta_set_aggregator::DeltaResult; pub use message_pack_format::portable::hll::{HllSketch, HllSketchDelta, HllVariant}; pub use message_pack_format::portable::hydra_kll::HydraKllSketch; -pub use message_pack_format::portable::kll::{KllSketch, KllSketchData}; pub use message_pack_format::portable::sampling::{ effective_sample_p, is_quantile_scale_invariant, rescale_count, rescale_count_with_env, sample_p_or_default, diff --git a/src/message_pack_format/native/kll.rs b/src/message_pack_format/native/kll.rs deleted file mode 100644 index e56487b7..00000000 --- a/src/message_pack_format/native/kll.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::kll::KLL`]. - -use serde::{Deserialize, Serialize}; - -use crate::common::numerical::NumericalValue; -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::kll::{KLL, KllWireItem}; - -impl MessagePackCodec for KLL -where - T: NumericalValue + KllWireItem + Serialize + for<'de> Deserialize<'de>, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.serialize_to_bytes()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::deserialize_from_bytes(bytes)?) - } -} diff --git a/src/message_pack_format/native/kll_dynamic.rs b/src/message_pack_format/native/kll_dynamic.rs deleted file mode 100644 index 7818a494..00000000 --- a/src/message_pack_format/native/kll_dynamic.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::kll_dynamic::KLLDynamic`]. - -use serde::{Deserialize, Serialize}; - -use crate::common::numerical::NumericalValue; -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::kll::KllWireItem; -use crate::sketches::kll_dynamic::KLLDynamic; - -impl MessagePackCodec for KLLDynamic -where - T: NumericalValue + KllWireItem + Serialize + for<'de> Deserialize<'de>, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.serialize_to_bytes()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::deserialize_from_bytes(bytes)?) - } -} diff --git a/src/message_pack_format/native/mod.rs b/src/message_pack_format/native/mod.rs index 0a83ef96..ad4d083b 100644 --- a/src/message_pack_format/native/mod.rs +++ b/src/message_pack_format/native/mod.rs @@ -12,7 +12,5 @@ pub mod countsketch; pub mod countsketch_topk; pub mod ddsketch; pub mod hll; -pub mod kll; -pub mod kll_dynamic; #[cfg(feature = "experimental")] pub mod kmv; diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index 11fa2ea2..96216b33 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -1,97 +1,39 @@ -//! Wire-format-aligned KLL sketch type for cross-language interop. -//! -//! `KllSketch` is a thin facade over the pure-Rust [`crate::sketches::kll::KLL`] -//! (parity already locked in by the existing -//! `test_update_then_envelope_matches_sketchlib_go_bytes` in this file -//! against `sketchlib-go::KLLSketch.SerializePortable`). The wire shape -//! also serves as a nested field for -//! [`crate::message_pack_format::portable::hydra_kll`]. +//! The per-cell KLL type of [`crate::message_pack_format::portable::hydra_kll`]. use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; use crate::sketches::kll::KLL; -/// Concrete KLL type backing the wire-format `KllSketch`. -pub type SketchlibKll = KLL; - -/// Creates a fresh sketchlib KLL sketch with the requested accuracy -/// parameter `k`. -pub fn new_sketchlib_kll(k: u16) -> SketchlibKll { - KLL::init_kll(k as i32) -} - -/// [`new_sketchlib_kll`] with an explicit compaction-coin seed, so the sketch -/// replays identically across runs. -pub fn new_sketchlib_kll_with_seed(k: u16, seed: u64) -> SketchlibKll { - KLL::init_kll_with_seed(k as i32, seed) -} - -/// Updates a sketchlib KLL with one numeric observation. -pub fn sketchlib_kll_update(inner: &mut SketchlibKll, value: f64) { - inner.update(&value); -} - -/// Queries a sketchlib KLL for the value at the requested quantile. -pub fn sketchlib_kll_quantile(inner: &SketchlibKll, q: f64) -> f64 { - inner.quantile(q) -} - -/// Merges `src` into `dst`. -pub fn sketchlib_kll_merge(dst: &mut SketchlibKll, src: &SketchlibKll) { - dst.merge(src); -} - -/// Serializes a sketchlib KLL into MessagePack bytes. -pub fn bytes_from_sketchlib_kll(inner: &SketchlibKll) -> Vec { - inner.serialize_to_bytes().unwrap() -} - -/// Deserializes a sketchlib KLL from MessagePack bytes. -pub fn sketchlib_kll_from_bytes(bytes: &[u8]) -> Result> { - Ok(KLL::deserialize_from_bytes(bytes)?) -} - -/// Wire-format KLL sketch — a thin facade over [`SketchlibKll`] with a -/// fixed `k` parameter and msgpack codec via [`KllSketchData`]. +/// One `HydraKllSketch` cell: a `KLL` with its `k`. +#[derive(Clone)] pub struct KllSketch { pub k: u16, - pub(crate) backend: SketchlibKll, + pub(crate) backend: KLL, } impl KllSketch { pub fn new(k: u16) -> Self { Self { k, - backend: new_sketchlib_kll(k), + backend: KLL::init_kll(k as i32), } } /// [`KllSketch::new`] with an explicit compaction-coin seed. - /// - /// `new` seeds the coin from the wall clock, so two runs over the same - /// input produce different sketches. Pass a seed when the result has to be - /// reproducible — replaying a stream, comparing two processes, or - /// asserting an accuracy bound in a test. pub fn with_seed(k: u16, seed: u64) -> Self { Self { k, - backend: new_sketchlib_kll_with_seed(k, seed), + backend: KLL::init_kll_with_seed(k as i32, seed), } } - /// The configured `k` parameter (compactor capacity). - pub fn k(&self) -> u16 { - self.k - } - - /// Returns the raw sketch bytes (for JSON serialization, etc.). + /// The cell's ASAPv1 KLL bytes. pub fn sketch_bytes(&self) -> Vec { - bytes_from_sketchlib_kll(&self.backend) + self.backend.serialize_to_bytes().unwrap() } pub fn update(&mut self, value: f64) { - sketchlib_kll_update(&mut self.backend, value); + self.backend.update(&value); } pub fn count(&self) -> u64 { @@ -103,11 +45,11 @@ impl KllSketch { if self.count() == 0 { return 0.0; } - sketchlib_kll_quantile(&self.backend, q) + self.backend.quantile(q) } - /// Merge another KllSketch into self in place. Both operands must - /// have identical `k`. + /// Merge another cell into self in place. Both operands must have + /// identical `k`. pub fn merge( &mut self, other: &KllSketch, @@ -115,71 +57,9 @@ impl KllSketch { if self.k != other.k { return Err(format!("KllSketch k mismatch: self={}, other={}", self.k, other.k).into()); } - sketchlib_kll_merge(&mut self.backend, &other.backend); + self.backend.merge(&other.backend); Ok(()) } - - /// Merge from references without cloning. - pub fn merge_refs( - sketches: &[&Self], - ) -> Result> { - if sketches.is_empty() { - return Err("No sketches to merge".into()); - } - let k = sketches[0].k; - for s in sketches { - if s.k != k { - return Err("Cannot merge KllSketch with different k values".into()); - } - } - let mut merged = Self::new(k); - for s in sketches { - sketchlib_kll_merge(&mut merged.backend, &s.backend); - } - Ok(merged) - } - - /// One-shot aggregation: build a sketch from a slice of values. - pub fn aggregate_kll(k: u16, values: &[f64]) -> Option> { - if values.is_empty() { - return None; - } - let mut sketch = Self::new(k); - for &value in values { - sketch.update(value); - } - sketch.to_msgpack().ok() - } -} - -impl Clone for KllSketch { - fn clone(&self) -> Self { - // Structural clone: the backing `KLL` derives `Clone` (its fields - // are `Box<[..]>`/`Vec`/scalars), so copy it directly instead of a full - // msgpack serialize -> deserialize -> rebuild round-trip. Produces an - // identical sketch far more cheaply (no rmp encode/decode, far fewer - // allocations). - Self { - k: self.k, - backend: self.backend.clone(), - } - } -} - -impl KllSketch { - /// Reconstruct directly from portable wire state (k + level-ordered items + - /// level boundaries) without replaying items through `update()`. Bit-exact. - pub fn from_portable_state( - k: u16, - items: &[f64], - levels: &[usize], - num_levels: usize, - ) -> Result { - Ok(Self { - k, - backend: SketchlibKll::from_portable_state(k as usize, items, levels, num_levels)?, - }) - } } impl std::fmt::Debug for KllSketch { @@ -191,601 +71,10 @@ impl std::fmt::Debug for KllSketch { } } -// Thread safety: sketchlib's KLL is used in a single-threaded context per -// accumulator instance and only shares read-only operations across threads. -unsafe impl Send for KllSketch {} -unsafe impl Sync for KllSketch {} - -// ----- Value-offset fixed-point representation (cross-language) ----- -// -// KLL `KLLState` (proto/kll/kll.proto) carries the retained samples in one of -// two forms: -// -// * RAW F64 — `items` (field 5), 8 bytes/sample. The original v1 form. -// * VALUE-OFFSET — `offset` (7) + `value_scale` (8) + `residuals` (9), -// where `value = residual * 10^value_scale + offset`. -// -// PR2 makes BOTH languages read both forms (this is the decode-first half of -// the rollout). `sketchlib-go` gates emission of the value-offset form behind -// an exactness guard; this crate currently only reads it but the -// encode/decode helpers below are symmetric so the Rust side can produce it -// too (used by the cross-language golden test and any future Rust producer). - -/// Per-sketch scale exponents tried by [`encode_value_offset`], in priority -/// order. Mirrors `kllScaleSweep` in `sketchlib-go`. -pub const KLL_SCALE_SWEEP: [i32; 7] = [0, -1, -2, -3, -4, -5, -6]; - -/// Reconstructs retained samples from the value-offset fixed-point fields: -/// `value = residual * 10^value_scale + offset`. Caller invokes this only when -/// `residuals` is non-empty. -pub fn decode_value_offset(offset: f64, value_scale: i32, residuals: &[i64]) -> Vec { - let mul = 10f64.powi(value_scale); - residuals.iter().map(|&r| r as f64 * mul + offset).collect() -} - -/// Attempts to represent `items` exactly as `residual * 10^scale + offset` -/// using the finite [`KLL_SCALE_SWEEP`]. `offset` is the minimum value. Returns -/// `None` (the exactness-guard fallback) when no candidate scale round-trips -/// every sample bit-exactly, or when the slice is empty / contains a non-finite -/// value. Symmetric with `sketchlib-go::encodeValueOffset`. -pub fn encode_value_offset(items: &[f64]) -> Option<(f64, i32, Vec)> { - if items.is_empty() { - return None; - } - let mut offset = items[0]; - for &v in items { - if !v.is_finite() { - return None; - } - if v < offset { - offset = v; - } - } - for &sc in &KLL_SCALE_SWEEP { - let mul = 10f64.powi(-sc); - let mut out = Vec::with_capacity(items.len()); - let mut good = true; - for &v in items { - let r = ((v - offset) * mul).round(); - if !r.is_finite() || r > i64::MAX as f64 || r < i64::MIN as f64 { - good = false; - break; - } - let ri = r as i64; - // Exactness guard: reconstruct and require bit-identical recovery. - let recon = ri as f64 * 10f64.powi(sc) + offset; - if recon != v { - good = false; - break; - } - out.push(ri); - } - if good { - return Some((offset, sc, out)); - } - } - None -} - -/// Materialized retained samples + level layout decoded from a proto -/// `KLLState`, dual-reading the raw-f64 and value-offset forms. -#[derive(Debug, Clone)] -pub struct KllProtoItems { - /// Retained samples in level order (length == `levels[num_levels]`). - pub items: Vec, - /// Level boundary indices, length == `num_levels + 1`. - pub levels: Vec, - pub num_levels: usize, - pub k: u32, - pub m: u32, -} - -impl KllProtoItems { - /// Decodes a proto `KLLState`, dual-reading: when `residuals` is non-empty - /// it reconstructs from (offset, value_scale, residuals); otherwise it - /// reads raw f64 `items`. Returns an error if both forms are populated or - /// the level layout is inconsistent. - pub fn from_state( - s: &crate::proto::sketchlib::KllState, - ) -> Result> { - let items = if !s.residuals.is_empty() { - if !s.items.is_empty() { - return Err("kll: both residuals and raw items[] set".into()); - } - decode_value_offset(s.offset, s.value_scale, &s.residuals) - } else { - s.items.clone() - }; - let levels: Vec = s.levels.iter().map(|&v| v as usize).collect(); - if levels.len() < 2 { - return Err(format!("kll: invalid levels length {}", levels.len()).into()); - } - if *levels.last().unwrap() != items.len() { - return Err("kll: invalid item layout".into()); - } - Ok(Self { - items, - levels, - num_levels: s.num_levels as usize, - k: s.k, - m: s.m, - }) - } - - /// Weighted (value, weight) pairs across all levels. - pub fn weighted_samples(&self) -> Vec<(f64, u64)> { - let mut out = Vec::with_capacity(self.items.len()); - for h in 0..self.num_levels { - let weight: u64 = 1 << h; - let idx = self.num_levels - 1 - h; - if idx + 1 >= self.levels.len() { - continue; - } - let start = self.levels[idx]; - let end = self.levels[idx + 1]; - for &v in &self.items[start..end] { - out.push((v, weight)); - } - } - out - } - - /// Estimated value at quantile `q ∈ [0, 1]` from the weighted samples. - pub fn quantile(&self, q: f64) -> f64 { - let mut samples = self.weighted_samples(); - if samples.is_empty() { - return 0.0; - } - samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - let total: u64 = samples.iter().map(|(_, w)| w).sum(); - let target = (q * total as f64).ceil() as u64; - let mut acc = 0u64; - for (v, w) in &samples { - acc += w; - if acc >= target { - return *v; - } - } - samples.last().unwrap().0 - } -} - -// ----- Wire format ----- - -/// Wire DTO for [`KllSketch`]. Also referenced as a nested field by +/// Wire DTO for one cell of /// [`crate::message_pack_format::portable::hydra_kll::HydraKllSketchWire`]. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct KllSketchData { pub k: u16, pub sketch_bytes: Vec, } - -impl MessagePackCodec for KllSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wire = KllSketchData { - k: self.k, - sketch_bytes: self.sketch_bytes(), - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wire: KllSketchData = rmp_serde::from_slice(bytes)?; - let backend = KLL::deserialize_from_bytes(&wire.sketch_bytes)?; - Ok(Self { k: wire.k, backend }) - } -} - -impl MessagePackCodec for KllSketchData { - /// Encodes the `(k, sketch_bytes)` DTO directly, byte-identical to - /// [`KllSketch::to_msgpack`] (which serializes this same struct). Lets - /// producers that hold a raw sketchlib KLL backend (via - /// `serialize_to_bytes`) emit the portable KLL msgpack form without - /// constructing a [`KllSketch`] facade. - fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn kll_data_msgpack_matches_kll_sketch() { - let mut sk = KllSketch::new(200); - for i in 0..500 { - sk.update(i as f64); - } - let via_sketch = sk.to_msgpack().unwrap(); - let data = KllSketchData { - k: sk.k(), - sketch_bytes: sk.sketch_bytes(), - }; - let via_data = data.to_msgpack().unwrap(); - assert_eq!( - via_sketch, via_data, - "KllSketchData codec must byte-match KllSketch" - ); - let rt = KllSketchData::from_msgpack(&via_data).unwrap(); - assert_eq!(rt.k, data.k); - assert_eq!(rt.sketch_bytes, data.sketch_bytes); - } - - #[test] - fn test_kll_creation() { - let kll = KllSketch::new(200); - assert_eq!(kll.count(), 0); - assert_eq!(kll.k, 200); - } - - #[test] - fn test_kll_update() { - let mut kll = KllSketch::new(200); - kll.update(10.0); - kll.update(20.0); - kll.update(15.0); - assert_eq!(kll.count(), 3); - } - - #[test] - fn test_kll_quantile() { - let mut kll = KllSketch::new(200); - for i in 1..=10 { - kll.update(i as f64); - } - assert_eq!(kll.quantile(0.0), 1.0); - assert_eq!(kll.quantile(1.0), 10.0); - let median = kll.quantile(0.5); - assert!( - (5.0..=6.0).contains(&median), - "median should be between 5 and 6; got {median}" - ); - } - - #[test] - fn test_kll_merge() { - let mut kll1 = KllSketch::new(200); - let mut kll2 = KllSketch::new(200); - - for i in 1..=5 { - kll1.update(i as f64); - } - for i in 6..=10 { - kll2.update(i as f64); - } - - kll1.merge(&kll2).unwrap(); - assert_eq!(kll1.count(), 10); - assert_eq!(kll1.quantile(0.0), 1.0); - assert_eq!(kll1.quantile(1.0), 10.0); - } - - #[test] - fn test_msgpack_round_trip() { - let mut kll = KllSketch::new(200); - for i in 1..=5 { - kll.update(i as f64); - } - - let bytes = kll.to_msgpack().unwrap(); - let deserialized = KllSketch::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.k, 200); - assert_eq!(deserialized.count(), 5); - assert_eq!(deserialized.quantile(0.0), 1.0); - assert_eq!(deserialized.quantile(1.0), 5.0); - } - - #[test] - fn test_aggregate_kll() { - let values = [1.0, 2.0, 3.0, 4.0, 5.0]; - let bytes = KllSketch::aggregate_kll(200, &values).unwrap(); - let kll = KllSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(kll.count(), 5); - assert_eq!(kll.quantile(0.0), 1.0); - assert_eq!(kll.quantile(1.0), 5.0); - } - - #[test] - fn test_aggregate_kll_empty() { - assert!(KllSketch::aggregate_kll(200, &[]).is_none()); - } - - /// Cross-language byte-parity guard against `sketchlib-go`'s - /// `KLLSketch.SerializePortable` output for the deterministic - /// input `(1..=50)` with `seed=42` and `k=200`. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - CoinState, KllState, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let mut sk: KLL = KLL::init_kll_with_seed(200, 42); - for i in 1..=50 { - sk.update(&(i as f64)); - } - - let (state, bit_cache, remaining_bits) = sk.wire_coin(); - let kll_state = KllState { - k: sk.wire_k(), - m: sk.wire_m(), - num_levels: sk.wire_num_levels(), - levels: sk.wire_levels(), - // This golden pins the RAW-F64 wire form; the value-offset fields - // stay at their proto defaults so the bytes are unchanged. - items: sk.wire_items(), - coin: Some(CoinState { - state, - bit_cache, - remaining_bits, - }), - offset: 0.0, - value_scale: 0, - residuals: vec![], - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Kll(kll_state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - const GOLDEN_HEX: &str = "08016aa20308c80110081801220200322a9003000000000000f03f000000000000004000000000000008400000000000001040000000000000144000000000000018400000000000001c40000000000000204000000000000022400000000000002440000000000000264000000000000028400000000000002a400000000000002c400000000000002e4000000000000030400000000000003140000000000000324000000000000033400000000000003440000000000000354000000000000036400000000000003740000000000000384000000000000039400000000000003a400000000000003b400000000000003c400000000000003d400000000000003e400000000000003f4000000000000040400000000000804040000000000000414000000000008041400000000000004240000000000080424000000000000043400000000000804340000000000000444000000000008044400000000000004540000000000080454000000000000046400000000000804640000000000000474000000000008047400000000000004840000000000080484000000000000049403202082a"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "KLL envelope bytes diverge from sketchlib-go golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - fn decode_hex(s: &str) -> Vec { - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - // ----- Value-offset fixed-point representation ----- - - use crate::proto::sketchlib::{KllState, SketchEnvelope, sketch_envelope::SketchState}; - use prost::Message; - - /// Round-trip exactness for the value-offset (fixed-point) form: integer - /// and fixed-decimal series must encode then decode back bit-exactly. - #[test] - fn value_offset_round_trip_exact_fixed_point() { - // Integer-ish series (scale 0). - let ints: Vec = (0..200).map(|i| (1_000_000 + i * 7) as f64).collect(); - let (offset, scale, residuals) = - encode_value_offset(&ints).expect("integer series must be representable"); - assert_eq!(scale, 0, "integer series should pick scale 0"); - let back = decode_value_offset(offset, scale, &residuals); - assert_eq!(ints, back, "fixed-point round-trip changed integer items"); - - // Fixed-decimal series (milli resolution → scale -3). - let decis: Vec = (0..128).map(|i| 100.0 + (i as f64) * 0.001).collect(); - let (o2, s2, r2) = - encode_value_offset(&decis).expect("milli-decimal series must be representable"); - assert!( - (-3..=0).contains(&s2), - "expected a small negative scale, got {s2}" - ); - let back2 = decode_value_offset(o2, s2, &r2); - assert_eq!(decis, back2, "fixed-point round-trip changed decimal items"); - } - - /// The exactness guard must reject a value not representable at any - /// candidate scale, so the caller falls back to raw f64. - #[test] - fn value_offset_guard_rejects_irrational() { - let items = vec![0.0, 1.0, std::f64::consts::PI, 3.0]; - assert!( - encode_value_offset(&items).is_none(), - "pi is not exactly representable at any decimal scale; guard must reject" - ); - // Empty slice falls back too. - assert!(encode_value_offset(&[]).is_none()); - // Non-finite values are rejected. - assert!(encode_value_offset(&[1.0, f64::NAN]).is_none()); - assert!(encode_value_offset(&[1.0, f64::INFINITY]).is_none()); - } - - /// `KllProtoItems::from_state` must dual-read both the raw-f64 and the - /// value-offset forms and yield identical items + quantiles. - #[test] - fn proto_dual_read_round_trip_both_forms() { - let mut sk: KLL = KLL::init_kll_with_seed(200, 7); - for i in 1..=5000 { - sk.update(&(i as f64)); - } - let raw_items = sk.wire_items(); - let levels = sk.wire_levels(); - let num_levels = sk.wire_num_levels(); - - // Raw-f64 form. - let raw_state = KllState { - k: sk.wire_k(), - m: sk.wire_m(), - num_levels, - levels: levels.clone(), - items: raw_items.clone(), - coin: None, - offset: 0.0, - value_scale: 0, - residuals: vec![], - }; - let from_raw = KllProtoItems::from_state(&raw_state).expect("decode raw-f64"); - - // Value-offset form (items empty, residuals populated). - let (offset, scale, residuals) = - encode_value_offset(&raw_items).expect("integer items representable"); - let fp_state = KllState { - k: sk.wire_k(), - m: sk.wire_m(), - num_levels, - levels, - items: vec![], - coin: None, - offset, - value_scale: scale, - residuals, - }; - let from_fp = KllProtoItems::from_state(&fp_state).expect("decode value-offset"); - - assert_eq!( - from_raw.items, from_fp.items, - "raw-f64 and value-offset decode to different items" - ); - for &q in &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0] { - assert_eq!( - from_raw.quantile(q), - from_fp.quantile(q), - "quantile mismatch at p={q} between the two decode paths" - ); - } - } - - /// Decoding must error if a malformed state sets BOTH raw items and - /// residuals. - #[test] - fn proto_rejects_both_forms_populated() { - let state = KllState { - k: 200, - m: 8, - num_levels: 1, - levels: vec![0, 2], - items: vec![1.0, 2.0], - coin: None, - offset: 0.0, - value_scale: 0, - residuals: vec![1, 2], - }; - assert!(KllProtoItems::from_state(&state).is_err()); - } - - /// Cross-language golden: decode the exact envelope bytes that - /// `sketchlib-go` emits for the deterministic `(1..=50)` integer input via - /// its value-offset encoder, and confirm the retained samples + quantiles - /// match. The Go side produces this with `SerializePortable` (fixed-point, - /// scale 0, offset = 1.0). See `sketches/KLL/portable_test.go:: - /// TestGoldenValueOffsetEnvelopeForRust` in sketchlib-go, which prints this - /// hex and asserts the identical bytes. - #[test] - fn golden_value_offset_envelope_from_go() { - // SketchEnvelope { format_version: 1, kll: KLLState { k: 200, m: 8, - // num_levels: 1, levels: [0, 50], offset: 1.0, value_scale: 0, - // residuals: [0..=49] } }, producer/hash_spec cleared. - const GOLDEN_HEX: &str = GO_VALUE_OFFSET_GOLDEN_HEX; - let bytes = decode_hex(GOLDEN_HEX); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode envelope"); - let state = match env.sketch_state { - Some(SketchState::Kll(s)) => s, - _ => panic!("envelope did not contain a KLLState"), - }; - assert!( - state.items.is_empty(), - "value-offset form must leave items[] empty" - ); - assert_eq!(state.value_scale, 0); - assert_eq!(state.offset, 1.0); - assert_eq!(state.residuals.len(), 50); - - let decoded = KllProtoItems::from_state(&state).expect("dual-read decode"); - let expected: Vec = (1..=50).map(|i| i as f64).collect(); - assert_eq!(decoded.items, expected, "value-offset decode != (1..=50)"); - // Quantile sanity: with k=200 and 50 inputs, no compaction occurred so - // these are exact order statistics. - assert_eq!(decoded.quantile(0.0), 1.0); - assert_eq!(decoded.quantile(1.0), 50.0); - } - - /// Reverse-direction cross-language golden: build the value-offset envelope - /// for (1..=50) with the Rust encoder and assert exact bytes. The hex - /// printed here is consumed by sketchlib-go `TestRustGoldenDecodesInGo`. - /// Run with `-- --nocapture` to (re)print when regenerating. - #[test] - fn golden_value_offset_envelope_for_go() { - let items: Vec = (1..=50).map(|i| i as f64).collect(); - let (offset, value_scale, residuals) = - encode_value_offset(&items).expect("integers representable"); - assert_eq!(offset, 1.0); - assert_eq!(value_scale, 0); - - let state = KllState { - k: 200, - m: 8, - num_levels: 1, - levels: vec![0, 50], - items: vec![], - // No coin: the reverse-decode in Go only checks the retained set, - // and an absent coin keeps the fixture minimal/stable. - coin: None, - offset, - value_scale, - residuals, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Kll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - let got_hex: String = got.iter().map(|b| format!("{b:02x}")).collect(); - println!( - "RUST_VALUE_OFFSET_GOLDEN_HEX ({} bytes):\n{got_hex}", - got.len() - ); - - const RUST_GOLDEN_HEX: &str = "08016a4808c801100818012202003239000000000000f03f4a3200020406080a0c0e10121416181a1c1e20222426282a2c2e30323436383a3c3e40424446484a4c4e50525456585a5c5e6062"; - assert_eq!( - got_hex, RUST_GOLDEN_HEX, - "Rust value-offset envelope bytes drifted; update both this constant \ - and sketchlib-go's RustValueOffsetGoldenHex" - ); - - // Self-consistency: our own dual-reader recovers (1..=50). - let decoded = KllProtoItems::from_state(match &envelope.sketch_state { - Some(SketchState::Kll(s)) => s, - _ => unreachable!(), - }) - .expect("decode"); - assert_eq!(decoded.items, items); - } - - // Captured from sketchlib-go's value-offset `SerializePortable` for input - // (1..=50), k=200, producer/hash_spec cleared. Generated and asserted by - // sketchlib-go `TestGoldenValueOffsetEnvelopeForRust` (run with `-v` to - // re-print). 80 bytes: SketchEnvelope{ format_version:1, kll: KLLState{ - // k:200, m:8, num_levels:1, levels:[0,50], coin:{state:42}, offset:1.0, - // value_scale:0, residuals:[0..=49] } }. - const GO_VALUE_OFFSET_GOLDEN_HEX: &str = "08016a4c08c80110081801220200323202082a39000000000000f03f4a3200020406080a0c0e10121416181a1c1e20222426282a2c2e30323436383a3c3e40424446484a4c4e50525456585a5c5e6062"; -} diff --git a/src/message_pack_format/portable/mod.rs b/src/message_pack_format/portable/mod.rs index 97e0d372..db33e47b 100644 --- a/src/message_pack_format/portable/mod.rs +++ b/src/message_pack_format/portable/mod.rs @@ -2,9 +2,9 @@ //! //! Each per-algorithm submodule holds one sketch's pre-envelope wire type. //! These carry a cross-language contract of their own, on goldens apart -//! from ASAPv1's `asapv1_golden/`: HLL, Count-Min, Count Sketch and KLL pin -//! byte parity against Go's serializers, KLL in both directions; DDSketch's -//! golden is a Rust self-pin, not yet reconciled. New code uses ASAPv1: +//! from ASAPv1's `asapv1_golden/`: HLL, Count-Min and Count Sketch pin byte +//! parity against Go's serializers; DDSketch's golden is a Rust self-pin, not +//! yet reconciled. New code uses ASAPv1: //! prefer a sketch's own `serialize_to_bytes`; do not add callers here. pub mod countminsketch; @@ -15,6 +15,6 @@ pub mod ddsketch; pub mod delta_set_aggregator; pub mod hll; pub mod hydra_kll; -pub mod kll; +pub(crate) mod kll; pub mod sampling; pub mod set_aggregator; diff --git a/src/proto/generated/sketchlib.v1.rs b/src/proto/generated/sketchlib.v1.rs index 2c098d3f..413043cf 100644 --- a/src/proto/generated/sketchlib.v1.rs +++ b/src/proto/generated/sketchlib.v1.rs @@ -877,7 +877,7 @@ pub struct SketchEnvelope { /// and stores the RAW SAMPLED sketch state — never the rescaled state. The /// consumer applies the `× 1/sample_p` rescale at QUERY time on the count-like /// estimators (HLL cardinality, CountMin / CountSketch frequency, SUM / COUNT); - /// quantile estimators (KLL, DDSketch) are scale-invariant under uniform + /// quantile estimators (DDSketch) are scale-invariant under uniform /// sampling so they carry `sample_p` but need no rescale. /// /// Lives on the envelope (not per-sketch state) so downstream consumers that @@ -892,7 +892,7 @@ pub struct SketchEnvelope { /// The sketch payload. Exactly one field must be set. #[prost( oneof = "sketch_envelope::SketchState", - tags = "10, 11, 12, 13, 14, 15, 16, 17, 18" + tags = "10, 11, 12, 14, 15, 16, 17, 18" )] pub sketch_state: ::core::option::Option, } @@ -907,8 +907,6 @@ pub mod sketch_envelope { CountSketch(super::CountSketchState), #[prost(message, tag = "12")] Hll(super::HyperLogLogState), - #[prost(message, tag = "13")] - Kll(super::KllState), #[prost(message, tag = "14")] Ddsketch(super::DdSketchState), #[prost(message, tag = "15")] diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index a19a3d30..59a764c5 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -329,66 +329,6 @@ impl KLL { s } - /// Reconstruct a KLL directly from portable wire state — the retained - /// `items` in level order plus the `levels` boundary array (proto contract: - /// `levels[0] == 0`, `levels[num_levels] == items.len()`). Avoids replaying - /// every item through `update()`: it places the items straight into the - /// internal buffer and fixes up the level boundaries, so the result is - /// bit-identical to the source sketch (identical quantiles) at a fraction - /// of the cost. Errors if the supplied state is inconsistent. - pub fn from_portable_state( - k: usize, - items: &[T], - levels: &[usize], - num_levels: usize, - ) -> Result { - let mut s = Self::init_kll(k as i32); - if items.is_empty() { - return Ok(s); - } - if num_levels == 0 || num_levels >= s.levels.len() { - return Err(format!( - "from_portable_state: invalid num_levels {num_levels}" - )); - } - if levels.len() != num_levels + 1 { - return Err(format!( - "from_portable_state: levels.len()={} != num_levels+1={}", - levels.len(), - num_levels + 1 - )); - } - if levels[0] != 0 || levels[num_levels] != items.len() { - return Err(format!( - "from_portable_state: level bounds [{}..{}] inconsistent with items.len()={}", - levels[0], - levels[num_levels], - items.len() - )); - } - if items.len() > s.max_capacity { - return Err(format!( - "from_portable_state: {} items exceed max_capacity {} for k={}", - items.len(), - s.max_capacity, - k - )); - } - // Live data occupies the high end of the buffer; free space at the front. - let offset = s.max_capacity - items.len(); - let max_cap = s.max_capacity; - s.items[offset..].clone_from_slice(items); - for (dst, &src) in s.levels.iter_mut().zip(levels[..=num_levels].iter()) { - *dst = src + offset; - } - for dst in s.levels.iter_mut().skip(num_levels + 1) { - *dst = max_cap; - } - s.num_levels = num_levels; - s.rebuild_capacity_cache(); // recomputes top_height + level0_capacity for num_levels - Ok(s) - } - /// Hot-path insert: decrement `levels[0]`, write item, check capacity. #[inline] fn push_value(&mut self, value: T) { @@ -785,12 +725,8 @@ impl KLL { self.num_levels as u32 } - /// Returns the compaction-coin state in `sketchlib-go::CoinState` - /// shape: `(state, bit_cache, remaining_bits)`. Wire-format wrappers - /// pack this directly into the `KLLState.coin` proto field; without - /// it the coin state would have to be poked from private internals - /// or reconstructed by a serde round-trip, neither of which is - /// stable across `asap_sketchlib` releases. + /// Returns the compaction-coin state as `(state, bit_cache, + /// remaining_bits)`, the ASAPv1 payload's `coin` array. #[inline] pub fn wire_coin(&self) -> (u64, u64, u32) { ( @@ -804,11 +740,9 @@ impl KLL { /// length `num_levels + 1`, starting at 0 and ending at the total /// number of retained items. The chunk /// `wire_items()[wire_levels()[i] .. wire_levels()[i+1]]` is the - /// **top-most-first** run for the proto's `KLLState.levels`/`items` - /// fields, matching `sketchlib-go::KLLSketch.SerializePortable`. - /// See the `KLLState` docstring in `proto/kll/kll.proto`: index `i` - /// in `levels` maps to compactor level `num_levels - 1 - i` in the - /// in-memory representation. + /// **top-most-first** run of the ASAPv1 payload's `levels`/`items`: + /// index `i` in `levels` maps to compactor level `num_levels - 1 - i` + /// in the in-memory representation. pub fn wire_levels(&self) -> Vec { // Walk from top compactor-level downward, accumulating sizes. let n = self.num_levels; @@ -1184,39 +1118,6 @@ mod tests { ); } - // Direct reconstruction from portable wire state must be BIT-EXACT: - // identical quantiles to the source sketch (unlike a lossy - // replay-through-`update()` reconstruction). - #[test] - fn from_portable_state_reproduces_source_exactly() { - let k = 200usize; - let mut src = KLL::::init_kll(k as i32); - for i in 0..200_000u32 { - src.update(&((i as f64) * 0.0007 + 3.0)); - } - // Extract the compacted portable form (proto contract: levels[0]==0, - // levels[num_levels]==items.len()). - let off = src.levels[0]; - let items: Vec = src.items[off..src.max_capacity].to_vec(); - let levels: Vec = (0..=src.num_levels).map(|h| src.levels[h] - off).collect(); - let n = src.num_levels; - - // (A) bit-exact: direct reconstruction reproduces the source exactly. - let rebuilt = KLL::::from_portable_state(k, &items, &levels, n).unwrap(); - for &q in &[0.0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0] { - assert_eq!( - rebuilt.quantile(q), - src.quantile(q), - "direct reconstruction quantile mismatch at q={q}" - ); - } - - // An empty sketch round-trips to an empty sketch. - let empty = KLL::::from_portable_state(k, &[], &[], 0).unwrap(); - assert_eq!(empty.num_levels, 1); - assert_eq!(empty.levels[0], empty.max_capacity); - } - // Ensure each 64-bit chunk is consumed bit-by-bit before refilling. #[test] fn coin_bit_cache_behavior() { diff --git a/src/sketches/kll/wire.rs b/src/sketches/kll/wire.rs index 63e4a3f9..30deba8a 100644 --- a/src/sketches/kll/wire.rs +++ b/src/sketches/kll/wire.rs @@ -20,9 +20,8 @@ //! ## Payload item order (cross-language contract) //! //! The payload's `levels` / `items` use the **top-most-level-first** layout, -//! byte-for-byte matching `sketchlib-go`'s `KLLState` (index `i` in `levels` -//! maps to compactor level `num_levels - 1 - i`; level 0's run is in input -//! order). For the compact KLL this is exactly what [`KLL::wire_levels`] / +//! byte-for-byte matching `sketchlib-go` (index `i` in `levels` maps to +//! compactor level `num_levels - 1 - i`; level 0's run is in input order). For the compact KLL this is exactly what [`KLL::wire_levels`] / //! [`KLL::wire_items`] emit (they reverse the leftward-grown L0 buffer back to //! input order); decode inverts that mapping. @@ -229,7 +228,7 @@ where { /// Serializes the sketch into an ASAPv1 MessagePack envelope /// (kind_id `0x06 0x00`). The retained samples use the top-most-level-first - /// layout that matches `sketchlib-go`'s `KLLState`. + /// layout that matches `sketchlib-go`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { let metadata = rmp_serde::to_vec_named(&kll_metadata::(self.k as u32, self.m as u32, self.seed))?; diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index cddee44f..07df409c 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -296,7 +296,7 @@ Compaction is randomized, so no law names retained items. Compared against: the - same seed, same stream: same items - `clear` re-seeds from the stored seed: replay == fresh, items and levels - wire - - msgpack keeps `k`, count, quantiles + - ASAPv1 keeps `k`, count, quantiles - ASAPv1 keeps count and quantiles under NaN, ±inf, ±0, f64 extremes ## kll_dynamic.rs — 14 diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 30b41651..186d0993 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -245,9 +245,7 @@ fn cs_i32_regular_2x4_matches_golden() { // --------------------------------------------------------------------------- // KLL: build known state (k=200, seed 42, integers 1..=50 — below the level-0 // capacity, so no compaction fires and the retained set is deterministic) -> -// serialize == golden, and golden round-trips. Matches the deterministic -// scenario the proto parity test uses (sketchlib-go's KLLSketch over the same -// input), so the coin state (42) lines up cross-language. +// serialize == golden, and golden round-trips. // --------------------------------------------------------------------------- /// A k=200 KLL over `1..=50` with a fixed compaction seed: fully deterministic. diff --git a/tests/e2e/composition.rs b/tests/e2e/composition.rs index 23e42c41..1aebd99e 100644 --- a/tests/e2e/composition.rs +++ b/tests/e2e/composition.rs @@ -17,7 +17,7 @@ use common::{FreqTruth, NumericTruth, uniform_u64, zipf_u64}; use asap_sketchlib::{ Classic, Count, CountMin, CountMinSketchWithHeap, CountSketchWithHeap, DataInput, - EnsembleSketch, ErtlMLE, FastPath, HashSketchEnsemble, HyperLogLog, HyperLogLogHIP, KllSketch, + EnsembleSketch, ErtlMLE, FastPath, HashSketchEnsemble, HyperLogLog, HyperLogLogHIP, KLL, MessagePackCodec, UnivMonQ, UnivMonQConfig, Vector2D, }; @@ -697,16 +697,15 @@ fn portable_count_sketch_with_heap_satisfies_the_l2_bound_through_merge_and_wire ); } -/// The portable KLL facade under the DataSketches maximum-rank-error -/// characterization, seeded so a failure reproduces. `KllSketch::new` seeds -/// from the wall clock; `with_seed` is what an accuracy test must use. +/// KLL under the DataSketches maximum-rank-error characterization through a +/// two-shard merge and an ASAPv1 round trip, seeded so a failure reproduces. /// /// Trial unit is one sketch, scored on its worst rank error over the `q` grid, /// with an independent compaction seed per trial. The post-wire sketch is *not* /// a separate trial — a round trip that preserves every answer bit for bit, as /// asserted below, gives literally the same numbers. #[test] -fn portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_and_wire() { +fn kll_satisfies_the_rank_error_characterization_through_merge_and_wire() { const K: u16 = 200; const TRIALS: u64 = 12; let values: Vec = uniform_u64(N, 1_000_000, STREAM_SEED) @@ -724,29 +723,29 @@ fn portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_a let mut tally = Tally::default(); for t in 0..TRIALS { let seed = 0x5EED_0400u64.wrapping_add(t.wrapping_mul(0x9E37_79B9_7F4A_7C15)); - let mut single = KllSketch::with_seed(K, seed); - let mut left = KllSketch::with_seed(K, seed ^ 0xAAAA); - let mut right = KllSketch::with_seed(K, seed ^ 0x5555); + let mut single = KLL::::init_kll_with_seed(K as i32, seed); + let mut left = KLL::::init_kll_with_seed(K as i32, seed ^ 0xAAAA); + let mut right = KLL::::init_kll_with_seed(K as i32, seed ^ 0x5555); for (i, v) in values.iter().enumerate() { - single.update(*v); + single.update(v); if i % 2 == 0 { - left.update(*v); + left.update(v); } else { - right.update(*v); + right.update(v); } } - left.merge(&right).expect("same-k merge"); + left.merge(&right); spec.record_trial( &mut tally, - &format!("portable KllSketch single pass seed={seed:#x}"), + &format!("KLL single pass seed={seed:#x}"), truth.sorted(), &qs, |q| single.quantile(q), ); spec.record_trial( &mut tally, - &format!("portable KllSketch two-shard merge seed={seed:#x}"), + &format!("KLL two-shard merge seed={seed:#x}"), truth.sorted(), &qs, |q| left.quantile(q), @@ -755,9 +754,9 @@ fn portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_a // The wire round trip must preserve every answer bit for bit, which is // an equality rather than a band — and is why the decoded sketch does // not enter the rank battery as a second trial. - let bytes = single.to_msgpack().expect("encode"); - let decoded = KllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(decoded.k(), K, "k must survive the wire"); + let bytes = single.serialize_to_bytes().expect("encode"); + let decoded = KLL::::deserialize_from_bytes(&bytes).expect("decode"); + assert_eq!(decoded.k(), K as usize, "k must survive the wire"); assert_eq!( decoded.count(), single.count(), @@ -768,21 +767,11 @@ fn portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_a let (a, b) = (single.quantile(q), decoded.quantile(q)); wire_tally.record(a == b, || format!("q={q}: before {a} after {b}")); } - wire_tally.assert_none( - &format!("portable KllSketch wire round trip (seed={seed:#x})"), - &context, - ); - - // A mismatched `k` must be refused rather than silently merged. - let other_k = KllSketch::with_seed(K * 2, seed); - assert!( - single.merge(&other_k).is_err(), - "merging sketches with different k must fail" - ); + wire_tally.assert_none(&format!("KLL wire round trip (seed={seed:#x})"), &context); } tally.assert_independent_binomial( - "portable KllSketch / maximum normalized rank error per compaction seed", + "KLL / maximum normalized rank error per compaction seed", spec.trial_failure_probability, &format!("{context}; single pass and two-shard merge, q grid {qs:?}"), ); diff --git a/tests/msgpack_compat.rs b/tests/msgpack_compat.rs index 267a51c6..7e56ccbd 100644 --- a/tests/msgpack_compat.rs +++ b/tests/msgpack_compat.rs @@ -29,10 +29,9 @@ use asap_sketchlib::message_pack_format::portable::countminsketch_topk::{ }; use asap_sketchlib::message_pack_format::portable::delta_set_aggregator::DeltaResult; use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketchWire; -use asap_sketchlib::message_pack_format::portable::kll::KllSketchData; use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, CountSketch, DdSketch, HllSketch, HllVariant, - HydraKllSketch, KllSketch, SetAggregator, + HydraKllSketch, SetAggregator, }; // ===== round-trip: every wire-format-aligned type ===== @@ -95,18 +94,6 @@ fn hll_sketch_round_trip() { assert_eq!(restored.registers.len(), s.registers.len()); } -#[test] -fn kll_sketch_round_trip() { - let mut s = KllSketch::with_seed(200, 0x5EED_0800); - for i in 0..100 { - s.update(i as f64); - } - let bytes = s.to_msgpack().expect("encode"); - let restored = KllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.k, 200); - assert_eq!(restored.count(), 100); -} - #[test] fn hydra_kll_sketch_round_trip() { let mut s = HydraKllSketch::with_seed(2, 4, 200, 0x5EED_0900); @@ -186,21 +173,7 @@ fn count_min_with_heap_wire_shape() { #[test] fn hydra_kll_wire_shape() { - let wire = HydraKllSketchWire { - rows: 2, - cols: 3, - sketches: vec![ - vec![ - KllSketchData { - k: 200, - sketch_bytes: vec![], - }; - 3 - ]; - 2 - ], - }; - let bytes = rmp_serde::to_vec(&wire).unwrap(); + let bytes = HydraKllSketch::new(2, 3, 200).to_msgpack().unwrap(); let restored: HydraKllSketchWire = rmp_serde::from_slice(&bytes).unwrap(); assert_eq!(restored.rows, 2); assert_eq!(restored.cols, 3); @@ -210,7 +183,7 @@ fn hydra_kll_wire_shape() { // ===== golden-bytes placeholders: an ignored, uncovered gap ===== // -// The five tests below are **empty and ignored**. They verify nothing today. +// The four tests below are **empty and ignored**. They verify nothing today. // // What they need is a msgpack payload produced by `sketchlib-go` and checked in // under `tests/fixtures/msgpack/`. That fixture cannot be generated here: this @@ -245,10 +218,6 @@ fn dd_sketch_decodes_go_bytes() {} #[test] fn hll_sketch_decodes_go_bytes() {} -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn kll_sketch_decodes_go_bytes() {} - #[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] #[test] fn hydra_kll_decodes_go_bytes() {} diff --git a/tests/pbt/kll.rs b/tests/pbt/kll.rs index f5c8d5ff..c540a143 100644 --- a/tests/pbt/kll.rs +++ b/tests/pbt/kll.rs @@ -1,5 +1,4 @@ -//! Property tests for KLL: the native sketch and the portable `KllSketch` -//! that carries it over the wire. +//! Property tests for the compact KLL sketch. //! //! Karnin, Lang, Liberty, FOCS '16. Compaction is randomized, so a law may //! not name the items it keeps. What survives: a quantile is monotone in q, @@ -12,8 +11,7 @@ //! `kll_the_ends_are_exact_until_the_first_compaction`. use crate::support::{QUANTILES, close, extreme_values}; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::{KLL, KllSketch}; +use asap_sketchlib::KLL; use proptest::prelude::*; const QUANTILE_PROBES: [f64; 7] = [0.0, 0.01, 0.25, 0.5, 0.75, 0.99, 1.0]; @@ -89,12 +87,8 @@ fn top_weight(s: &KLL) -> f64 { (1u64 << (s.wire_num_levels() - 1)) as f64 } -fn kll_of(k: u16, values: &[f64]) -> KllSketch { - let mut s = KllSketch::with_seed(k, 0xC0FFEE); - for v in values { - s.update(*v); - } - s +fn kll_of(k: u16, values: &[f64]) -> KLL { + native_kll(k as i32, 0xC0FFEE, values) } proptest! { @@ -106,7 +100,7 @@ proptest! { values in prop::collection::vec(-1e5f64..1e5, 0..8), ) { let s = kll_of(k, &values); - prop_assert_eq!(s.count(), values.len() as u64); + prop_assert_eq!(s.count(), values.len()); } #[test] @@ -116,7 +110,7 @@ proptest! { b in prop::collection::vec(-1e5f64..1e5, 0..400), ) { let mut merged = kll_of(k, &a); - merged.merge(&kll_of(k, &b)).expect("merge"); + merged.merge(&kll_of(k, &b)); let n = (a.len() + b.len()) as f64; let got = merged.count() as f64; @@ -164,18 +158,15 @@ proptest! { k in 8u16..256, values in prop::collection::vec(-1e6f64..1e6, 0..300), ) { - let mut s = KllSketch::with_seed(k, 0x5EED); - for v in &values { - s.update(*v); - } + let s = native_kll(k as i32, 0x5EED, &values); - let bytes = s.to_msgpack().expect("encode"); - let restored = KllSketch::from_msgpack(&bytes).expect("decode"); + let bytes = s.serialize_to_bytes().expect("encode"); + let restored = KLL::::deserialize_from_bytes(&bytes).expect("decode"); prop_assert_eq!(restored.k(), s.k()); prop_assert_eq!(restored.count(), s.count()); for q in QUANTILE_PROBES { - prop_assert_eq!(restored.quantile(q), s.quantile(q), "q={}", q); + prop_assert_eq!(restored.quantile(q).to_bits(), s.quantile(q).to_bits(), "q={}", q); } } diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs index 0113ff45..dd12bcc6 100644 --- a/tests/xtest_consumer.rs +++ b/tests/xtest_consumer.rs @@ -6,7 +6,6 @@ //! //! Files consumed (from $XTEST_DIR/): //! countmin.pb — CountMinState with float64 counters -//! kll.pb — KLLState with quantile items and coin RNG state //! ddsketch.pb — DDSketchState with alpha + bucket array //! hll.pb — HyperLogLogState (ErtlMLE variant) //! countsketch.pb — CountSketchState with float64 signed counters @@ -117,48 +116,6 @@ fn cross_language_proto() { println!("[CountMin] PASS"); } - // ----------------------------------------------------------------------- - // KLL - // ----------------------------------------------------------------------- - println!(); - println!("[KLL] Step 1/3 — Read kll.pb"); - let bytes = read_file(in_dir.join("kll.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode kll envelope"); - - println!( - "[KLL] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let kll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(ref s)) => s.clone(), - other => panic!("expected KLL sketch_state, got {other:?}"), - }; - - println!( - "[KLL] k={} m={} num_levels={} items={} levels={}", - kll_state.k, - kll_state.m, - kll_state.num_levels, - kll_state.items.len(), - kll_state.levels.len() - ); - - let kll = KllFromProto::from_state(&kll_state); - let p50 = kll.quantile(0.50); - let p99 = kll.quantile(0.99); - println!("[KLL] Step 3/3 — p50 ≈ {p50:.1} (expect ~5000) p99 ≈ {p99:.1} (expect ~9900)"); - - let p50_ok = (p50 - 5000.0).abs() / 5000.0 < 0.05; - let p99_ok = (p99 - 9900.0).abs() / 9900.0 < 0.05; - if p50_ok && p99_ok { - println!("[KLL] PASS"); - } else { - eprintln!("[KLL] FAIL: p50={p50:.1} p99={p99:.1}"); - all_ok = false; - } - // ----------------------------------------------------------------------- // DDSketch // ----------------------------------------------------------------------- @@ -1097,74 +1054,6 @@ fn cm_query_min(cm: &CountMinState, hash: u64) -> f64 { if min_val == f64::MAX { 0.0 } else { min_val } } -// --------------------------------------------------------------------------- -// Minimal KLL deserialization + CDF query -// --------------------------------------------------------------------------- - -struct KllFromProto { - items: Vec, - levels: Vec, - num_levels: usize, -} - -impl KllFromProto { - fn from_state(s: &KllState) -> Self { - // Dual-read: prefer the value-offset fixed-point representation when - // residuals are present, else fall back to raw f64 items[] (v1). - let items: Vec = if !s.residuals.is_empty() { - asap_sketchlib::message_pack_format::portable::kll::decode_value_offset( - s.offset, - s.value_scale, - &s.residuals, - ) - } else { - s.items.clone() - }; - let levels: Vec = s.levels.iter().map(|&v| v as usize).collect(); - let num_levels = s.num_levels as usize; - Self { - items, - levels, - num_levels, - } - } - - fn weighted_samples(&self) -> Vec<(f64, u64)> { - let mut out = Vec::with_capacity(self.items.len()); - for h in 0..self.num_levels { - let weight: u64 = 1 << h; - let idx = self.num_levels - 1 - h; - if idx + 1 >= self.levels.len() { - continue; - } - let start = self.levels[idx]; - let end = self.levels[idx + 1]; - for &v in &self.items[start..end] { - out.push((v, weight)); - } - } - out - } - - fn quantile(&self, q: f64) -> f64 { - let mut samples = self.weighted_samples(); - if samples.is_empty() { - return 0.0; - } - samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - let total: u64 = samples.iter().map(|(_, w)| w).sum(); - let target = (q * total as f64).ceil() as u64; - let mut acc = 0u64; - for (v, w) in &samples { - acc += w; - if acc >= target { - return *v; - } - } - samples.last().unwrap().0 - } -} - // --------------------------------------------------------------------------- // Minimal DDSketch deserialization + quantile query // --------------------------------------------------------------------------- From 0153ee4dc3b25fd62e9bf7210b0db7d136a6480d Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:52:07 -0600 Subject: [PATCH 2/2] test(kll): ASAPv1 goldens for KLLDynamic; check decoded KLL state - asapv1_golden -> 13797db: kll_dynamic_f64_k200 and kll_dynamic_i64_k200 (kind_id 0x06 0x01, seed 42, no compaction; i64 items cross every msgpack integer width in both signs). - tests/asapv1_golden.rs: all four KLL cases read the golden payload by hand and compare levels, items and coin to the known state; compact decodes compare wire_levels/wire_items/wire_coin to the source; each KLL kind_id is rejected by the other variant's decoder. - KLLDynamic has seeded constructors: correct the wire.rs comment and the Q-KLL-SEED wording; fixture counts in the docs say seven kind_ids. - CHANGELOG: HydraKllSketch's cell type is reachable but unnameable; Added entry for the fixtures. - Re-wrap kll/wire.rs and e2e_testing_harness.md; drop kll.proto from the sketchlib.proto header list. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +- asapv1_golden | 2 +- docs/asapv1_wire_format.md | 6 +- docs/e2e_testing_harness.md | 5 +- docs/message_pack_format.md | 6 +- docs/tests.md | 6 +- proto/sketchlib.proto | 1 - src/sketches/kll/wire.rs | 3 +- src/sketches/kll_dynamic/wire.rs | 4 +- tests/asapv1_golden.rs | 157 +++++++++++++++++++++++++++++-- 10 files changed, 174 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0a2911..0f3d0237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,9 @@ signals a backwards-compatible change. worker's seed and the aggregator's from one seed. The unseeded constructors draw from the thread generator. The seed is not serialized, and the ASAPv1 payload is unchanged. +- **ASAPv1 golden fixtures for `KLLDynamic`** (kind_id `0x06 0x01`): + `kll_dynamic_f64_k200` and `kll_dynamic_i64_k200`, the i64 items crossing + every msgpack integer width in both signs, checked in `tests/asapv1_golden.rs`. ### Changed @@ -105,7 +108,9 @@ signals a backwards-compatible change. `MessagePackCodec` impls for `KllSketch` and `KllSketchData`, the value-offset codec (`KLL_SCALE_SWEEP`, `encode_value_offset`, `decode_value_offset`, `KllProtoItems`), and the `unsafe impl Send/Sync - for KllSketch`. `HydraKllSketch` keeps its per-cell type crate-internally; + for KllSketch`. `HydraKllSketch`'s per-cell type stays reachable through + the public fields `HydraKllSketch.sketch` and + `HydraKllSketchWire.sketches`, but can no longer be named; - `message_pack_format::native::{kll, kll_dynamic}`, the `MessagePackCodec` impls for `KLL` and `KLLDynamic`; - `KLL::from_portable_state`. diff --git a/asapv1_golden b/asapv1_golden index 8287a4ae..13797db1 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit 8287a4ae04e6cfbfb3ed33dafbf1dac77d8f7fc1 +Subproject commit 13797db10c2782946ccc17ad9b1233c636faa1d9 diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index c4586bb9..7046da2d 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -1341,14 +1341,14 @@ Good direction (more compact, higher fidelity, less Rust-internal duplication), 2. **Golden byte-vector fixtures** in one shared repo, [`sketchlib-golden-bytes`](https://github.com/ProjectASAP/sketchlib-golden-bytes), which each implementation mounts at `asapv1_golden/`; both languages decode and re-encode them byte-identically. These replace the `portable`-as-oracle round-trip test. 3. **This registry**, mirrored, never independently allocated. -Fixtures exist for six `kind_id`s — HLL's three estimators, Count-Min, Count Sketch and compact KLL — and `sketchlib-go` mirrors those. Every other kind here has a payload and **no fixture**, so this document is its only contract; `asapv1_golden/README.md` lists the gap. `portable` carries its own Go goldens. +Fixtures exist for seven `kind_id`s — HLL's three estimators, Count-Min, Count Sketch and both KLL variants — and `sketchlib-go` mirrors those. Every other kind here has a payload and **no fixture**, so this document is its only contract; `asapv1_golden/README.md` lists the gap. `portable` carries its own Go goldens. **Hash profile on the Go side.** Rust derives the hash spec from a generic `HashProfile` bound on the hasher type; Go has no generic hasher type, so there is nothing to derive from. On the Go side the profile is simply **written into** the metadata on encode and **read from** it on decode. Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketch is only mergeable/queryable if its `hash_profile_id` + seeds match the profile Go is prepared to reproduce. -Sequencing: (2) covers six `kind_id`s, and the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. +Sequencing: (2) covers seven `kind_id`s, and the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. Retire `portable` once the fixtures cover the rest. --- @@ -1361,7 +1361,7 @@ Retire `portable` once the fixtures cover the rest. - **Q-PROFILE**: the hash-spec metadata is **derived from the hasher's `HashProfile`** (`hll_metadata::` / `cms_metadata::`) and never hardcoded, so it is always truthful to the hasher. Custom hash profiles are **supported and self-describing**. Merge compatibility is hash-spec equality, so a custom-profile sketch is not mergeable with a standard one. - **Q-CMS**: Count-Min is one `kind_id` (`0x02 0x00`); counter type and mode live in the metadata, so the id stays single. The counter-type domain is `"i32"`, `"i64"` and `"f64"`, with no `i128` (no msgpack integer form); `i32` is recorded at its own width and pinned on decode. - **Q-KLL**: KLL metadata carries **no hash-spec group** — KLL is comparison-based and never hashes, so those fields have no truthful value. Its metadata is structural-only (`metadata_version`, `k`, `m`, `item_type`, optional `seed`) and is *not* `HashProfile`-derived. The two KLL variants (compact `0x06 0x00`, dynamic `0x06 0x01`) share one payload `[levels, items, coin]` and differ only by `kind_id`. `item_type` (`"f64"`/`"i64"`) is a metadata param, not a separate `kind_id` (mirrors Q-CMS's `counter_type`). Retained samples use the top-most-level-first layout that matches `sketchlib-go`. -- **Q-KLL-SEED**: KLL records its reproducible compaction `seed` as an **optional** metadata key. It is construction config (so metadata, not payload, per the config→metadata rule), and it is the first optional key in v1: present only when the sketch carries a seed, omitted otherwise. Rationale: the payload's `coin` already carries the RNG's *current* position (enough to resume compaction), but `seed` is what a later `clear()` re-seeds from — so serializing it lets a decoded sketch keep `clear()`-reproducibility instead of falling back to wall-clock. Cost of omitting it is bounded (only a decoded-then-`clear()`ed sketch loses cross-run byte reproducibility — never correctness), but it is cheap to carry and future-proofs the checkpoint/restore path. `KLLDynamic` has no seed concept and never emits the key; the two variants are deliberately **not** forced to be symmetric here. Go carries and preserves the key without interpreting it. +- **Q-KLL-SEED**: KLL records its reproducible compaction `seed` as an **optional** metadata key. It is construction config (so metadata, not payload, per the config→metadata rule), and it is the first optional key in v1: present only when the sketch carries a seed, omitted otherwise. Rationale: the payload's `coin` already carries the RNG's *current* position (enough to resume compaction), but `seed` is what a later `clear()` re-seeds from — so serializing it lets a decoded sketch keep `clear()`-reproducibility instead of falling back to wall-clock. Cost of omitting it is bounded (only a decoded-then-`clear()`ed sketch loses cross-run byte reproducibility — never correctness), but it is cheap to carry and future-proofs the checkpoint/restore path. `KLLDynamic` never emits the key, even when built with a seed; the two variants are deliberately **not** forced to be symmetric here. Go carries and preserves the key without interpreting it. - **Q-CS**: Count Sketch is one `kind_id` (`0x04 0x00`) and mirrors Count-Min's metadata and `[counts]` payload, including `counter_type`. Its type domain differs: `"i32"` and `"i64"`, with no `"f64"` — not because a float fails a bound, but because **no `CountSketchCounter` impl exists for it**: the trait covers `i32`, `i64` and `i128` and nothing else, so Count-Min's `f64` has no Count Sketch counterpart to serialize — and no `i128` (no msgpack integer form), which leaves the two `CsWireCounter` covers. Structural-param order matches Q-CMS: `... matrix_seed_index, rows, cols, counter_type, mode`. **`i32` is not widened to `i64`** — unlike Count-Min, whose counters are plain numbers, a Count Sketch appears *nested* inside `HydraCounter` and `EHSketchList` as `Vector2D` and must decode back into that variant, so the width is identity and is recorded exactly. Cells are signed, so a decoder must not assume monotonicity. - **Q-CMS-DIMS**: Count-Min `rows`/`cols` are **metadata** and the payload omits them. They are configuration that shapes the payload (like HLL's `precision`), so per the config-to-metadata rule they belong in the descriptor. The payload is then just `[counts]`. Canonical structural-param order: `... matrix_seed_index, rows, cols, counter_type, mode`. - **Q-VER**: no payload version field. A new incompatible encoding gets a **new `kind_id`**; retired ids are reserved forever and never recycled. diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index e7848ca8..db1f2caf 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -325,8 +325,9 @@ error against its bound. - No assertion depends on unseeded randomness — no `rand::rng()`, no wall-clock seeding, no implicit RNG inside a constructor. An unseeded - sketch appears only where the assertion holds for every draw. `KLL::init_kll_with_seed`, - `KLLDynamic::init_kll_with_seed`, `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, + sketch appears only where the assertion holds for every draw. + `KLL::init_kll_with_seed`, `KLLDynamic::init_kll_with_seed`, + `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, `Coco::init_with_size_and_seed` and `CocoOctoPlan::with_seed` exist for this. - An estimator whose guarantee is unbiasedness gets no per-key band: its accuracy is the mean over independent seeds, within standard errors of the diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 56e2da00..f1847daf 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -169,9 +169,9 @@ decode → re-encode them byte-identically. The `kind_id` registry is mirrored verbatim with Go's `wire/asapmsgpack/magic_ids.go`, never independently allocated. -That proof covers the six `kind_id`s a fixture exists for — HLL's three -estimators, Count-Min, Count Sketch and compact KLL. Every other implemented -kind has **no ASAPv1 golden and therefore no ASAPv1 drift guard**; +That proof covers the seven `kind_id`s a fixture exists for — HLL's three +estimators, Count-Min, Count Sketch and both KLL variants. Every other +implemented kind has **no ASAPv1 golden and therefore no ASAPv1 drift guard**; [its README](https://github.com/ProjectASAP/sketchlib-golden-bytes#coverage) lists what is covered. The older `portable` path is guarded separately, by the `sketchlib-go` goldens of its own listed above. diff --git a/docs/tests.md b/docs/tests.md index ed4984b7..42396907 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -2106,8 +2106,10 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `hll_classic_p12_matches_golden` | A known P12 Classic register pattern serializes to its golden bytes on both the portable and native paths. | Builds `HllSketch::from_raw(HllVariant::Regular, 12, regs, 0.0, 0.0, 0.0)` where `regs` is 4,096 zeros with `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, verifies `to_msgpack()` equals `asapv1_golden/hll_classic_p12.hex`, that `HyperLogLogP12::::deserialize_from_bytes` on the fixture recovers those registers and re-serializes identically, and that `HllSketch::from_msgpack` reports the same registers, `precision` `12` and `HllVariant::Regular`. | | `hll_ertl_mle_p12_matches_golden` | The Datafusion variant tag reaches the bytes for the same P12 register pattern. | Builds `HllSketch::from_raw(HllVariant::Datafusion, 12, regs, 0.0, 0.0, 0.0)` over the shared P12 pattern (`[0] = 1`, `[1] = 7`, `[100] = 42`, `[4095] = 3` in 4,096 registers), verifies `to_msgpack()` equals `asapv1_golden/hll_ertl_mle_p12.hex`, that `HyperLogLogP12::::deserialize_from_bytes` recovers the registers and re-serializes identically, and that `HllSketch::from_msgpack` reports `HllVariant::Datafusion`. | | `hll_hip_p12_matches_golden` | The HIP variant's three running scalars travel on the wire beside its registers. | Builds `HllSketch::from_raw(HllVariant::Hip, 12, regs, 1.5, 2.5, 3.0)` over the shared P12 pattern, verifies `to_msgpack()` equals `asapv1_golden/hll_hip_p12.hex`, that `HyperLogLogHIPP12::deserialize_from_bytes` re-serializes the fixture identically, and that `HllSketch::from_msgpack` reports the registers, `HllVariant::Hip`, `hip_kxq0` `1.5`, `hip_kxq1` `2.5` and `hip_est` `3.0`. | -| `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | -| `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | +| `kll_dynamic_f64_k200_matches_golden` | A deterministic f64 `KLLDynamic` serializes to its golden bytes under kind_id `0x06 0x01`, and the golden decodes to that state. | Verifies `KLLDynamic::::init_kll_with_seed(200, 42)` fed `[2.5, -1.0, 0.0, 1e300, -0.125, 42.0, 3.0e-5]` in order (no compaction) serializes to `asapv1_golden/kll_dynamic_f64_k200.hex`; that the fixture's own payload, read from the envelope by hand, is kind_id `0x06 0x01`, `levels` `[0, 7]`, those items bit for bit in input order, and coin `(42, 0, 0)`; that `deserialize_from_bytes` reports `count()` `7`, `quantile(0.0)` `-1.0`, `quantile(1.0)` `1e300` and the source's `rank` at every input, and re-serializes identically; and that the compact `KLL` decoder rejects the fixture while the `KLLDynamic` decoder rejects `kll_f64_k200.hex`. | +| `kll_dynamic_i64_k200_matches_golden` | A deterministic i64 `KLLDynamic` whose items cross every msgpack integer width serializes to its golden bytes, and the golden decodes to that state. | Verifies `KLLDynamic::::init_kll_with_seed(200, 42)` fed 21 values from `0` through `i64::MAX` and `i64::MIN` — positive fixint / uint8 / uint16 / uint32 / uint64 and negative fixint / int8 / int16 / int32 / int64 — serializes to `asapv1_golden/kll_dynamic_i64_k200.hex`; that the hand-read payload is kind_id `0x06 0x01`, `levels` `[0, 21]`, those items in input order, and coin `(42, 0, 0)`; that the decode reports `count()` `21`, `quantile(0.0)` `i64::MIN`, `quantile(1.0)` `i64::MAX` and the source's `rank` at every input, and re-serializes identically; and that each KLL kind_id is rejected by the other variant's decoder. | +| `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, and the golden decodes to that state. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`; that the hand-read payload is kind_id `0x06 0x00`, `levels` `[0, 50]`, items `1.0..=50.0` and coin `(42, 0, 0)`; and that decoding the fixture gives the source's `wire_levels()`, `wire_items()` and `wire_coin()`, `count()` `50`, `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`, and re-serializes to the same bytes. | +| `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, and the golden decodes to that state. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`; that the hand-read payload is kind_id `0x06 0x00`, `levels` `[0, 50]`, items `1..=50` and coin `(42, 0, 0)`; and that decoding the fixture gives the source's `wire_levels()`, `wire_items()` and `wire_coin()`, `count()` `50`, `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`, and re-serializes to the same bytes. | ### MessagePack Envelope Compatibility diff --git a/proto/sketchlib.proto b/proto/sketchlib.proto index 6c618327..702541b9 100644 --- a/proto/sketchlib.proto +++ b/proto/sketchlib.proto @@ -7,7 +7,6 @@ // countminsketch/countminsketch.proto — CountMinState // countsketch/countsketch.proto — CountSketchState, TopKState, HeapEntry // hll/hll.proto — HyperLogLogState, HLLVariant -// kll/kll.proto — KLLState, CoinState // ddsketch/ddsketch.proto — DDSketchState // univmon/univmon.proto — UnivMonState, UnivMonLayer // hydra/hydra.proto — HydraState, HydraCounterType, HydraCell diff --git a/src/sketches/kll/wire.rs b/src/sketches/kll/wire.rs index 30deba8a..af6c8b78 100644 --- a/src/sketches/kll/wire.rs +++ b/src/sketches/kll/wire.rs @@ -21,7 +21,8 @@ //! //! The payload's `levels` / `items` use the **top-most-level-first** layout, //! byte-for-byte matching `sketchlib-go` (index `i` in `levels` maps to -//! compactor level `num_levels - 1 - i`; level 0's run is in input order). For the compact KLL this is exactly what [`KLL::wire_levels`] / +//! compactor level `num_levels - 1 - i`; level 0's run is in input order). For +//! the compact KLL this is exactly what [`KLL::wire_levels`] / //! [`KLL::wire_items`] emit (they reverse the leftward-grown L0 buffer back to //! input order); decode inverts that mapping. diff --git a/src/sketches/kll_dynamic/wire.rs b/src/sketches/kll_dynamic/wire.rs index 207dd6a2..b5b56b4f 100644 --- a/src/sketches/kll_dynamic/wire.rs +++ b/src/sketches/kll_dynamic/wire.rs @@ -37,8 +37,8 @@ where /// top-most-level-first / input-order-L0 layout, so `levels` and `items` are /// copied straight out. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - // `KLLDynamic` has no reproducible-seed concept, so `seed` is always - // `None` and the metadata key is omitted. + // `KLLDynamic` never emits the `seed` metadata key, even when built + // with one. let metadata = rmp_serde::to_vec_named(&kll_metadata::(self.k as u32, self.m as u32, None))?; let (state, bit_cache, remaining_bits) = self.co.to_wire(); diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 186d0993..fdc4bd1a 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -10,8 +10,9 @@ use asap_sketchlib::{ Classic, Count, CountMin, ErtlMLE, FastPath, HllSketch, HllVariant, HyperLogLogHIPP12, - HyperLogLogP12, KLL, MessagePackCodec, RegularPath, Vector2D, + HyperLogLogP12, KLL, KLLDynamic, MessagePackCodec, RegularPath, Vector2D, }; +use serde::de::DeserializeOwned; fn decode_hex(s: &str) -> Vec { let s = s.trim(); @@ -32,6 +33,8 @@ const GOLDEN_CS_FAST: &str = include_str!("../asapv1_golden/cs_i64_fast_2x4.hex" const GOLDEN_CS_I32: &str = include_str!("../asapv1_golden/cs_i32_regular_2x4.hex"); const GOLDEN_KLL_F64: &str = include_str!("../asapv1_golden/kll_f64_k200.hex"); const GOLDEN_KLL_I64: &str = include_str!("../asapv1_golden/kll_i64_k200.hex"); +const GOLDEN_KLL_DYN_F64: &str = include_str!("../asapv1_golden/kll_dynamic_f64_k200.hex"); +const GOLDEN_KLL_DYN_I64: &str = include_str!("../asapv1_golden/kll_dynamic_i64_k200.hex"); /// The known P12 register pattern shared by all three HLL fixtures. fn p12_registers() -> Vec { @@ -243,11 +246,33 @@ fn cs_i32_regular_2x4_matches_golden() { } // --------------------------------------------------------------------------- -// KLL: build known state (k=200, seed 42, integers 1..=50 — below the level-0 -// capacity, so no compaction fires and the retained set is deterministic) -> -// serialize == golden, and golden round-trips. +// KLL: build known state (k=200, seed 42, inputs below the level-0 capacity, so +// no compaction fires and the retained set is deterministic) -> serialize == +// golden; golden decodes to that state and re-encodes byte-identically. // --------------------------------------------------------------------------- +/// The coin of a sketch seeded with 42 that has not compacted. +const KLL_COIN_SEED_42: (u64, u64, u32) = (42, 0, 0); + +/// A KLL payload `[levels, items, coin]`, read straight from the envelope. +type KllPayloadView = (Vec, Vec, (u64, u64, u32)); + +/// Splits an ASAPv1 envelope by hand and decodes its KLL payload. +fn kll_envelope(bytes: &[u8]) -> (Vec, KllPayloadView) { + assert_eq!(&bytes[..7], b"ASAPv1\x01", "magic and version"); + let kind_len = bytes[7] as usize; + let kind_id = bytes[8..8 + kind_len].to_vec(); + let at = 8 + kind_len; + let meta_len = u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize; + let payload_len = u32::from_be_bytes(bytes[at + 4..at + 8].try_into().unwrap()) as usize; + let payload = &bytes[at + 8 + meta_len..]; + assert_eq!(payload.len(), payload_len, "payload length prefix"); + ( + kind_id, + rmp_serde::from_slice(payload).expect("KLL payload"), + ) +} + /// A k=200 KLL over `1..=50` with a fixed compaction seed: fully deterministic. fn kll_1to50 + asap_sketchlib::NumericalValue>(seed: u64) -> KLL { let mut sketch = KLL::::init_kll_with_seed(200, seed); @@ -265,11 +290,20 @@ fn kll_f64_k200_matches_golden() { let got = sketch.serialize_to_bytes().expect("serialize"); assert_eq!(got, want, "KLL f64 bytes diverge from golden"); - // Golden round-trips: decode, and re-encode is byte-identical. + let (kind_id, (levels, items, coin)) = kll_envelope::(&want); + assert_eq!(kind_id, [0x06, 0x00]); + assert_eq!(levels, [0, 50]); + assert_eq!(items, (1..=50).map(f64::from).collect::>()); + assert_eq!(coin, KLL_COIN_SEED_42); + let decoded = KLL::::deserialize_from_bytes(&want).expect("decode"); - assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); + assert_eq!(decoded.wire_levels(), sketch.wire_levels()); + assert_eq!(decoded.wire_items(), sketch.wire_items()); + assert_eq!(decoded.wire_coin(), sketch.wire_coin()); + assert_eq!(decoded.count(), 50); assert_eq!(decoded.quantile(0.0), 1.0); assert_eq!(decoded.quantile(1.0), 50.0); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); } #[test] @@ -280,8 +314,117 @@ fn kll_i64_k200_matches_golden() { let got = sketch.serialize_to_bytes().expect("serialize"); assert_eq!(got, want, "KLL i64 bytes diverge from golden"); + let (kind_id, (levels, items, coin)) = kll_envelope::(&want); + assert_eq!(kind_id, [0x06, 0x00]); + assert_eq!(levels, [0, 50]); + assert_eq!(items, (1..=50).collect::>()); + assert_eq!(coin, KLL_COIN_SEED_42); + let decoded = KLL::::deserialize_from_bytes(&want).expect("decode"); - assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); + assert_eq!(decoded.wire_levels(), sketch.wire_levels()); + assert_eq!(decoded.wire_items(), sketch.wire_items()); + assert_eq!(decoded.wire_coin(), sketch.wire_coin()); + assert_eq!(decoded.count(), 50); assert_eq!(decoded.quantile(0.0), 1.0); assert_eq!(decoded.quantile(1.0), 50.0); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} + +/// Negative, zero, fractional and extreme f64 samples, in insertion order. +const KLL_DYN_F64_VALS: [f64; 7] = [2.5, -1.0, 0.0, 1e300, -0.125, 42.0, 3.0e-5]; + +/// i64 samples crossing every msgpack integer width: positive fixint / uint8 / +/// uint16 / uint32 / uint64 and negative fixint / int8 / int16 / int32 / int64. +const KLL_DYN_I64_VALS: [i64; 21] = [ + 0, + 1, + -1, + 127, + -32, + 128, + -33, + 255, + -128, + 256, + -129, + 65535, + -32768, + 65536, + -32769, + 4294967295, + -2147483648, + 4294967296, + -2147483649, + i64::MAX, + i64::MIN, +]; + +fn kll_dynamic_of(values: &[T]) -> KLLDynamic { + let mut sketch = KLLDynamic::::init_kll_with_seed(200, 42); + for v in values { + sketch.update(v); + } + sketch +} + +#[test] +fn kll_dynamic_f64_k200_matches_golden() { + let want = decode_hex(GOLDEN_KLL_DYN_F64); + + let sketch = kll_dynamic_of(&KLL_DYN_F64_VALS); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "KLLDynamic f64 bytes diverge from golden"); + + let (kind_id, (levels, items, coin)) = kll_envelope::(&want); + assert_eq!(kind_id, [0x06, 0x01]); + assert_eq!(levels, [0, KLL_DYN_F64_VALS.len() as u32]); + assert_eq!( + items.iter().map(|v| v.to_bits()).collect::>(), + KLL_DYN_F64_VALS + .iter() + .map(|v| v.to_bits()) + .collect::>() + ); + assert_eq!(coin, KLL_COIN_SEED_42); + + // Encode copies the buffer, levels and coin verbatim, so equal bytes mean + // equal state. + let decoded = KLLDynamic::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.count(), KLL_DYN_F64_VALS.len()); + assert_eq!(decoded.quantile(0.0), -1.0); + assert_eq!(decoded.quantile(1.0), 1e300); + for v in KLL_DYN_F64_VALS { + assert_eq!(decoded.rank(v), sketch.rank(v), "rank({v})"); + } + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); + + assert!(KLL::::deserialize_from_bytes(&want).is_err()); + assert!(KLLDynamic::::deserialize_from_bytes(&decode_hex(GOLDEN_KLL_F64)).is_err()); +} + +#[test] +fn kll_dynamic_i64_k200_matches_golden() { + let want = decode_hex(GOLDEN_KLL_DYN_I64); + + let sketch = kll_dynamic_of(&KLL_DYN_I64_VALS); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "KLLDynamic i64 bytes diverge from golden"); + + let (kind_id, (levels, items, coin)) = kll_envelope::(&want); + assert_eq!(kind_id, [0x06, 0x01]); + assert_eq!(levels, [0, KLL_DYN_I64_VALS.len() as u32]); + assert_eq!(items, KLL_DYN_I64_VALS); + assert_eq!(coin, KLL_COIN_SEED_42); + + let decoded = KLLDynamic::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.count(), KLL_DYN_I64_VALS.len()); + assert_eq!(decoded.quantile(0.0), i64::MIN as f64); + assert_eq!(decoded.quantile(1.0), i64::MAX as f64); + for v in KLL_DYN_I64_VALS { + assert_eq!(decoded.rank(v as f64), sketch.rank(v as f64), "rank({v})"); + } + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); + + assert!(KLL::::deserialize_from_bytes(&want).is_err()); + assert!(KLLDynamic::::deserialize_from_bytes(&decode_hex(GOLDEN_KLL_I64)).is_err()); }