From 8684e42ad9d70137598f48b6ab65bbf97ea8871b Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:44:33 -0600 Subject: [PATCH 1/3] refactor(hll)!: drop the proto and portable formats HyperLogLog serializes only as ASAPv1. Remove the portable HllSketch, HllSketchDelta and HllVariant with the sparse-register helpers, the native MessagePackCodec shims, the HLLDelta proto message and the SketchEnvelope hll field (12, now reserved). HyperLogLogState stays for HydraCell. Add HyperLogLogImpl::from_storage and HyperLogLogHIPImpl::from_storage, and check the three HLL ASAPv1 goldens through the native types. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 19 +- docs/api/api_hyperloglog.md | 2 + docs/asapv1_wire_format.md | 2 +- docs/features.md | 2 +- docs/library_map.md | 2 +- docs/message_pack_format.md | 2 +- docs/tests.md | 45 +- examples/accuracy_probe.rs | 16 - proto/hll/hll.proto | 37 +- proto/sketchlib.proto | 5 +- src/lib.rs | 3 +- src/message_pack_format/native/hll.rs | 27 - src/message_pack_format/native/mod.rs | 1 - src/message_pack_format/portable/hll.rs | 1056 ----------------- src/message_pack_format/portable/mod.rs | 3 +- src/message_pack_format/portable/sampling.rs | 11 +- src/proto/generated/sketchlib.v1.rs | 36 +- src/sketches/hll.rs | 23 +- src/sketches/hll/wire.rs | 66 +- src/sketches/testdata/hll_envelope_golden.hex | 1 - tests/PBT_COVERAGE.md | 12 +- tests/asapv1_golden.rs | 66 +- tests/conformance_kit.rs | 21 - tests/e2e/cardinality.rs | 72 -- tests/msgpack_compat.rs | 19 +- tests/pbt/hll.rs | 128 +- tests/proptest-regressions/hll.txt | 7 + tests/sketches_go_parity_probe.rs | 61 +- tests/xtest_consumer.rs | 150 --- 29 files changed, 134 insertions(+), 1761 deletions(-) delete mode 100644 src/message_pack_format/native/hll.rs delete mode 100644 src/message_pack_format/portable/hll.rs delete mode 100644 src/sketches/testdata/hll_envelope_golden.hex create mode 100644 tests/proptest-regressions/hll.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 80d85f98..5266204e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ signals a backwards-compatible change. law that sketch claims — merge algebra, path equivalence, the paper's own model, and the wire / ASAPv1 round trip. Shared generators, the `grid` reader and the `round_trip!` macro live in `tests/pbt/support.rs`; four - modules sit behind `experimental`. 435 tests under `--all-features`, 367 + modules sit behind `experimental`. 433 tests under `--all-features`, 365 under the default feature set. Every law was mutation-checked: the implementation was broken deliberately, the law confirmed red, and confirmed not to take unrelated laws down with it; a law no semantically real mutation @@ -66,6 +66,9 @@ signals a backwards-compatible change. worker's seed and the aggregator's from one seed. The unseeded constructors draw from the thread generator. The seed is not serialized, and the ASAPv1 payload is unchanged. +- **HyperLogLog `from_storage` constructors.** `HyperLogLogImpl::from_storage(registers)` + and `HyperLogLogHIPImpl::from_storage(registers, kxq0, kxq1, est)` build a + sketch from known register storage (and HIP running state). ### Changed @@ -88,6 +91,20 @@ signals a backwards-compatible change. as an empty window like any other, and that window takes a place in `max_windows`. +### Removed + +- **HyperLogLog's protobuf and portable formats.** HyperLogLog serializes only + as ASAPv1 (`serialize_to_bytes` / `deserialize_from_bytes`). Removed: the + portable `HllSketch`, `HllSketchDelta` and `HllVariant` (module + `message_pack_format::portable::hll` and their crate-root re-exports), with + `registers_from_state`, `decode_sparse_registers` and + `encode_sparse_registers`; the `MessagePackCodec` impls for + `HyperLogLogImpl` and `HyperLogLogHIPImpl` (`message_pack_format::native::hll`); + the `HLLDelta` proto message (`proto::sketchlib::HllDelta`); and the `hll` + field (12) of `SketchEnvelope`, now reserved. `HyperLogLogState`, + `HLLSparseRegisters` and `HLLVariant` remain only as the `HydraCell` HLL + state. + ### Fixed - `KLLDynamic` panicked on every quantile query once a NaN had entered the diff --git a/docs/api/api_hyperloglog.md b/docs/api/api_hyperloglog.md index 4e2752bf..aaf3d482 100644 --- a/docs/api/api_hyperloglog.md +++ b/docs/api/api_hyperloglog.md @@ -40,6 +40,8 @@ but improves accuracy. ```rust fn new() -> Self fn default() -> Self +fn from_storage(registers: Registers) -> Self // HyperLogLogImpl +fn from_storage(registers: Registers, kxq0: f64, kxq1: f64, est: f64) -> Self // HyperLogLogHIPImpl ``` ## Insert / Update diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index fd14e70a..e4540fbe 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -1348,7 +1348,7 @@ Rust derives the hash spec from a generic `HashProfile` bound on the hasher type On the Go side the profile is simply **written into** the metadata on encode and **read from** it on decode. Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketch is only mergeable/queryable if its `hash_profile_id` + seeds match the profile Go is prepared to reproduce. -Sequencing: (2) covers six `kind_id`s, and the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. +Sequencing: (2) covers six `kind_id`s. Retire `portable` once the fixtures cover the rest. --- diff --git a/docs/features.md b/docs/features.md index 64f296a3..9f0a5b65 100644 --- a/docs/features.md +++ b/docs/features.md @@ -112,7 +112,7 @@ MessagePack (`rmp-serde`) support. **serde support** means the type derives `Ser | EHSketchList | Yes | Yes | | ExponentialHistogram | No | Yes | -Protobuf (prost): `.proto` definitions exist for CountMin, Count, HLL, DDSketch, KLL, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. +Protobuf (prost): `.proto` definitions exist for CountMin, Count, DDSketch, KLL, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. ### API Stability diff --git a/docs/library_map.md b/docs/library_map.md index a8e614b3..48dc23e6 100644 --- a/docs/library_map.md +++ b/docs/library_map.md @@ -26,7 +26,7 @@ - **`src/message_pack_format/`** - Serialization plumbing ([message_pack_format.md](./message_pack_format.md)). The current format is **ASAPv1**, specified in [asapv1_wire_format.md](./asapv1_wire_format.md) - `envelope.rs` — the shared, sketch-agnostic ASAPv1 framing (magic/version/`kind_id` + length prefixes, `encode`/`split`); every `wire.rs` under `src/sketches/` and `src/sketch_framework/` calls into it - `codec.rs` — the `MessagePackCodec` trait; `error.rs` — the unified `Error` - - `portable/` — **deprecated**, being retired. The older per-sketch wire types (`CountMinSketch`, `HllSketch`, …); ASAPv1 (per-sketch `wire.rs`) is now what `sketchlib-go` mirrors, not these + - `portable/` — **deprecated**, being retired. The older per-sketch wire types (`CountMinSketch`, `KllSketch`, …); ASAPv1 (per-sketch `wire.rs`) is now what `sketchlib-go` mirrors, not these - `native/` — **deprecated**, being retired. Older `MessagePackCodec` shims over `src/sketches/` byte serialization ## Documentation diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 12a2b91c..3cecb28a 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -150,7 +150,7 @@ being **phased out** in favor of the per-sketch `wire.rs` + the shared `envelope.rs`. - `portable/` holds per-sketch wire types that predate the envelope. They - carry their own byte-parity goldens against `sketchlib-go` — HLL, Count-Min, + carry their own byte-parity goldens against `sketchlib-go` — Count-Min, Count Sketch and KLL (both directions) — separate from `asapv1_golden/`. - `native/` is a set of thin `MessagePackCodec` shims over the sketches' `serialize_to_bytes` / `deserialize_from_bytes`. Every type it wraps emits the diff --git a/docs/tests.md b/docs/tests.md index ccb1571c..66513dab 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -110,7 +110,6 @@ Wire tests: [`src/sketches/hll/wire.rs`](../src/sketches/hll/wire.rs) | `hll_correctness_test` | Register update logic matches expected bucket/index behavior for all HLL variants. | Runs fixed hashed inserts against Classic, ErtlMLE, and HIP variants; asserts exact expected register values at specific bucket indices and confirms an untouched bucket remains zero. | | `hll_envelope_structure_and_kind_id_guard` | The envelope frames an Ertl-MLE sketch under kind_id `0x01 0x02`, and a Classic decoder refuses it. | For 1,000 inserts into `HyperLogLog`, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x01 0x02`, that the decoded registers match the source, and that a `HyperLogLog` decode fails. | | `hll_hip_round_trip_preserves_state` | The HIP running scalars travel beside the registers. | For 1,000 inserts into `HyperLogLogHIP`, verifies the bytes carry kind_id `0x01 0x03` and that the decoded sketch matches the source on the registers and on `kxq0`, `kxq1`, and `est`. | -| `native_and_portable_hll_bytes_match` | The native and portable encoders emit the same envelope. | For 1,000 inserts each, verifies an `ErtlMLE` sketch's bytes equal the portable `HllSketch` `Datafusion` encoding of its registers, and a `HyperLogLogHIP` sketch's bytes equal the portable `Hip` encoding of its registers plus `kxq0`, `kxq1`, and `est`. | | `hll_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a P14 `ErtlMLE` sketch over a hasher declaring its own `HashProfile`, verifies the registers round-trip, the bytes differ from the standard-profile sketch's over the same 1,000 inserts, and a standard-profile decode rejects them. | | `hll_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `HllMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HllMetadata`. | | `hll_precision_cross_rejection` | The metadata `precision` pins the register storage the bytes belong to. | Verifies a populated `HyperLogLogP12` envelope fails to decode as the P14 `HyperLogLog`. | @@ -1327,36 +1326,6 @@ Test file: [`src/message_pack_format/portable/delta_set_aggregator.rs`](../src/m | `test_msgpack_round_trip` | Both key sets survive the round trip. | Round-trips a `DeltaResult` with `added` `{"web", "api"}` and `removed` `{"db"}` and verifies the decode holds 2 added keys including both names and 1 removed key. | | `test_empty_sets` | A `DeltaResult` holding nothing round-trips. | Verifies a `DeltaResult` with both sets empty encodes and decodes back with both sets still empty. | -### Portable HllSketch - -Test file: [`src/message_pack_format/portable/hll.rs`](../src/message_pack_format/portable/hll.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `estimate_holds_past_the_32_bit_range` | The estimator follows the HyperLogLog formula rather than saturating. | For each rank in `12`, `16`, `18`, and `20` filled across all 16,384 registers of a precision-14 sketch, verifies `estimate()` is within a relative `1e-9` of `alpha_m * m * 2^rank` with `alpha_m = 0.7213 / (1 + 1.079 / m)` and `m = 16384`. | -| `msgpack_delta_against_empty_round_trips` | A msgpack register delta against an empty snapshot carries every register. | For a precision-12 sketch fed the little-endian bytes of `0..2000u64`, verifies the delta against an empty sketch at threshold `1`, applied to a fresh sketch, reproduces the register array exactly and the estimate to within 0.1%. | -| `msgpack_delta_wire_layout_is_parallel_arrays` | The register delta is two parallel arrays of index and value. | For a precision-4 sketch fed `b"a"` and `b"b"`, verifies the delta bytes equal `rmp_serde::to_vec(&(reg_idx: Vec, reg_val: Vec))` recomputed from the non-zero registers, and that decoding yields one update per non-zero register. | -| `test_new_empty` | A fresh sketch is `2^precision` zeroed registers. | For `HllSketch::new(Regular, 4)`, verifies the register array is 16 long and every register is `0`. | -| `test_merge_register_wise_max` | Merge takes the register-wise maximum. | Merges the precision-2 registers `[4, 2, 6, 0]` into `[1, 5, 3, 7]` and verifies the result is `[4, 5, 6, 7]`. | -| `test_apply_delta_max_semantics` | A delta raises registers to the maximum, never lowers them. | Applies the updates `(0,4)`, `(1,2)`, `(2,6)`, `(3,0)` to the precision-2 registers `[1, 5, 3, 7]` and verifies the result is `[4, 5, 6, 7]`. | -| `test_apply_delta_out_of_range` | A register index past the array is refused. | Verifies a delta carrying update `(7, 3)` against a precision-2 (four-register) sketch fails to apply. | -| `test_apply_delta_matches_full_merge` | Applying a delta lands where merging the same addition lands. | For the precision-2 base `[1, 5, 3, 7]`, verifies merging the addition `[4, 0, 6, 0]` and applying the delta updates `(0,4)` and `(2,6)` give the same register array. | -| `test_merge_variant_mismatch` | Two estimator variants are not mergeable. | Verifies merging a `Datafusion` precision-4 sketch into a `Regular` precision-4 one fails. | -| `test_merge_precision_mismatch` | Two precisions are not mergeable. | Verifies merging a precision-5 `Regular` sketch into a precision-4 one fails. | -| `test_merge_refs` | `merge_refs` takes the maximum across every input. | Merges three precision-1 sketches holding `[1, 0]`, `[0, 3]`, and `[2, 2]` and verifies the result is `[2, 3]`. | -| `test_update_then_estimate_within_2pct` | The estimate lands within 2% on a 50,000-key stream. | Feeds `key-0`..`key-49999` into a precision-12 `Regular` sketch and verifies the relative error of `estimate()` against `50_000` is below `0.02`. | -| `test_estimate_empty_is_zero` | An empty sketch estimates zero. | Verifies a precision-4 `Regular` sketch reports an `estimate()` of exactly `0.0`. | -| `test_msgpack_round_trip` | The encoding is an ASAPv1 envelope and the HIP state survives it. | Round-trips a `Hip` precision-3 sketch holding registers `[0, 1, 2, 3, 4, 5, 6, 7]` with `hip_kxq0` `1.0`, `hip_kxq1` `2.0`, and `hip_est` `3.0`, and verifies the bytes open with the `ASAPv1` magic and that the decode matches on registers, precision, and a `hip_kxq0` of `1.0`. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The proto envelope for a populated register array is byte-identical to the checked-in golden. | For a `Datafusion` precision-14 sketch fed the little-endian `f64` bytes of `1..=50`, builds a `HyperLogLogState` from `wire_proto_variant()`, the precision, and the registers with `registers_sparse` unset, wraps it in a `SketchEnvelope` with `format_version` `1` and `sample_p` `0.0`, and verifies the prost encoding matches the 16,398-byte [`hll_envelope_golden.hex`](../src/sketches/testdata/hll_envelope_golden.hex) on both length and content. | -| `sparse_round_trip_exact` | The sparse register encoding is exact. | For a 16,384-register array holding `1` at index `0`, `7` at `1`, `51` at `300`, `12` at `8191`, and `3` at `16383`, verifies `encode_sparse_registers` reports `num_registers` `16384` and that the decode returns the array unchanged. | -| `registers_from_state_reads_both` | `registers_from_state` reads the sparse and the dense form alike. | For the same 16,384-register array at `variant` `2` and `precision` `14`, verifies the registers come back identical whether the state carries `registers_sparse` with `registers` empty or `registers` with `registers_sparse` unset. | -| `sparse_empty_is_all_zero` | An all-zero register array encodes to nothing. | For 16,384 zeroed registers, verifies the sparse `packed` field is empty and the decode returns the zeros. | -| `sparse_rejects_out_of_range_index` | A sparse index past `num_registers` is refused. | Verifies `HllSparseRegisters { num_registers: 4, packed: [5, 3] }`, whose first uvarint delta of `5` lands outside a four-register array, fails to decode. | -| `test_compute_delta_against_empty_round_trips` | A proto register delta against an empty snapshot carries every register. | For a `Datafusion` precision-14 sketch fed the little-endian bytes of `0..5000u64`, verifies the delta against an empty sketch at threshold `0`, applied to a fresh sketch, reproduces the register array. | -| `test_compute_delta_then_apply_matches_current` | A delta between two non-empty snapshots reconstructs the later one. | For a `Datafusion` precision-12 base fed `0..2000u64` and a current fed `2000..5000u64` on top, verifies the delta applied to the base reproduces the current register array. | -| `test_compute_delta_matches_go_golden_bytes` | The packed register delta bytes are pinned by a golden. | For a `Datafusion` precision-14 sketch fed the little-endian `f64` bytes of `1..=50`, verifies the delta against an empty sketch at threshold `0` equals a 132-byte inline golden in the packed `(index_delta, value)` form. | -| `test_compute_subwindow_delta_matches_go_golden_bytes` | A sub-window delta's bytes are pinned by a golden and still reconstruct the window. | For a `Datafusion` precision-14 base fed `0..2000` as `f64` little-endian bytes and a current fed `2000..5000` on top, verifies the delta carries only the registers that grew and equals a 5,097-byte inline golden, then that applying it to the base reproduces the current register array. | - ### Portable HydraKllSketch Test file: [`src/message_pack_format/portable/hydra_kll.rs`](../src/message_pack_format/portable/hydra_kll.rs) @@ -1632,7 +1601,6 @@ Test file: [`tests/e2e/cardinality.rs`](../tests/e2e/cardinality.rs) | --- | --- | --- | | `hll_{classic,ertl_mle,hip}_p{12,14,16}_satisfies_..._error_model` | Each core HLL alias tracks the true cardinality inside its own estimator's `z = 4` band, ignores replayed identities, and reproduces a single pass from disjoint shards. | One `hll_battery!` body per invocation, over `CHECKPOINTS = [10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]`. A fresh sketch per checkpoint is fed `DataInput::U64(base + k)` from its own namespace `base = (1 << 40) * (i + 1)`, and each `estimate()` goes through `CardinalityConfidenceSpec::check` at `z = 4.0`: relative band `4 * 1.04/sqrt(m)` for the `hll` model, `4 * sqrt(ln 2 / m)` for `hll_hip`. Then re-inserting the first `min(target, 200_000)` already-seen identities must leave `estimate()` bit-identical (`assert_eq!`). Then over a fresh `200_000`-identity namespace, an even/odd shard merge must give `registers_as_slice()` and `estimate()` equal to the single pass. Invocations: `hll_classic_p12_satisfies_its_register_error_model` (`HyperLogLogP12`, `hll`, mergeable), `hll_classic_p14_satisfies_its_register_error_model` (`HyperLogLogP14`), `hll_classic_p16_satisfies_its_register_error_model` (`HyperLogLogP16`), `hll_ertl_mle_p12_satisfies_the_cramer_rao_error_model` (`HyperLogLogP12`), `hll_ertl_mle_p14_satisfies_the_cramer_rao_error_model` (`HyperLogLogP14`), `hll_ertl_mle_p16_satisfies_the_cramer_rao_error_model` (`HyperLogLogP16`), `hll_hip_p12_satisfies_the_hip_error_model` (`HyperLogLogHIPP12`, `hll_hip`, not_mergeable), `hll_hip_p14_satisfies_the_hip_error_model` (`HyperLogLogHIPP14`), `hll_hip_p16_satisfies_the_hip_error_model` (`HyperLogLogHIPP16`). The three `not_mergeable` HIP invocations still build and fill the three `200_000`-identity shard sketches but assert nothing about them. | | `hll_{classic,ertl_mle,hip}_custom_p{10..18}_satisfies_..._error_model` | Every custom register geometry from p10 to p18 holds its estimator's `z = 4` band over its own checkpoint grid, is unmoved by a replay, and merges shard-for-shard. | The same `hll_battery!` body as the core aliases (fresh sketch per checkpoint on namespace `(1 << 40) * (i + 1)`, `CardinalityConfidenceSpec::check` at `z = 4.0`, replay of `min(target, 200_000)` identities asserted bit-identical, `200_000`-identity even/odd merge asserted equal to the single pass register for register), but over `HyperLogLogImpl` (`hll`, mergeable), `HyperLogLogImpl` (`hll`, mergeable) and `HyperLogLogHIPImpl` (`hll_hip`, not_mergeable, so its merge block asserts nothing), each with the precision's own checkpoints: p10 `[100, 1_000, 20_000, 200_000]` for `hll_classic_custom_p10_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p10_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p10_satisfies_the_hip_error_model`; p11 `[200, 2_000, 40_000, 200_000]` for `hll_classic_custom_p11_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p11_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p11_satisfies_the_hip_error_model`; p12 `[400, 4_000, 50_000, 300_000]` for `hll_classic_custom_p12_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p12_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p12_satisfies_the_hip_error_model`; p13 `[1_000, 10_000, 100_000, 500_000]` for `hll_classic_custom_p13_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p13_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p13_satisfies_the_hip_error_model`; p14 `[2_000, 16_000, 200_000, 800_000]` for `hll_classic_custom_p14_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p14_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p14_satisfies_the_hip_error_model`; p15 `[4_000, 32_000, 300_000, 1_000_000]` for `hll_classic_custom_p15_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p15_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p15_satisfies_the_hip_error_model`; p16 `[5_000, 60_000, 500_000, 1_200_000]` for `hll_classic_custom_p16_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p16_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p16_satisfies_the_hip_error_model`; p17 `[10_000, 100_000, 800_000, 1_500_000]` for `hll_classic_custom_p17_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p17_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p17_satisfies_the_hip_error_model`; p18 `[10_000, 100_000, 1_500_000]` for `hll_classic_custom_p18_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p18_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p18_satisfies_the_hip_error_model`. | -| `portable_hll_variants_and_precisions_satisfy_the_register_error_model` | Every portable wire variant and precision holds the register error model and merges by register-wise maximum. | For each of `HllVariant::Regular`, `HllVariant::Datafusion`, `HllVariant::Hip` crossed with precisions `12`, `14`, `16`, draws its own stream `uniform_u64(200_000, u64::MAX / 4, seed)` with `seed = 2001 + (v * 3 + p)`, feeds `HllSketch::update` with `to_be_bytes()`, and checks `HllSketch::estimate()` against the `HashSet` distinct count via `CardinalityConfidenceSpec::hll(precision, 4.0)` — all three tags are held to `4 * 1.04/sqrt(m)`, since `estimate()` runs the same register formula for each. Then `merge` of a second sketch built over the same distinct identities must leave `estimate()` unchanged, and an even/odd shard merge of the same stream must give exactly that same `estimate()`. | | `hll_accuracy_improves_with_precision_as_the_error_model_predicts` | A larger precision actually buys accuracy, and the measured RSE stays near the predicted `1.04/sqrt(m)`. | Measures the RMS relative error of `HyperLogLogP12`, `HyperLogLogP14` and `HyperLogLogP16` over `BLOCKS = 6` disjoint blocks of `N = 1_000_000` identities (`DataInput::U64(b * N + i)`), asserts each measured RSE is `<= 2.0 *` `CardinalityConfidenceSpec::hll(p, 4.0).sigma_rel()`, and asserts the p12 RSE is at least `2.0x` the p16 RSE; the model's predicted 4x improvement is not asserted, only 2x. | | `hll_classic_switchover_band_stays_within_the_documented_empirical_band` | The Classic estimator's error at the linear-counting switchover stays inside a 1.5x-to-10x empirical band around the asymptotic RSE. | For `HyperLogLogP12/P14/P16` at `n = 2.5m` (`10_240`, `40_960`, `163_840`), computes the RMS relative error over `BLOCKS = 6` disjoint blocks (identities `b * 20_000_000 + i`) and forms `ratio = rse / (1.04/sqrt(m))` from `CardinalityConfidenceSpec::hll(p, 4.0).sigma_rel()`; asserts `ratio <= 10.0` and `ratio >= 1.5`, so both a widened cliff and a vanished one fail. | | `set_aggregator_union_is_exact` | A `SetAggregator` union keeps every member exactly. | Feeds `uniform_u64(20_000, 500, 2002)` as `member-{k}` strings into `SetAggregator::update`, merges a second aggregator holding `extra-a` and `extra-b`, then asserts `agg.values.len()` equals the length of a parallel `HashSet` of the same members and that `agg.values` contains every one of them; the distinct count is compared against the in-test truth set rather than a pinned number. | @@ -2126,9 +2094,9 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `cs_i32_regular_2x4_matches_golden` | The i32 counter type reaches the metadata, and an i32 golden is not an i64 sketch. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` cast to `i32` serializes to `asapv1_golden/cs_i32_regular_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, that the i32 and i64 regular fixtures have equal length yet differ, and that the i32 fixture fails to decode as `Count, RegularPath>`. | | `cs_i64_fast_2x4_matches_golden` | The fast-path mode reaches the bytes, so the same matrix encodes differently per path. | Verifies `Count::, FastPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` serializes to `asapv1_golden/cs_i64_fast_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, and that the fixture differs from `cs_i64_regular_2x4.hex` built from the same matrix. | | `cs_i64_regular_2x4_matches_golden` | A known signed Count-Sketch matrix serializes to its checked-in golden bytes. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` — sweeping the msgpack integer widths in both directions — serializes to `asapv1_golden/cs_i64_regular_2x4.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `4`, the eight cells, and a byte-identical re-serialization. | -| `hll_classic_p12_matches_golden` | A known P12 Classic register pattern serializes to its golden bytes on both the portable and native paths. | Builds `HllSketch::from_raw(HllVariant::Regular, 12, regs, 0.0, 0.0, 0.0)` where `regs` is 4,096 zeros with `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, verifies `to_msgpack()` equals `asapv1_golden/hll_classic_p12.hex`, that `HyperLogLogP12::::deserialize_from_bytes` on the fixture recovers those registers and re-serializes identically, and that `HllSketch::from_msgpack` reports the same registers, `precision` `12` and `HllVariant::Regular`. | -| `hll_ertl_mle_p12_matches_golden` | The Datafusion variant tag reaches the bytes for the same P12 register pattern. | Builds `HllSketch::from_raw(HllVariant::Datafusion, 12, regs, 0.0, 0.0, 0.0)` over the shared P12 pattern (`[0] = 1`, `[1] = 7`, `[100] = 42`, `[4095] = 3` in 4,096 registers), verifies `to_msgpack()` equals `asapv1_golden/hll_ertl_mle_p12.hex`, that `HyperLogLogP12::::deserialize_from_bytes` recovers the registers and re-serializes identically, and that `HllSketch::from_msgpack` reports `HllVariant::Datafusion`. | -| `hll_hip_p12_matches_golden` | The HIP variant's three running scalars travel on the wire beside its registers. | Builds `HllSketch::from_raw(HllVariant::Hip, 12, regs, 1.5, 2.5, 3.0)` over the shared P12 pattern, verifies `to_msgpack()` equals `asapv1_golden/hll_hip_p12.hex`, that `HyperLogLogHIPP12::deserialize_from_bytes` re-serializes the fixture identically, and that `HllSketch::from_msgpack` reports the registers, `HllVariant::Hip`, `hip_kxq0` `1.5`, `hip_kxq1` `2.5` and `hip_est` `3.0`. | +| `hll_classic_p12_matches_golden` | A known P12 Classic register pattern serializes to its golden bytes, which round-trip unchanged. | Builds `HyperLogLogP12::::from_storage` over 4,096 registers that are zero except `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_classic_p12.hex`, and that `deserialize_from_bytes` on the fixture recovers those registers and re-serializes identically. | +| `hll_ertl_mle_p12_matches_golden` | The ErtlMLE kind_id reaches the bytes for the same P12 register pattern. | Builds `HyperLogLogP12::::from_storage` over the shared P12 pattern, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_ertl_mle_p12.hex`, and that `deserialize_from_bytes` recovers the registers and re-serializes identically. | +| `hll_hip_p12_matches_golden` | The HIP variant's three running scalars travel on the wire beside its registers. | Builds `HyperLogLogHIPP12::from_storage` over the shared P12 pattern with `kxq0` `1.5`, `kxq1` `2.5` and `est` `3.0`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_hip_p12.hex`, that the decode's `Debug` form (registers and all three scalars) equals the built sketch's, and that it re-serializes identically. | | `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | @@ -2147,8 +2115,6 @@ Test file: [`tests/msgpack_compat.rs`](../tests/msgpack_compat.rs) | `dd_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | | `dd_sketch_round_trip` | A portable `DdSketch` survives a msgpack round trip carrying its full count in the bucket array. | Updates `DdSketch::new(0.01)` with `1.0`, `10.0` and `100.0`, then verifies the decode's `total_count()` — recovered by summing the buckets, since the count is not on the wire — is `3`; `alpha`, the bucket offset and the quantiles are not checked. | | `delta_result_round_trip` | A `DeltaResult` survives a msgpack round trip with both of its sets. | Serializes a `DeltaResult` whose `added` holds `a` and whose `removed` holds `b`, then verifies the decode's `added` contains `a` and its `removed` contains `b`. | -| `hll_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `hll_sketch_round_trip` | A portable `HllSketch` survives a msgpack round trip with its register count intact. | Updates `HllSketch::new(HllVariant::Regular, 8)` with `b"a"`, `b"b"` and `b"c"`, then verifies only that the decode's `registers` has the same length as the source's; the register values, the precision and the variant are not checked. | | `hydra_kll_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | | `hydra_kll_sketch_round_trip` | A portable `HydraKllSketch` survives a msgpack round trip with its grid shape intact. | Updates `HydraKllSketch::with_seed(2, 4, 200, 0x5EED_0900)` with `("a", 1.0)`, `("a", 2.0)` and `("b", 3.0)`, then verifies the decode reports `rows` `2` and `cols` `4`; the nested KLL payloads are not checked. | | `hydra_kll_wire_shape` | The `HydraKllSketchWire` DTO round-trips through `rmp_serde` with its two levels of nesting intact. | Serializes a wire struct of `rows: 2`, `cols: 3` and a `2x3` grid of `KllSketchData { k: 200, sketch_bytes: vec![] }`, then verifies the `rmp_serde` decode reports `rows` `2`, `cols` `3`, an outer length of `2` and an inner length of `3`. | @@ -2168,7 +2134,7 @@ Test file: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) | `countsketch_passes_signed_frequency_conformance` | `Count` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `Count::, RegularPath>::with_dimensions(5, 4096)` over the 60,000-draw Zipf stream through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0`, `turnstile_battery` on key `42` via `insert_many` (`+500 - 200` must read `300` within `1.0`, then cancel to within `1e-6`), and `merge_equivalence_battery` at the same spec. | | `ddsketch_passes_relative_quantile_conformance` | The portable `DdSketch` holds its relative-value-error promise under its own rank convention. | Runs `relative_quantile_battery` on `PortableDds::new(0.01)` over the positive samples of `normal_f64(40_000, 500.0, 80.0, 7001)`, requiring at q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` a relative error of at most `0.01` plus a few-ULP numerical slack against the exact `floor(q*(n-1))` order statistic. | | `heap_backed_frequency_sketches_pass_frequency_and_merge_conformance` | Both heap-backed matrix sketches pass the frequency and merge-equivalence batteries. | Over the 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`), drives `CMSHeap::, FastPath>::new(4, 4_096, 64)` through `frequency_battery` and `merge_equivalence_battery` one-sided at `rel_tol` `0.01` and `abs_tol` `8.0`, and `CSHeap::, RegularPath>::new(5, 4_096, 64)` through both two-sided at `rel_tol` `0.06` and `abs_tol` `25.0`. | -| `hll_variants_pass_cardinality_conformance` | All three native HyperLogLog estimators and the portable one estimate within 3%. | Runs `cardinality_battery` at `rel_tol` `0.03` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)` on `HyperLogLog`, `HyperLogLog`, `HyperLogLogHIP` and the portable `HllSketch::new(HllVariant::Regular, 14)`; each must estimate `100_000` within 3% and stay in band after the whole stream is replayed as duplicates. | +| `hll_variants_pass_cardinality_conformance` | All three HyperLogLog estimators estimate within 3%. | Runs `cardinality_battery` at `rel_tol` `0.03` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)` on `HyperLogLog`, `HyperLogLog` and `HyperLogLogHIP`; each must estimate `100_000` within 3% and stay in band after the whole stream is replayed as duplicates. | | `kll_family_passes_quantile_conformance` | The three KLL adapters hold the shared rank-error band. | Runs `quantile_battery` at the default `rank_tol` of `0.03` over q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` on the positive samples of `normal_f64(40_000, 500.0, 80.0, 7001)` for `KLL::init_kll_with_seed(200, 0x4017_0001)`, `KLLDynamic::::init_kll_with_seed(200, 0x4017_0001)`, and a `KLL` at the same `k` and seed queried through `quantile_cached`. | | `kmv_passes_cardinality_conformance` | `KMV` estimates a 100,000-key distinct count within 8%. | Compiled only under the `experimental` feature: runs `cardinality_battery` at `rel_tol` `0.08` on `KMV::new(4_096)` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)`, requiring the estimate in band both after the first pass and after the duplicate replay. | | `uniform_sampling_at_full_rate_passes_quantile_conformance` | A sampler retaining everything answers quantiles exactly. | Compiled only under the `experimental` feature: runs `quantile_battery` at `rank_tol` `0.0` on `UniformSampling::with_seed(1.0, 7_704)` over 20,000 Zipf draws (domain 4,096, exponent 1.1, seed `7_703`), with each quantile read from the sorted retained samples at index `ceil(q * len)`. | @@ -2194,7 +2160,6 @@ Test file: [`tests/sketches_go_parity_probe.rs`](../tests/sketches_go_parity_pro | `sketches_count_fastpath_matches_go_count_sketch_envelope` | A `sketches::Count` fast-path matrix encodes to the checked-in Count-Sketch envelope bytes. | Fills `Count, FastPath, DefaultXxHasher>` at `3x512` with 25 `insert_many` calls over the five keys `k-a`..`k-e`, packs the row-major cells and per-row L2 sums into a `CountSketchState` with `counter_type` `INT64` inside a `SketchEnvelope { format_version: 1 }` with no producer or hash_spec, and verifies the prost encoding equals the 1,577-byte fixture `tests/cs_envelope_golden.hex`. | | `sketches_countmin_fastpath_matches_go_count_min_envelope` | A `sketches::CountMin` fast-path matrix encodes to the checked-in Count-Min envelope bytes. | Fills `CountMin, FastPath, DefaultXxHasher>` at `4x2048` with 50 `insert_many` calls over the ten keys `flow-0`..`flow-9`, packs the cells cast to `i64` plus per-row L1 and L2 sums into a `CountMinState` with `counter_type` `INT64`, and verifies the prost encoding matches the 8,275-byte fixture `src/sketches/testdata/cms_envelope_golden.hex` in length and then byte for byte. | | `sketches_ddsketch_matches_go_envelope` | A `sketches::DDSketch` encodes to the DDSketch envelope hex constant held in the test file. | Fills `DDSketch::new(0.01)` with `1..=50` as `f64`, encodes a `DdSketchState` carrying `store_counts`, `store_offset` and an alpha round-tripped through gamma (`gamma = (1 + a) / (1 - a)`, `alpha_wire = (gamma - 1) / (gamma + 1)`), and verifies the envelope matches the 403-byte inline hex constant in length and then byte for byte; the comment on that constant records it as this crate's own output rather than a fixture from another implementation, so no cross-language parity is pinned. | -| `sketches_hll_classic_matches_go_envelope` | An ErtlMLE P14 register set encodes to the checked-in HyperLogLog envelope bytes. | Feeds `HyperLogLogImpl` the `hash64_seeded(CANONICAL_HASH_SEED, ...)` digests of the IEEE-754 little-endian bytes of `1.0..=50.0` through `insert_with_hash`, encodes a `HyperLogLogState` with `variant` `2`, `precision` `14`, the dense registers, zeroed HIP scalars and no sparse form, and verifies the envelope matches the 16,398-byte fixture `src/sketches/testdata/hll_envelope_golden.hex` in length and then byte for byte; despite the name, the variant under test is `ErtlMLE`, not `Classic`. | ### Cross-Language Proto Consumer @@ -2202,7 +2167,7 @@ Test file: [`tests/xtest_consumer.rs`](../tests/xtest_consumer.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes nine `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `hll.pb` an ErtlMLE cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | +| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes eight `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` exists, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | ### HLL Custom Precision diff --git a/examples/accuracy_probe.rs b/examples/accuracy_probe.rs index 643bfc3e..7265278f 100644 --- a/examples/accuracy_probe.rs +++ b/examples/accuracy_probe.rs @@ -9,7 +9,6 @@ use asap_sketchlib::common::input::{HydraCounter, HydraQuery}; use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketch; use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; use asap_sketchlib::message_pack_format::portable::kll::KllSketch as PortableKll; use asap_sketchlib::{ @@ -1043,21 +1042,6 @@ fn probe_portable_wire_types() { bad == 0, ); - // Portable HLL precision 12. - let mut ph = HllSketch::new(HllVariant::Regular, 12); - for i in 0..100_000u64 { - ph.update(i.to_le_bytes().as_slice()); - } - let est = ph.estimate(); - p.check( - "portable HLL p12 within 3%", - format!( - "expected 100000, got {est:.0} (rel {:.4})", - rel_err(est, 100_000.0) - ), - rel_err(est, 100_000.0) < 0.03, - ); - // Portable KLL. let mut pk = PortableKll::new(200); let mut rng = StdRng::seed_from_u64(55); diff --git a/proto/hll/hll.proto b/proto/hll/hll.proto index 1cdf6124..eba61943 100644 --- a/proto/hll/hll.proto +++ b/proto/hll/hll.proto @@ -1,4 +1,4 @@ -// hll.proto — Portable wire format for HyperLogLog sketch state. +// hll.proto — HyperLogLog cell state for HydraCell (hydra/hydra.proto). syntax = "proto3"; @@ -97,38 +97,3 @@ enum HLLVariant { // Requires hip_kxq0, hip_kxq1, hip_est to be populated. HLL_VARIANT_HIP = 3; } - -// ============================================================================ -// HyperLogLog Delta -// ============================================================================ - -// HLLDelta carries the registers that increased between two consecutive -// snapshots. HLL uses max semantics, so only increases are meaningful; -// at a fixed precision a register can never decrease. Each update is applied -// losslessly on the receiver via register[index] = max(register[index], value) -// — no register update is ever dropped (there is no threshold to apply). -// -// The increased registers are varint-packed exactly like -// HLLSparseRegisters.packed: sorted ascending by index, each register emitted -// as (index_delta, value) where index_delta = index - prev_index (prev_index -// starts at 0) and value is the new (larger) register value: -// -// for each increased register, in ascending index order: -// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -// uvarint(value) // 1..=Q+1, always 1 byte in practice -// -// A single-emit delta against the all-zero snapshot is therefore the same size -// as the full sparse frame, and a sub-window delta is strictly smaller (it -// packs only the registers that actually grew). This replaces the previous -// per-register sub-message encoding, eliminating ~6–8 bytes of tag/length -// overhead per register. -// -// This message is byte-identical to the Go reference implementation's HLLDelta -// so the two runtimes emit byte-identical delta frames for identical window -// state (cross-language byte parity). -message HLLDelta { - // Varint-packed (index_delta, value) pairs for the registers that increased, - // in ascending index order. Same layout as HLLSparseRegisters.packed. Empty - // when no register changed. - bytes packed_updates = 1; -} diff --git a/proto/sketchlib.proto b/proto/sketchlib.proto index 0a6d8193..d6f253b9 100644 --- a/proto/sketchlib.proto +++ b/proto/sketchlib.proto @@ -6,7 +6,6 @@ // common/common.proto — HashSpec, ProducerInfo, CounterType, ... // countminsketch/countminsketch.proto — CountMinState // countsketch/countsketch.proto — CountSketchState, TopKState, HeapEntry -// hll/hll.proto — HyperLogLogState, HLLVariant // kll/kll.proto — KLLState, CoinState // ddsketch/ddsketch.proto — DDSketchState // univmon/univmon.proto — UnivMonState, UnivMonLayer @@ -31,7 +30,6 @@ package sketchlib.v1; import "common/common.proto"; import "countminsketch/countminsketch.proto"; import "countsketch/countsketch.proto"; -import "hll/hll.proto"; import "kll/kll.proto"; import "ddsketch/ddsketch.proto"; import "univmon/univmon.proto"; @@ -68,7 +66,6 @@ message SketchEnvelope { oneof sketch_state { CountMinState count_min = 10; CountSketchState count_sketch = 11; - HyperLogLogState hll = 12; KLLState kll = 13; DDSketchState ddsketch = 14; UnivMonState univmon = 15; @@ -77,6 +74,8 @@ message SketchEnvelope { ElasticState elastic = 18; } + reserved 12; + reserved "hll"; reserved 19 to 31; // Reserved for future sketch types. // Geometric / NitroSketch sampling probability for the producing sketch. diff --git a/src/lib.rs b/src/lib.rs index b062b8b9..3aca2dda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,7 @@ //! UnivMon, tumbling windows, and batch/parallel execution helpers. //! - [`message_pack_format`]: MessagePack/proto wire format shared with //! `sketchlib-go`. Owns the wire-DTO sketch types consumed by the ASAP -//! query engine (`CountMinSketch`, `CountSketch`, `KllSketch`, `HllSketch`, +//! query engine (`CountMinSketch`, `CountSketch`, `KllSketch`, //! `DdSketch`, `HydraKllSketch`, `CountMinSketchWithHeap`, //! `SetAggregator`, `DeltaResult`). //! @@ -49,7 +49,6 @@ pub use message_pack_format::portable::countsketch::{ pub use message_pack_format::portable::countsketch_topk::{CountSketchWithHeap, CsHeapItem}; pub use message_pack_format::portable::ddsketch::{DDSKETCH_GROW_CHUNK, DdSketch, DdSketchDelta}; pub use message_pack_format::portable::delta_set_aggregator::DeltaResult; -pub use message_pack_format::portable::hll::{HllSketch, HllSketchDelta, HllVariant}; pub use message_pack_format::portable::hydra_kll::HydraKllSketch; pub use message_pack_format::portable::kll::{KllSketch, KllSketchData}; pub use message_pack_format::portable::sampling::{ diff --git a/src/message_pack_format/native/hll.rs b/src/message_pack_format/native/hll.rs deleted file mode 100644 index b5de3118..00000000 --- a/src/message_pack_format/native/hll.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Native MessagePack codec impls for [`crate::sketches::hll`]. - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::hll::{HllWireVariant, HyperLogLogHIPImpl, HyperLogLogImpl}; -use crate::{HashProfile, HllRegisterStorage, SketchHasher}; - -impl - MessagePackCodec for HyperLogLogImpl -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.serialize_to_bytes()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::deserialize_from_bytes(bytes)?) - } -} - -impl MessagePackCodec for HyperLogLogHIPImpl { - fn to_msgpack(&self) -> Result, Error> { - Ok(self.serialize_to_bytes()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::deserialize_from_bytes(bytes)?) - } -} diff --git a/src/message_pack_format/native/mod.rs b/src/message_pack_format/native/mod.rs index 0a83ef96..91f3493b 100644 --- a/src/message_pack_format/native/mod.rs +++ b/src/message_pack_format/native/mod.rs @@ -11,7 +11,6 @@ pub mod countminsketch; pub mod countsketch; pub mod countsketch_topk; pub mod ddsketch; -pub mod hll; pub mod kll; pub mod kll_dynamic; #[cfg(feature = "experimental")] diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs deleted file mode 100644 index b70dfe80..00000000 --- a/src/message_pack_format/portable/hll.rs +++ /dev/null @@ -1,1056 +0,0 @@ -//! Wire-format-aligned HyperLogLog types. -//! -//! Self-contained: maintains its own `Vec` register array. Parity -//! with Go's `HyperLogLog.SerializePortable` (and equivalence with -//! `sketches::HyperLogLogImpl<.., ErtlMLE, P14>`) is locked in by the -//! `test_update_then_envelope_matches_sketchlib_go_bytes` test below and -//! by `tests/sketches_go_parity_probe.rs`. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::{CANONICAL_HASH_SEED, DataInput, hash64_seeded}; - -// --------------------------------------------------------------------------- -// DEPRECATED — temporary compatibility shim, slated for removal. -// -// A few call sites still depend on this portable `HllSketch`. Its ASAPv1 codec -// now reuses the shared `envelope` framing plus the HLL metadata/payload types -// in `crate::sketches::hll`, so it stays byte-identical with the native -// `HyperLogLogImpl` / `HyperLogLogHIPImpl` serialization. Do NOT add new callers. -// -// Prefer instead the native path: -// `sketches::hll::HyperLogLogImpl::serialize_to_bytes` / -// `HyperLogLogHIPImpl::serialize_to_bytes` (+ `deserialize_from_bytes`). -// --------------------------------------------------------------------------- - -use crate::message_pack_format::envelope; -use crate::sketches::hll::{ - HLL_KIND_CLASSIC, HLL_KIND_ERTL_MLE, HLL_KIND_HIP, HllMetadata, HllPayloadHip, HllPayloadPlain, - standard_hll_metadata, -}; - -/// Retained only for `test_msgpack_round_trip`'s magic-prefix assertion. -#[cfg(test)] -const HLL_WRAPPER_MAGIC: &[u8; 6] = b"ASAPv1"; - -/// Map a portable [`HllVariant`] to its ASAPv1 wire `kind_id`. `Unspecified` is -/// a placeholder with no wire form. -fn wire_kind_id(variant: HllVariant) -> Result<&'static [u8], MsgPackError> { - match variant { - HllVariant::Regular => Ok(HLL_KIND_CLASSIC), - HllVariant::Datafusion => Ok(HLL_KIND_ERTL_MLE), - HllVariant::Hip => Ok(HLL_KIND_HIP), - HllVariant::Unspecified => Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized( - "ASAPv1 HLL: the Unspecified variant is not serializable".to_string(), - ), - )), - } -} - -/// HLL estimator variant. Mirrors `asap_sketchlib::proto::sketchlib::HllVariant`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum HllVariant { - Unspecified, - Regular, - Datafusion, - Hip, -} - -/// Sparse delta between two consecutive HLL snapshots — input shape for -/// [`HllSketch::apply_delta`]. Mirrors the `HLLDelta` proto in -/// `sketchlib-go/proto/hll/hll.proto`. -#[derive(Debug, Clone, Default)] -pub struct HllSketchDelta { - pub updates: Vec<(u32, u8)>, -} - -/// Minimal HLL state — registers + variant + precision. Register-wise -/// mergeable (max over aligned cells). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HllSketch { - pub variant: HllVariant, - pub precision: u32, - pub registers: Vec, - pub hip_kxq0: f64, - pub hip_kxq1: f64, - pub hip_est: f64, -} - -impl HllSketch { - pub fn new(variant: HllVariant, precision: u32) -> Self { - let n = 1usize << precision; - Self { - variant, - precision, - registers: vec![0u8; n], - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - } - } - - pub fn from_raw( - variant: HllVariant, - precision: u32, - registers: Vec, - hip_kxq0: f64, - hip_kxq1: f64, - hip_est: f64, - ) -> Self { - Self { - variant, - precision, - registers, - hip_kxq0, - hip_kxq1, - hip_est, - } - } - - pub fn merge( - &mut self, - other: &HllSketch, - ) -> Result<(), Box> { - if self.variant != other.variant { - return Err(format!( - "HllSketch variant mismatch: self={:?}, other={:?}", - self.variant, other.variant - ) - .into()); - } - if self.precision != other.precision { - return Err(format!( - "HllSketch precision mismatch: self={}, other={}", - self.precision, other.precision - ) - .into()); - } - if self.registers.len() != other.registers.len() { - return Err(format!( - "HllSketch register-length mismatch: self={}, other={}", - self.registers.len(), - other.registers.len() - ) - .into()); - } - for (s, o) in self.registers.iter_mut().zip(other.registers.iter()) { - if *o > *s { - *s = *o; - } - } - if self.variant == HllVariant::Hip { - self.hip_kxq0 += other.hip_kxq0; - self.hip_kxq1 += other.hip_kxq1; - self.hip_est += other.hip_est; - } - Ok(()) - } - - pub fn apply_delta( - &mut self, - delta: &HllSketchDelta, - ) -> Result<(), Box> { - let n = self.registers.len(); - for (idx, new_val) in &delta.updates { - let i = *idx as usize; - if i >= n { - return Err(format!( - "HllSketchDelta index {i} out of range (precision={} → {n} registers)", - self.precision - ) - .into()); - } - if *new_val > self.registers[i] { - self.registers[i] = *new_val; - } - } - Ok(()) - } - - /// Compute a sparse, proto-marshalled `HLLDelta` of `self` against a - /// `snapshot`. A register update is included when `self`'s register - /// value increased over the snapshot's (`self[i] > snapshot[i]`); its - /// carried value is the new (larger) register value. - /// - /// This is the Rust twin of the Go reference implementation's - /// `ComputeRegisterDelta` + `SerializeRegisterDelta`. HLL uses max - /// semantics, so only increases are meaningful and the delta is - /// **lossless** — every increased register is carried, regardless of - /// `threshold` (the parameter is accepted for a uniform delta API and - /// has no effect here; an HLL register update is never dropped). The - /// returned bytes are a `prost`-encoded - /// [`crate::proto::sketchlib::HllDelta`], byte-identical to the Go - /// `proto.Marshal(HLLDelta)` output for the same inputs (cross-language - /// byte parity). - /// - /// Delta-against-empty: when `snapshot` is the all-zero sketch, every - /// non-zero register of `self` is carried, so the result is this - /// window's full register state encoded as a delta (no cross-window - /// subtraction). HLL deltas carry only register max-updates — there are - /// no DataPoint-level metric scalars to drop. - pub fn compute_delta(&self, snapshot: &HllSketch, _threshold: u64) -> Vec { - use crate::proto::sketchlib::HllDelta as ProtoDelta; - use prost::Message; - - let cur = &self.registers; - let snap = &snapshot.registers; - let n = cur.len().min(snap.len()); - - // Varint-pack the increased registers as (index_delta, value) pairs in - // ascending index order — the same layout `encode_sparse_registers` - // uses for the full sparse state. Walking registers in index order - // yields a sorted sequence, so `prev` only ever advances. - let mut packed: Vec = Vec::new(); - let mut prev: usize = 0; - for i in 0..n { - if cur[i] > snap[i] { - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, cur[i] as u64); - prev = i; - } - } - // Guard: if `self` has more registers than the snapshot (should not - // happen at a fixed precision), carry all non-zero extras. Matches - // the Go reference's trailing-register guard. - for (i, &v) in cur.iter().enumerate().take(cur.len()).skip(n) { - if v > 0 { - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, v as u64); - prev = i; - } - } - - ProtoDelta { - packed_updates: packed, - } - .encode_to_vec() - } - - /// Apply a `prost`-encoded [`crate::proto::sketchlib::HllDelta`] to this - /// sketch in place (register max-merge). The Rust twin of the Go - /// reference implementation's `DeserializeRegisterDelta` + - /// `ApplyRegisterDelta`: each update sets - /// `register[index] = max(register[index], value)`. - /// - /// Returns `Err` if `bytes` is not a valid `HLLDelta` proto or a - /// register index is out of range for this sketch's precision. - pub fn apply_delta_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - use crate::proto::sketchlib::HllDelta as ProtoDelta; - use prost::Message; - - let proto = ProtoDelta::decode(bytes)?; - - // Unpack the (index_delta, value) blob — inverse of `compute_delta`, - // same layout as `decode_sparse_registers` but bounded by the blob's - // own contents rather than a fixed register count. - let packed = &proto.packed_updates; - let mut updates: Vec<(u32, u8)> = Vec::new(); - let mut off = 0usize; - let mut prev: u32 = 0; - let mut first = true; - while off < packed.len() { - let (delta, used) = - read_uvarint(&packed[off..]).ok_or("hll: delta index-delta varint corrupt")?; - off += used; - let (val, used) = - read_uvarint(&packed[off..]).ok_or("hll: delta value varint corrupt")?; - off += used; - - let idx = prev + delta as u32; - // First delta is an absolute index (prev=0); later deltas must be - // > 0 so indices stay strictly increasing and unique. - if !first && delta == 0 { - return Err(format!("hll: delta non-increasing index at idx {idx}").into()); - } - first = false; - if val > 0xff { - return Err(format!("hll: delta value {val} exceeds u8 at idx {idx}").into()); - } - updates.push((idx, val as u8)); - prev = idx; - } - - let delta = HllSketchDelta { updates }; - self.apply_delta(&delta) - } - - /// MessagePack twin of [`Self::compute_delta`]. Carries every register - /// that increased over `snapshot` (`self[i] > snapshot[i]`) as an - /// `(index, new_value)` pair, serialized as the parallel-array - /// MessagePack layout via [`HllSketchDelta`]'s [`MessagePackCodec`] - /// instead of proto. HLL uses max semantics, so the delta is lossless - /// regardless of `_threshold` (accepted for a uniform delta API). - /// Delta-against-empty carries every non-zero register — this window's - /// full register state encoded as a delta. - pub fn compute_delta_msgpack(&self, snapshot: &HllSketch, _threshold: u64) -> Vec { - let cur = &self.registers; - let snap = &snapshot.registers; - let n = cur.len().min(snap.len()); - let mut updates: Vec<(u32, u8)> = Vec::new(); - for i in 0..n { - if cur[i] > snap[i] { - updates.push((i as u32, cur[i])); - } - } - // Guard: registers present in `self` beyond the snapshot length - // (should not happen at a fixed precision) carry their non-zero - // values. Matches the proto path's trailing-register guard. - for (i, &v) in cur.iter().enumerate().skip(n) { - if v > 0 { - updates.push((i as u32, v)); - } - } - HllSketchDelta { updates } - .to_msgpack() - .expect("HllSketchDelta msgpack encode is infallible for owned update arrays") - } - - /// MessagePack twin of [`Self::apply_delta_bytes`]. Decodes the - /// parallel-array MessagePack [`HllSketchDelta`] and applies it in - /// place (register max-merge). - /// - /// Returns `Err` if `bytes` is not a valid MessagePack `HllSketchDelta` - /// or a register index is out of range for this sketch's precision. - pub fn apply_delta_msgpack_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - let delta = HllSketchDelta::from_msgpack(bytes)?; - self.apply_delta(&delta) - } - - pub fn merge_refs( - inputs: &[&HllSketch], - ) -> Result> { - let first = inputs - .first() - .ok_or("HllSketch::merge_refs called with empty input")?; - let mut merged = HllSketch::new(first.variant, first.precision); - for hll in inputs { - merged.merge(hll)?; - } - Ok(merged) - } - - /// Insert a value. Math mirrors `sketches::HyperLogLogImpl::insert_with_hash` - /// (CANONICAL_HASH_SEED, packed bucket/leading-zero); locked in by - /// `tests/sketches_go_parity_probe.rs`. - pub fn update(&mut self, value: &[u8]) { - let hashed_val = hash64_seeded(CANONICAL_HASH_SEED, &DataInput::Bytes(value)); - let p = self.precision as usize; - let register_bits = (u64::BITS as usize) - p; - let p_mask: u64 = (1u64 << p) - 1; - let bucket_num = ((hashed_val >> register_bits) & p_mask) as usize; - let leading_zero = ((hashed_val << p) + p_mask).leading_zeros() as u8 + 1; - if bucket_num < self.registers.len() && leading_zero > self.registers[bucket_num] { - self.registers[bucket_num] = leading_zero; - } - } - - /// Estimate cardinality (Classic HLL estimator with the small-range - /// correction). Ranks are drawn from a 64-bit hash, so the estimate needs - /// no large-range correction. - pub fn estimate(&self) -> f64 { - let m = self.registers.len() as f64; - if m == 0.0 { - return 0.0; - } - let mut z = 0.0_f64; - let mut zero_count = 0usize; - for ®_val in &self.registers { - if reg_val == 0 { - zero_count += 1; - } - z += 2f64.powi(-(reg_val as i32)); - } - let indicator = 1.0 / z; - - let alpha_m = 0.7213 / (1.0 + 1.079 / m); - let mut est = alpha_m * m * m * indicator; - - if est <= m * 5.0 / 2.0 && zero_count != 0 { - est = m * (m / zero_count as f64).ln(); - } - est - } - - /// Return the proto-enum value Go's `HyperLogLog.SerializePortable` - /// emits for this sketch's [`HllVariant`]. - #[inline] - pub fn wire_proto_variant(&self) -> i32 { - match self.variant { - HllVariant::Unspecified => 0, - HllVariant::Regular => 1, - HllVariant::Datafusion => 2, - HllVariant::Hip => 3, - } - } -} - -// --------------------------------------------------------------------------- -// Proto dual-read decode (DENSE tag 3 + SPARSE tag 7) -// --------------------------------------------------------------------------- - -use crate::proto::sketchlib::{HllSparseRegisters, HyperLogLogState}; - -/// Decode the full dense register array from a `HyperLogLogState`, accepting -/// BOTH wire encodings emitted by sketchlib-go: -/// - DENSE (`registers`, tag 3): raw 1-byte-per-register array, or -/// - SPARSE (`registers_sparse`, tag 7): HLL++ style varint-packed -/// (index_delta, value) pairs for non-zero registers only. -/// -/// The SPARSE field takes priority when present; otherwise the DENSE field is -/// used. Both reconstruct the identical `2^precision`-byte register array, so -/// downstream estimation / merge is encoding-agnostic. This is the Rust half of -/// the backend-decode-first rollout: it must ship before any producer emits -/// sparse. Mirrors the Go decoder in -/// `sketchlib-go/sketches/HLL/portable.go::registersFromState` + -/// `sparse.go::decodeSparseRegisters`. -pub fn registers_from_state( - state: &HyperLogLogState, -) -> Result, Box> { - if let Some(sp) = &state.registers_sparse { - return decode_sparse_registers(sp); - } - Ok(state.registers.clone()) -} - -/// Reconstruct the dense register array from a sparse message. Inverse of the -/// Go encoder `sparse.go::encodeSparseRegisters`. -pub fn decode_sparse_registers( - sp: &HllSparseRegisters, -) -> Result, Box> { - let n = sp.num_registers as usize; - let mut regs = vec![0u8; n]; - - let packed = &sp.packed; - let mut off = 0usize; - let mut prev: usize = 0; - let mut first = true; - while off < packed.len() { - let (delta, used) = - read_uvarint(&packed[off..]).ok_or("hll: sparse index-delta varint corrupt")?; - off += used; - let (val, used) = read_uvarint(&packed[off..]).ok_or("hll: sparse value varint corrupt")?; - off += used; - - let idx = prev + delta as usize; - // First delta is an absolute index (prev=0); later deltas must be > 0 - // so indices stay strictly increasing and unique. - if !first && delta == 0 { - return Err(format!("hll: sparse non-increasing index at idx {idx}").into()); - } - first = false; - if idx >= n { - return Err(format!("hll: sparse index {idx} out of range [0,{n})").into()); - } - if val > 0xff { - return Err(format!("hll: sparse value {val} exceeds u8 at idx {idx}").into()); - } - regs[idx] = val as u8; - prev = idx; - } - Ok(regs) -} - -/// Encode the non-zero registers of `regs` into an [`HllSparseRegisters`] -/// message, byte-identical to the Go encoder `sparse.go::encodeSparseRegisters`. -/// Provided so a Rust producer (or the reverse-direction golden test) can emit -/// the same sparse wire form Go does. -pub fn encode_sparse_registers(regs: &[u8]) -> HllSparseRegisters { - let mut packed: Vec = Vec::new(); - let mut prev: usize = 0; - for (i, &v) in regs.iter().enumerate() { - if v == 0 { - continue; - } - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, v as u64); - prev = i; - } - HllSparseRegisters { - num_registers: regs.len() as u32, - packed, - } -} - -/// Append one unsigned LEB128 varint to `out`. Matches Go's -/// `encoding/binary.PutUvarint`. -fn write_uvarint(out: &mut Vec, mut v: u64) { - while v >= 0x80 { - out.push((v as u8) | 0x80); - v >>= 7; - } - out.push(v as u8); -} - -/// Read one unsigned LEB128 varint, returning (value, bytes_consumed). Matches -/// Go's `encoding/binary.Uvarint`. Returns None on truncation / overflow. -fn read_uvarint(buf: &[u8]) -> Option<(u64, usize)> { - let mut result: u64 = 0; - let mut shift = 0u32; - for (i, &b) in buf.iter().enumerate() { - if i >= 10 { - return None; // varint too long for u64 - } - if shift == 63 && b > 1 { - return None; // overflow - } - result |= ((b & 0x7f) as u64) << shift; - if b < 0x80 { - return Some((result, i + 1)); - } - shift += 7; - } - None -} - -impl MessagePackCodec for HllSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let kind_id = wire_kind_id(self.variant)?; - let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(self.precision))?; - let payload = if self.variant == HllVariant::Hip { - rmp_serde::to_vec(&HllPayloadHip { - registers: self.registers.clone(), - hip_kxq0: self.hip_kxq0, - hip_kxq1: self.hip_kxq1, - hip_est: self.hip_est, - })? - } else { - rmp_serde::to_vec(&HllPayloadPlain { - registers: self.registers.clone(), - })? - }; - Ok(envelope::encode(kind_id, &metadata, &payload)) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, metadata, payload) = envelope::split(bytes) - .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; - - // Validate the hash spec is the standard profile (precision is read from - // the metadata, so compare against a standard block built with it). - let meta: HllMetadata = rmp_serde::from_slice(metadata)?; - if meta != standard_hll_metadata(meta.precision) { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized( - "ASAPv1 HLL envelope: hash metadata mismatch".to_string(), - ), - )); - } - - let variant = if kind_id == HLL_KIND_CLASSIC { - HllVariant::Regular - } else if kind_id == HLL_KIND_ERTL_MLE { - HllVariant::Datafusion - } else if kind_id == HLL_KIND_HIP { - HllVariant::Hip - } else { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: unsupported kind_id {kind_id:?}" - )), - )); - }; - - let (registers, hip_kxq0, hip_kxq1, hip_est) = if variant == HllVariant::Hip { - let p: HllPayloadHip = rmp_serde::from_slice(payload)?; - (p.registers, p.hip_kxq0, p.hip_kxq1, p.hip_est) - } else { - let p: HllPayloadPlain = rmp_serde::from_slice(payload)?; - (p.registers, 0.0, 0.0, 0.0) - }; - - // Fail closed if the register bin does not match `2^precision` - // (doc §2 validation rule 3). `checked_shl` avoids a shift-overflow - // panic on a crafted out-of-range `precision`. - let expected = 1usize.checked_shl(meta.precision).ok_or_else(|| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: precision {} out of range", - meta.precision - ))) - })?; - if registers.len() != expected { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: {} registers, expected 2^{} = {expected}", - registers.len(), - meta.precision - )), - )); - } - - Ok(HllSketch::from_raw( - variant, - meta.precision, - registers, - hip_kxq0, - hip_kxq1, - hip_est, - )) - } -} - -impl MessagePackCodec for HllSketchDelta { - /// Encodes the register delta as the 2-element MessagePack array - /// `[ reg_idx:[]u32, reg_val:[]u8 ]` — parallel arrays of register - /// index → new (max) value — byte-identical to Rust - /// `rmp_serde::to_vec(&(Vec, Vec))` (compact mode). The - /// `Vec` serializes as a msgpack array-of-int (not `bin`), matching - /// the Go `asapmsgpack` register-array convention. - fn to_msgpack(&self) -> Result, MsgPackError> { - let reg_idx: Vec = self.updates.iter().map(|(i, _)| *i).collect(); - let reg_val: Vec = self.updates.iter().map(|(_, v)| *v).collect(); - Ok(rmp_serde::to_vec(&(reg_idx, reg_val))?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (reg_idx, reg_val): (Vec, Vec) = rmp_serde::from_slice(bytes)?; - Ok(HllSketchDelta { - updates: reg_idx.into_iter().zip(reg_val).collect(), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Ranks are drawn from a 64-bit hash, so a register state whose estimate - /// runs past the 32-bit range must still follow the HyperLogLog formula - /// rather than saturate or turn into a NaN. - #[test] - fn estimate_holds_past_the_32_bit_range() { - for rank in [12u8, 16, 18, 20] { - let mut sketch = HllSketch::new(HllVariant::Regular, 14); - sketch.registers.fill(rank); - let m = sketch.registers.len() as f64; - let alpha_m = 0.7213 / (1.0 + 1.079 / m); - let expected = alpha_m * m * 2f64.powi(rank as i32); - let estimate = sketch.estimate(); - let error = (estimate - expected).abs() / expected; - assert!( - error <= 1e-9, - "rank {rank} estimate {estimate} deviates from {expected} by {error:.6}" - ); - } - } - - #[test] - fn msgpack_delta_against_empty_round_trips() { - let mut w = HllSketch::new(HllVariant::Regular, 12); - for i in 0..2000u64 { - w.update(&i.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Regular, 12); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut recon = HllSketch::new(HllVariant::Regular, 12); - recon.apply_delta_msgpack_bytes(&bytes).unwrap(); - // Register delta over an empty base carries every non-zero register, - // so reconstructed registers are identical. - assert_eq!(recon.registers, w.registers); - let (got, want) = (recon.estimate(), w.estimate()); - assert!((got - want).abs() <= want * 0.001, "got={got} want={want}"); - } - - #[test] - fn msgpack_delta_wire_layout_is_parallel_arrays() { - use crate::message_pack_format::MessagePackCodec; - let mut w = HllSketch::new(HllVariant::Regular, 4); - w.update(b"a"); - w.update(b"b"); - let empty = HllSketch::new(HllVariant::Regular, 4); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut reg_idx: Vec = Vec::new(); - let mut reg_val: Vec = Vec::new(); - for (i, &v) in w.registers.iter().enumerate() { - if v > 0 { - reg_idx.push(i as u32); - reg_val.push(v); - } - } - let expected = rmp_serde::to_vec(&(reg_idx, reg_val)).unwrap(); - assert_eq!(bytes, expected); - let d = HllSketchDelta::from_msgpack(&bytes).unwrap(); - assert_eq!( - d.updates.len(), - w.registers.iter().filter(|&&v| v > 0).count() - ); - } - - #[test] - fn test_new_empty() { - let h = HllSketch::new(HllVariant::Regular, 4); - assert_eq!(h.registers.len(), 16); - assert!(h.registers.iter().all(|&r| r == 0)); - } - - #[test] - fn test_merge_register_wise_max() { - let mut a = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let b = HllSketch::from_raw(HllVariant::Regular, 2, vec![4, 2, 6, 0], 0.0, 0.0, 0.0); - a.merge(&b).unwrap(); - assert_eq!(a.registers, vec![4, 5, 6, 7]); - } - - #[test] - fn test_apply_delta_max_semantics() { - let mut h = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let delta = HllSketchDelta { - updates: vec![(0, 4), (1, 2), (2, 6), (3, 0)], - }; - h.apply_delta(&delta).unwrap(); - assert_eq!(h.registers, vec![4, 5, 6, 7]); - } - - #[test] - fn test_apply_delta_out_of_range() { - let mut h = HllSketch::new(HllVariant::Regular, 2); - let delta = HllSketchDelta { - updates: vec![(7, 3)], - }; - assert!(h.apply_delta(&delta).is_err()); - } - - #[test] - fn test_apply_delta_matches_full_merge() { - let base = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let addition = HllSketch::from_raw(HllVariant::Regular, 2, vec![4, 0, 6, 0], 0.0, 0.0, 0.0); - let mut via_merge = base.clone(); - via_merge.merge(&addition).unwrap(); - - let delta = HllSketchDelta { - updates: vec![(0, 4), (2, 6)], - }; - let mut via_delta = base; - via_delta.apply_delta(&delta).unwrap(); - assert_eq!(via_delta.registers, via_merge.registers); - } - - #[test] - fn test_merge_variant_mismatch() { - let mut a = HllSketch::new(HllVariant::Regular, 4); - let b = HllSketch::new(HllVariant::Datafusion, 4); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_merge_precision_mismatch() { - let mut a = HllSketch::new(HllVariant::Regular, 4); - let b = HllSketch::new(HllVariant::Regular, 5); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_merge_refs() { - let a = HllSketch::from_raw(HllVariant::Regular, 1, vec![1, 0], 0.0, 0.0, 0.0); - let b = HllSketch::from_raw(HllVariant::Regular, 1, vec![0, 3], 0.0, 0.0, 0.0); - let c = HllSketch::from_raw(HllVariant::Regular, 1, vec![2, 2], 0.0, 0.0, 0.0); - let merged = HllSketch::merge_refs(&[&a, &b, &c]).unwrap(); - assert_eq!(merged.registers, vec![2, 3]); - } - - #[test] - fn test_update_then_estimate_within_2pct() { - let n: usize = 50_000; - let mut h = HllSketch::new(HllVariant::Regular, 12); - for i in 0..n { - let key = format!("key-{i}"); - h.update(key.as_bytes()); - } - let est = h.estimate(); - let rel_err = (est - n as f64).abs() / n as f64; - assert!( - rel_err < 0.02, - "HLL estimate {est} not within 2% of {n} (rel_err {rel_err:.4})", - ); - } - - #[test] - fn test_estimate_empty_is_zero() { - let h = HllSketch::new(HllVariant::Regular, 4); - assert_eq!(h.estimate(), 0.0); - } - - #[test] - fn test_msgpack_round_trip() { - let original = HllSketch::from_raw( - HllVariant::Hip, - 3, - vec![0, 1, 2, 3, 4, 5, 6, 7], - 1.0, - 2.0, - 3.0, - ); - let bytes = original.to_msgpack().unwrap(); - assert!(bytes.starts_with(HLL_WRAPPER_MAGIC)); - let decoded = HllSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(decoded.registers, original.registers); - assert_eq!(decoded.precision, original.precision); - assert_eq!(decoded.hip_kxq0, 1.0); - } - - /// Cross-language byte-parity guard against - /// `sketchlib-go::HyperLogLog.SerializePortable` for the - /// deterministic input stream `(1..=50i32).map(|i| (i as f64) - /// .to_le_bytes())`. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - HyperLogLogState, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let mut sk = HllSketch::new(HllVariant::Datafusion, 14); - for i in 1..=50i32 { - let v = i as f64; - sk.update(&v.to_le_bytes()); - } - - let state = HyperLogLogState { - variant: sk.wire_proto_variant(), - precision: sk.precision, - registers: sk.registers.clone(), - hip_kxq0: sk.hip_kxq0, - hip_kxq1: sk.hip_kxq1, - hip_est: sk.hip_est, - // This parity golden predates the sparse field and intentionally - // pins the DENSE wire form, so leave the sparse field unset. - registers_sparse: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - // Golden pins the pre-sampling wire form; 0.0 is the proto3 default - // (dual-read as 1.0) so the encoded bytes are unchanged. - sample_p: 0.0, - sketch_state: Some(SketchState::Hll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - const GOLDEN_HEX: &str = include_str!("../../sketches/testdata/hll_envelope_golden.hex"); - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "HLL envelope length differs: got {} bytes, want {} bytes", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "HLL envelope bytes diverge from sketchlib-go golden" - ); - } - - fn decode_hex(s: &str) -> Vec { - let s = s.trim(); - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - // ----- sparse full-state encoding ----- - - use crate::proto::sketchlib::HyperLogLogState; - - /// A handful of (index, value) pairs that exercise small and large index - /// deltas plus the maximum register value. - fn sample_regs(n: usize) -> Vec { - let mut regs = vec![0u8; n]; - regs[0] = 1; // first delta is absolute index 0 - regs[1] = 7; // adjacent index (delta 1) - regs[300] = 51; // max p=14 register value (Q+1) - regs[8191] = 12; // mid-range, multi-byte index delta - regs[n - 1] = 3; // last register - regs - } - - #[test] - fn sparse_round_trip_exact() { - let regs = sample_regs(16384); - let sp = encode_sparse_registers(®s); - assert_eq!(sp.num_registers, 16384); - let decoded = decode_sparse_registers(&sp).unwrap(); - assert_eq!(decoded, regs, "sparse round trip must be exact"); - } - - #[test] - fn registers_from_state_reads_both() { - let regs = sample_regs(16384); - - // SPARSE: registers_sparse set, registers empty. - let sparse_state = HyperLogLogState { - variant: 2, - precision: 14, - registers: Vec::new(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: Some(encode_sparse_registers(®s)), - }; - assert_eq!(registers_from_state(&sparse_state).unwrap(), regs); - - // DENSE: registers set, registers_sparse None. - let dense_state = HyperLogLogState { - variant: 2, - precision: 14, - registers: regs.clone(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: None, - }; - assert_eq!(registers_from_state(&dense_state).unwrap(), regs); - } - - #[test] - fn sparse_empty_is_all_zero() { - let regs = vec![0u8; 16384]; - let sp = encode_sparse_registers(®s); - assert!(sp.packed.is_empty()); - assert_eq!(decode_sparse_registers(&sp).unwrap(), regs); - } - - #[test] - fn sparse_rejects_out_of_range_index() { - // packed = uvarint(delta=5) , uvarint(value=3) but num_registers=4. - let sp = HllSparseRegisters { - num_registers: 4, - packed: vec![5, 3], - }; - assert!(decode_sparse_registers(&sp).is_err()); - } - - /// `compute_delta` against an EMPTY snapshot reconstructs the window's - /// full register state when its bytes are applied to a fresh empty - /// sketch (register max-merge round-trip). HLL deltas are lossless — - /// every increased register is carried. - #[test] - fn test_compute_delta_against_empty_round_trips() { - let mut window = HllSketch::new(HllVariant::Datafusion, 14); - for i in 0..5000u64 { - window.update(&i.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Datafusion, 14); - - let delta_bytes = window.compute_delta(&empty, 0); - - let mut reconstructed = HllSketch::new(HllVariant::Datafusion, 14); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - - assert_eq!(reconstructed.registers, window.registers); - } - - /// A delta computed between two non-empty snapshots reconstructs the - /// current sketch when applied to the base (max-merge of the registers - /// that increased). - #[test] - fn test_compute_delta_then_apply_matches_current() { - let mut base = HllSketch::new(HllVariant::Datafusion, 12); - for i in 0..2000u64 { - base.update(&i.to_le_bytes()); - } - let mut current = base.clone(); - for i in 2000..5000u64 { - current.update(&i.to_le_bytes()); - } - - let delta_bytes = current.compute_delta(&base, 0); - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - assert_eq!(reconstructed.registers, current.registers); - } - - /// Cross-language byte-parity guard: `compute_delta` against an empty - /// snapshot must emit bytes identical to the Go reference - /// implementation's `SerializeRegisterDelta(ComputeRegisterDelta(empty, - /// current))` for the same precision-14 sketch fed `(1..=50)` as f64 - /// little-endian byte values. A delta-against-empty carries every - /// non-zero register as an `(index, value)` update; the golden hex was - /// captured from a `proto.Marshal` of the Go reference's `HLLDelta`. - #[test] - fn test_compute_delta_matches_go_golden_bytes() { - let mut current = HllSketch::new(HllVariant::Datafusion, 14); - for i in 1..=50i32 { - let v = i as f64; - current.update(&v.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Datafusion, 14); - let got = current.compute_delta(&empty, 0); - - // Captured from the Go reference implementation's - // SerializeRegisterDelta(ComputeRegisterDelta(empty, current)) for the - // same input, with the packed (index_delta, value) HLLDelta encoding. - const GOLDEN_HEX: &str = "0a81015801bb0402800c015001970401dd0301b10101b2080132016a01a901022f018402020902a10a029a03027501c7020428042b05ec0702a60102850205c2030258021101fe030128022002a60403cd0102e7060211060b0334020b069c0101ff06012a015e0581020170041201f80401a70404a60101bc03019e0103cc0201880203"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "HLL delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - /// Cross-language byte-parity guard for a SUB-WINDOW delta (not against - /// empty): a precision-14 sketch is fed `(0..2000)` as f64 little-endian - /// bytes (the base/snapshot), then `(2000..5000)`; the delta carries only - /// the registers that grew between the two, packed as (index_delta, value) - /// pairs. Must be byte-identical to the Go reference implementation's - /// `SerializeRegisterDelta(ComputeRegisterDelta(base, current))`. - #[test] - fn test_compute_subwindow_delta_matches_go_golden_bytes() { - let mut base = HllSketch::new(HllVariant::Datafusion, 14); - for i in 0..2000i32 { - base.update(&(i as f64).to_le_bytes()); - } - let mut current = base.clone(); - for i in 2000..5000i32 { - current.update(&(i as f64).to_le_bytes()); - } - let got = current.compute_delta(&base, 0); - - // Captured from the Go reference implementation's - // SerializeRegisterDelta(ComputeRegisterDelta(base, current)) for the - // same input, with the packed (index_delta, value) HLLDelta encoding. - const GOLDEN_HEX: &str = "0ae627000402030401020302030c0109020502040201011108100202021304030101010b010e02040102030b0407020201050405021404060105010301010101020101030306020201040102011101010407010106080104030101010103020c0204021105040108010101180201030f010b01010103010e01120201030101010303030301030107030a0108010a0108012106060309010302070203010202030101010e0102020a020102060101020a01040101011702020203021003060103030e02110206010102040604010d010a050c011104010104050a0103020f0203010305050209010301020401020805010101010201020104010201060501040601020203020601020304010401020101010601010419010e010503050107020302010102010d010501110105050503020104050201020215050401030101020d041a010601190209020301020404070e01090202010b01030107040a0103010403010117050602040601010301040101030203030105010401020108030b0517020c01010506010402030103040704010102020d010a011201060304020c010c020e010b020702070101040101010104021504070109040804060502021802030104021605010107020603010101010201100104010605020205010e010406040102020f0203020a02040201030201010303010401030302010a030201060514020d02050104010201050103020802040105020602080213020d02020105020301140204010a0509060d03070206020902070101080a0202010a010e0102020503080201010e0101010403080104020d0102010501040403060401050405011203010101040e010202040303020c010203090201030d010a0306020201030208020e0103010701030113020202030501020c010b0101040502010105010203060105020a0315050b030c0101010f060405040204030b03010502020101080302020c010b02060403010404030102010f020f0107030601040109060301040301010703040303020801010104010b011101070102011101010102030201070106010601030201020301030302020e030501190108030201040205020503030106030a01050112020102010107030b02060209050201010304010502030106060601050108020602080201010905080205010204010203020303060205010102070201010308160304040f02010105020605070407030b0104010701010205020c02060103010b040c0102030202060102010e030102070108030301030205010101040301030303010203020802190105020601120101020701010101010302080303010f0104020202060402020403010103010503060203030701010105030903030508010101020103020305060605030e0203020603010102060f010201040202030504030103020f01130204010201020104010102010401010403050201030502040306060d0302010a010a02080103020101020208030301030102010403030305011003030206010f01010609030302020102030402020102010103020408010502130306010303060102010c01030202030401030101020101070202010901150104010f010401080107010601010206020b0201010d040303070202011403090210010302090403010403050201010b05010401010301090102010c010b04040206010d0102010b05040301020702020302010d0204030102110106010803020108030101070104020104040201030605050102040101010302010b01020103030e031101030101020103070112010403010101010b011001060313030301070203030803050101030601050207030a0104030302020202030c010101080103010c0102020404020102030c01110102030901140307010602020303020401070102010b020402030707010101010201070705130102011b0201060501010101010201020109010601020205010b0103040504070202010501080108020703030104030401010201030a010201030206010101020104020d02040102010102040301040605100201020801030306020e010402110106030602080105011101060106020d0216010702050301020101040409010403010201040102020205010401070207010d0205040704050202021201030510040303020602020703090204060102070102010301020101010401200101030202040103021301010107050a010702030311010303020103020b010505100302060102010104020a010402070210010704030304050a02010409010101010106030901030110031002020103010d0301020202040101020806080105010401060126020201080109020e0102030503120106010901010301010b01040201020402020115020d0108030f010c0303020c0101010602080504030802050202030f030202050404010603030208010a0204010d02020202030a0209030e030304030101060d0104020702010209020a0406010101070204020b0201030d021201020106031a0504050202040102010101020103030601080102020f021a020201120112020d02030405020403090304020102010108010203080101010704060104010902010401020203040107010e02070102060601080105010101060204010e010c020d010601050108010801020508010102070302030201030101010b020b02060107010c020b01020104020302060203040c02020105010301030202020902090103030503070606010103040216020401050205020404020208010b0207010502050108010502030102040f0206021203050105010a010a010b04090101030401040c03010102040104020c04090203020301040202020101060205020d010303040202020b010c010f0205030d010301050202030a02030309020203110103010102090106020402010202010303020206010a0104010601030208020803080110020702030205050201130102022102060204021504040102010a040c0401010503030207010701050103010d0104040601100201010201020302010a010501110102080504010106030101060601040a01050202080203020509010c010102090203010d0101020102120713021104070204010201010208020502060408010309020101020d0104010206030102010201010202050202040101010102050102030201060209020b01020105040202060401020f010103090504030101030111020e031001110204020602010201011101050204010a01040402030501010202010602070503011002020406010401030103020f0403050e0104020201010101040b0105030103010106020803080201010101010202050404010103011c0306030201010105010202060206010102020101010703010502010c010303020206030c021601130121020f03020307030e0109010602040102011f01090506010b01050402020e01020303020a040201020202010301030204030a010201080404030304110104010403070203060301020107020c050502020109030901020102010801030203090d0104021502010104040504090105030a010503110102020a010203050303040e030502030101030105020108010103020102030103010209030202010205010301010202020804010207010d020d020a010605080306010c0202021101090202030f010101040109020f0301020501010103020501040109021202080601070f0204020105070119030101110208040a0203020c010201070503030b0108030201040204020402010203010c022603040505010e04010103010702010109050b020801020405020e01140204020201050406020a020301060404010c010102050106020207010102010801050202020105040103010108010113050d0109020703070205010f0703050b0112010f02010103070b010201050101030201010203010705090605020f0103040b010801020210020501060204040c010104070105020a020104040201010403040109010c0101050a0305010301090108010b030b0201010601030202030e020203030101050202010106010d010d010601020101010c0304010901010103010401030201010902010101050b01010107010105020105020c01030405011404020104030a01030105010801050202020a0107010c04150105030301050401020603020203040b01100101010301030103070101010215020302010101010101080203020a010a021d031302030104020201030305030b01030203030b03010212020302030202050d010703040202040501040101030203060405031802010206020402020109020202070305021c01010105031c010503050305030d010e01090105030401070104010302010103010201100103020b0107050301090104010203030205030103010105010504080104020302020111050601050404022202070102010201090705080b01030109012903090206030104040107010102070201040f020201030103030b020804020207012d0105010301080103020201030202050501070105010202010304020303020401030802020103020102010501010c0106021201050317010a0304010101020207090102030205010b0204010b010801080312010f01150102020802040102011102080101040a0102010c020403020203010603050203010301040111031505040202030e01010102050607050101010502020301010c0210010d0407010903020202010e01050106020401070402010c010602090311060202100206020202020316020103010101010a030301060101010203150201050e04030204021c020c020c030102030101010502060102010d02010411040c0208010101010101040201010205050101060214010901010407020206160101050407050104010101010205050501060103010101060411020302070211010201090203021a0602010f0106030a04010106010301090101070501020107060102010404020b0102010202040110010201050106020901090309020d0105020601080101010501010205030403180401030d0105040802240209010a02020305020502040102010301090303020802040108010405020201040d0403010501020102020503040215010d030902030116010f010d0107030204070204040a0109011002090119031e020502090104031b050605020201010c0207020503010104010c0102010a01060502010c020d020202050101011102050107040f0102010201010106020305020101010402010302010c0306051505010104020802010102070c010803010506011101050207020302040103010301080103080d030c0103040d01010209040501030101010402020301010f01020102010301010401010202050408011c050904080106040401040105010402120102020103030201010a030103020402021303020305020403030201030901010504010902010305010301030203020a0108010702050107020a01160205020a060201030106030e02030103031001070102030d0119020202070505021202030108010101060808010202050108010f0102010701030404020802020214030c0211010b020301080303020105080105010401100106011309040203010103010208030b020102010301010e0107021501080203010402010101010606030204010d0108010e0102010401010105030903020302030c0103010104080101010405030201020c020101070201020102010401010301080102041f01070110030101060107010201060108010d010a030a0114041003020104010302020115011a02070802010301010106020b0101010201050109020703020205010a05050402030b0108010901070201020101020105040401110101020101040103010301010107010201180101020a030201030211011a02110103010102050103040e02160106010b02050a0e0202010203030101010402120305010301030305010805070304010402050102020402040106040105060105010b01110104010c0202020b01020104020e0201020704070206010c02030105010601030106011303070103010c0601010e021a01010704020801020201030c0506010a010203010202030e0311010201050105010b010504040203040a0102010302030302010102090301010403010104010e010501050a04070201060304030b0203010e01030201011b0102020c010701040303020d0604011001130202021304020505020201010105010a010f0106020801030113010802020104040b020104080104010e03090205020905020205010e040306020304010204100106010101150103050501050101010c0102010401050201030203010315010f0111020e0403010702030203030c0101020d02030205030a01040605050a02020101011201060502020201020101010a030f010706040105010101080502020301010608040b050101010104010d0103032001030201010f01040103011003010102060401070107010f01030116030301030205030902020611030102010101010101010201020202050403011b0208010b02030202010e0108010b010e01010508040701010103050604040515020202030104020703060203010601060106020f0101020302040102010c0302010101030105010a03020311020301030201010d03040107020502010109030b0302090801070304010301010402020501010201010501050202040302030305010101040201020802020101020f0107010b020302070313010b06050201030c010b04010106050302030102030401090208010901050307020d030b0309030e01090106010302020202010501040708010e0102010401010101020304010303010505050104020201070107020101040102010e0104011a0105050a030d030f010a02130103030b0107050e0203040403010208030d0701030101080206010a010e0203020501070303011102030203010101050117010e040d040701180201010e0201010402090204012c031c010e0204020502080113010201"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "HLL sub-window delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - - // Sanity: applying the packed sub-window delta to the base - // reconstructs the current register state (register max-merge). - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&got).unwrap(); - assert_eq!(reconstructed.registers, current.registers); - } -} diff --git a/src/message_pack_format/portable/mod.rs b/src/message_pack_format/portable/mod.rs index 97e0d372..cc6d0e63 100644 --- a/src/message_pack_format/portable/mod.rs +++ b/src/message_pack_format/portable/mod.rs @@ -2,7 +2,7 @@ //! //! Each per-algorithm submodule holds one sketch's pre-envelope wire type. //! These carry a cross-language contract of their own, on goldens apart -//! from ASAPv1's `asapv1_golden/`: HLL, Count-Min, Count Sketch and KLL pin +//! from ASAPv1's `asapv1_golden/`: Count-Min, Count Sketch and KLL pin //! byte parity against Go's serializers, KLL in both directions; DDSketch's //! golden is a Rust self-pin, not yet reconciled. New code uses ASAPv1: //! prefer a sketch's own `serialize_to_bytes`; do not add callers here. @@ -13,7 +13,6 @@ pub mod countsketch; pub mod countsketch_topk; pub mod ddsketch; pub mod delta_set_aggregator; -pub mod hll; pub mod hydra_kll; pub mod kll; pub mod sampling; diff --git a/src/message_pack_format/portable/sampling.rs b/src/message_pack_format/portable/sampling.rs index 9f62b452..e6951d18 100644 --- a/src/message_pack_format/portable/sampling.rs +++ b/src/message_pack_format/portable/sampling.rs @@ -71,7 +71,6 @@ pub const fn is_quantile_scale_invariant() -> bool { #[cfg(test)] mod tests { use super::*; - use crate::proto::sketchlib::{HyperLogLogState, SketchEnvelope, sketch_envelope::SketchState}; fn env_with_p(p: f64) -> SketchEnvelope { SketchEnvelope { @@ -79,15 +78,7 @@ mod tests { producer: None, hash_spec: None, sample_p: p, - sketch_state: Some(SketchState::Hll(HyperLogLogState { - variant: 2, - precision: 14, - registers: Vec::new(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: None, - })), + sketch_state: None, } } diff --git a/src/proto/generated/sketchlib.v1.rs b/src/proto/generated/sketchlib.v1.rs index 2c098d3f..8b1303a4 100644 --- a/src/proto/generated/sketchlib.v1.rs +++ b/src/proto/generated/sketchlib.v1.rs @@ -461,38 +461,6 @@ pub struct HllSparseRegisters { #[prost(bytes = "vec", tag = "2")] pub packed: ::prost::alloc::vec::Vec, } -/// HLLDelta carries the registers that increased between two consecutive -/// snapshots. HLL uses max semantics, so only increases are meaningful; -/// at a fixed precision a register can never decrease. Each update is applied -/// losslessly on the receiver via register\[index\] = max(register\[index\], value) -/// — no register update is ever dropped (there is no threshold to apply). -/// -/// The increased registers are varint-packed exactly like -/// HLLSparseRegisters.packed: sorted ascending by index, each register emitted -/// as (index_delta, value) where index_delta = index - prev_index (prev_index -/// starts at 0) and value is the new (larger) register value: -/// -/// for each increased register, in ascending index order: -/// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -/// uvarint(value) // 1..=Q+1, always 1 byte in practice -/// -/// A single-emit delta against the all-zero snapshot is therefore the same size -/// as the full sparse frame, and a sub-window delta is strictly smaller (it -/// packs only the registers that actually grew). This replaces the previous -/// per-register sub-message encoding, eliminating ~6–8 bytes of tag/length -/// overhead per register. -/// -/// This message is byte-identical to the Go reference implementation's HLLDelta -/// so the two runtimes emit byte-identical delta frames for identical window -/// state (cross-language byte parity). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HllDelta { - /// Varint-packed (index_delta, value) pairs for the registers that increased, - /// in ascending index order. Same layout as HLLSparseRegisters.packed. Empty - /// when no register changed. - #[prost(bytes = "vec", tag = "1")] - pub packed_updates: ::prost::alloc::vec::Vec, -} /// HLLVariant identifies which HLL estimator algorithm the registers belong to. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -892,7 +860,7 @@ pub struct SketchEnvelope { /// The sketch payload. Exactly one field must be set. #[prost( oneof = "sketch_envelope::SketchState", - tags = "10, 11, 12, 13, 14, 15, 16, 17, 18" + tags = "10, 11, 13, 14, 15, 16, 17, 18" )] pub sketch_state: ::core::option::Option, } @@ -905,8 +873,6 @@ pub mod sketch_envelope { CountMin(super::CountMinState), #[prost(message, tag = "11")] CountSketch(super::CountSketchState), - #[prost(message, tag = "12")] - Hll(super::HyperLogLogState), #[prost(message, tag = "13")] Kll(super::KllState), #[prost(message, tag = "14")] diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 5464cd0a..08bb8c05 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -42,10 +42,6 @@ use std::marker::PhantomData; mod wire; pub use wire::HllWireVariant; -pub(crate) use wire::{ - HLL_KIND_CLASSIC, HLL_KIND_ERTL_MLE, HLL_KIND_HIP, HllMetadata, HllPayloadHip, HllPayloadPlain, - standard_hll_metadata, -}; /// Generic HyperLogLog sketch parameterized by estimation variant, register storage, and hasher. #[derive(Clone, Debug, Serialize, Deserialize)] @@ -140,8 +136,13 @@ impl HyperLogLogImpl { fn new_base() -> Self { + Self::from_storage(Registers::default()) + } + + /// Creates a sketch from an existing register storage instance. + pub fn from_storage(registers: Registers) -> Self { Self { - registers: Registers::default(), + registers, _marker: PhantomData, _hasher: PhantomData, } @@ -340,6 +341,18 @@ impl HyperLogLogHIPImpl { est: 0.0, } } + + /// Creates a sketch from register storage and the HIP running state + /// (`kxq0`, `kxq1`, `est`). + pub fn from_storage(registers: Registers, kxq0: f64, kxq1: f64, est: f64) -> Self { + Self { + registers, + kxq0, + kxq1, + est, + } + } + /// Inserts a pre-hashed value, updating both the register and the HIP running estimate. #[inline(always)] pub fn insert_with_hash(&mut self, hashed: u64) { diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index 56a0ff84..bb4e0f5e 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -91,12 +91,6 @@ pub(crate) fn hll_metadata(precision: u32) -> HllMetadata { } } -/// The standard ProjectASAP profile metadata (the [`DefaultXxHasher`] profile). -/// Used by the portable path, which only represents standard-profile sketches. -pub(crate) fn standard_hll_metadata(precision: u32) -> HllMetadata { - hll_metadata::(precision) -} - /// Validate the envelope for a known target and return the raw payload bytes. /// The metadata is checked against the profile of `H`, so bytes hashed under a /// different profile are rejected (fail closed). @@ -215,8 +209,9 @@ impl HyperLogLogHIPImpl { /// than bytes that would be refused on decode. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { let registers = registers_to_bytes(&self.registers)?; - let metadata = - rmp_serde::to_vec_named(&standard_hll_metadata(Registers::PRECISION as u32))?; + let metadata = rmp_serde::to_vec_named(&hll_metadata::( + Registers::PRECISION as u32, + ))?; let payload = rmp_serde::to_vec(&HllPayloadHip { registers, hip_kxq0: self.kxq0, @@ -247,9 +242,9 @@ impl HyperLogLogHIPImpl { #[cfg(test)] mod tests { use super::*; + use crate::DataInput; use crate::sketches::hll::{HyperLogLog, HyperLogLogHIP, HyperLogLogHIPP12, HyperLogLogP12}; use crate::structures::fixed_structure::{HllBucketListP12, HllBucketListP14}; - use crate::{DataInput, HllBucketList}; const ERROR_TOLERANCE: f64 = 0.02; const SERDE_SAMPLE: usize = 100_000; @@ -444,52 +439,6 @@ mod tests { assert_eq!(decoded.est, sketch.est); } - /// Drift guard: the native path and the (deprecated) portable path must emit - /// byte-identical ASAPv1 envelopes. Keep until golden byte-vectors exist. - #[test] - fn native_and_portable_hll_bytes_match() { - use crate::message_pack_format::MessagePackCodec; - use crate::message_pack_format::portable::hll::{HllSketch, HllVariant}; - - // Ertl-MLE / Datafusion: registers-only payload. - let mut native = HyperLogLog::::default(); - for v in 0..1000 { - native.insert(&DataInput::U64(v)); - } - let native_bytes = native.serialize_to_bytes().expect("native serialize"); - let portable = HllSketch::from_raw( - HllVariant::Datafusion, - HllBucketList::PRECISION as u32, - native.registers_as_slice().to_vec(), - 0.0, - 0.0, - 0.0, - ); - assert_eq!( - native_bytes, - portable.to_msgpack().expect("portable serialize") - ); - - // HIP: registers + three running scalars. - let mut hip = HyperLogLogHIP::default(); - for v in 0..1000 { - hip.insert(&DataInput::U64(v)); - } - let hip_bytes = hip.serialize_to_bytes().expect("native serialize"); - let portable_hip = HllSketch::from_raw( - HllVariant::Hip, - HllBucketList::PRECISION as u32, - hip.registers.as_slice().to_vec(), - hip.kxq0, - hip.kxq1, - hip.est, - ); - assert_eq!( - hip_bytes, - portable_hip.to_msgpack().expect("portable serialize") - ); - } - // A test-only custom hasher: it hashes exactly like `DefaultXxHasher` // (delegation) but declares a DIFFERENT `HashProfile` (distinct profile id // and seed list). Serialization is derived from the profile, so an @@ -590,7 +539,7 @@ mod tests { precision: u32, bogus_field: u8, // key not in HllMetadata } - let std = standard_hll_metadata(14); + let std = hll_metadata::(14); let extra = WithExtra { metadata_version: std.metadata_version, hash_profile_id: std.hash_profile_id.clone(), @@ -626,7 +575,8 @@ mod tests { /// Builds a well-framed envelope around a crafted register bin. fn crafted_plain(kind_id: &[u8], precision: u32, registers: Vec) -> Vec { - let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(precision)).expect("meta"); + let metadata = + rmp_serde::to_vec_named(&hll_metadata::(precision)).expect("meta"); let payload = rmp_serde::to_vec(&HllPayloadPlain { registers }).expect("payload"); envelope::encode(kind_id, &metadata, &payload) } @@ -682,7 +632,7 @@ mod tests { ); // The HIP payload carries the same bin and is held to the same bound. - let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(14)).expect("meta"); + let metadata = rmp_serde::to_vec_named(&hll_metadata::(14)).expect("meta"); let payload = rmp_serde::to_vec(&HllPayloadHip { registers: bin(p14, P14_MAX + 1), hip_kxq0: p14 as f64, diff --git a/src/sketches/testdata/hll_envelope_golden.hex b/src/sketches/testdata/hll_envelope_golden.hex deleted file mode 100644 index cd7760ec..00000000 --- a/src/sketches/testdata/hll_envelope_golden.hex +++ /dev/null @@ -1 +0,0 @@ -0801628880010802100e1a80800100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000006000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index cddee44f..a7d1d923 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -2,8 +2,8 @@ One `proptest` law per line. Each compares the sketch against an answer computed without it; what that answer is comes first in every section. One -file per sketch under `tests/pbt/`. `cargo test --test pbt` runs 367 laws; -with `--all-features`, 435, adding the four **experimental** modules. Every +file per sketch under `tests/pbt/`. `cargo test --test pbt` runs 365 laws; +with `--all-features`, 433, adding the four **experimental** modules. Every law was mutation-checked: break the implementation, the law goes red. Shared generators, `grid` and `round_trip!` live in `support.rs`. @@ -228,7 +228,7 @@ Compared against: the offers sorted and truncated. `HHHeap` is path dependent on - one offer per key: the `k` largest counts resident - `update` returns true iff the key was resident or there was room, i.e. nothing was displaced -## hll.rs — 101 +## hll.rs — 99 Models use only what the paper fixes; register indexing and rank mapping stay free. For `Classic` and `ErtlMLE`, `estimate` is a function of the registers, so laws relate estimates rather than restate the formula. Registers `<= 40`, so `sum(2^-M[j])` is exact in f64. @@ -252,10 +252,8 @@ Models use only what the paper fixes; register indexing and rank mapping stay fr - `estimate` non-decreasing; a repeated key adds 0 - order matters: a descending run into one register estimates exactly 1, an ascending run `> 1` - an arrival that raises a register adds `m / sum(2^-M[j])` over the prior registers; one that raises nothing adds 0; both sides exact in f64, compared after truncation to `usize` -- portable `HllSketch` (5) - - merge commutative, idempotent, empty identity, `== S(xs ++ ys)`; precisions 4-11 - - msgpack keeps variant, precision, registers, estimate; `Regular`, `Datafusion`, `Hip`; precisions 4-13 - - ASAPv1 on `HyperLogLog` keeps registers and estimate; empty included +- wire (3) + - ASAPv1 on `HyperLogLog` and `HyperLogLog` keeps registers and estimate, on `HyperLogLogHIP` keeps estimate; re-encode byte-identical; empty included ## hydra.rs — 7 diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 30b41651..b820c978 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -9,8 +9,8 @@ //! See `asapv1_golden/README.md`. use asap_sketchlib::{ - Classic, Count, CountMin, ErtlMLE, FastPath, HllSketch, HllVariant, HyperLogLogHIPP12, - HyperLogLogP12, KLL, MessagePackCodec, RegularPath, Vector2D, + Classic, Count, CountMin, ErtlMLE, FastPath, HllBucketListP12, HllRegisterStorage, + HyperLogLogHIPP12, HyperLogLogP12, KLL, RegularPath, Vector2D, }; fn decode_hex(s: &str) -> Vec { @@ -43,6 +43,12 @@ fn p12_registers() -> Vec { r } +fn p12_storage() -> HllBucketListP12 { + let mut storage = HllBucketListP12::default(); + storage.as_mut_slice().copy_from_slice(&p12_registers()); + storage +} + const I64_VALS: [[i64; 3]; 2] = [[0, 1, 127], [128, 300, 65536]]; const F64_VALS: [[f64; 3]; 2] = [[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]; @@ -59,65 +65,41 @@ const CS_VALS: [[i64; 4]; 2] = [[0, 127, 128, 65536], [-1, -33, -32768, -2147483 #[test] fn hll_classic_p12_matches_golden() { let want = decode_hex(GOLDEN_CLASSIC); - let regs = p12_registers(); - // Build known state and serialize. - let got = HllSketch::from_raw(HllVariant::Regular, 12, regs.clone(), 0.0, 0.0, 0.0) - .to_msgpack() - .expect("serialize"); + let sketch = HyperLogLogP12::::from_storage(p12_storage()); + let got = sketch.serialize_to_bytes().expect("serialize"); assert_eq!(got, want, "Classic P12 bytes diverge from golden"); - // Native wire path (src/sketches/hll.rs) round-trips the golden identically. - let native = HyperLogLogP12::::deserialize_from_bytes(&want).expect("native decode"); - assert_eq!(native.registers_as_slice(), regs.as_slice()); - assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); - - // Portable decode round-trips to the same known state. - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.precision, 12); - assert_eq!(decoded.variant, HllVariant::Regular); + let decoded = HyperLogLogP12::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.registers_as_slice(), p12_registers().as_slice()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); } #[test] fn hll_ertl_mle_p12_matches_golden() { let want = decode_hex(GOLDEN_ERTL); - let regs = p12_registers(); - let got = HllSketch::from_raw(HllVariant::Datafusion, 12, regs.clone(), 0.0, 0.0, 0.0) - .to_msgpack() - .expect("serialize"); + let sketch = HyperLogLogP12::::from_storage(p12_storage()); + let got = sketch.serialize_to_bytes().expect("serialize"); assert_eq!(got, want, "Ertl-MLE P12 bytes diverge from golden"); - let native = HyperLogLogP12::::deserialize_from_bytes(&want).expect("native decode"); - assert_eq!(native.registers_as_slice(), regs.as_slice()); - assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); - - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.variant, HllVariant::Datafusion); + let decoded = HyperLogLogP12::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.registers_as_slice(), p12_registers().as_slice()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); } #[test] fn hll_hip_p12_matches_golden() { let want = decode_hex(GOLDEN_HIP); - let regs = p12_registers(); - let got = HllSketch::from_raw(HllVariant::Hip, 12, regs.clone(), 1.5, 2.5, 3.0) - .to_msgpack() - .expect("serialize"); + let sketch = HyperLogLogHIPP12::from_storage(p12_storage(), 1.5, 2.5, 3.0); + let got = sketch.serialize_to_bytes().expect("serialize"); assert_eq!(got, want, "HIP P12 bytes diverge from golden"); - // Native HIP wire path round-trips the golden identically. - let native = HyperLogLogHIPP12::deserialize_from_bytes(&want).expect("native decode"); - assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); - - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.variant, HllVariant::Hip); - assert_eq!(decoded.hip_kxq0, 1.5); - assert_eq!(decoded.hip_kxq1, 2.5); - assert_eq!(decoded.hip_est, 3.0); + // `Debug` prints the registers and the three HIP scalars exactly. + let decoded = HyperLogLogHIPP12::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(format!("{decoded:?}"), format!("{sketch:?}")); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); } // --------------------------------------------------------------------------- diff --git a/tests/conformance_kit.rs b/tests/conformance_kit.rs index dd57a219..682b158f 100644 --- a/tests/conformance_kit.rs +++ b/tests/conformance_kit.rs @@ -11,7 +11,6 @@ use common::conformance::{ use common::{FreqTruth, zipf_u64}; use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::{ Bloom, CMSHeap, CSHeap, CountL2HH, CountMin, DDSketch, DataInput, FastPath, FoldCMS, FoldCS, HyperLogLog, HyperLogLogHIP, KLL, KLLDynamic, RegularPath, SpaceSaving, UnivMonQ, Vector2D, @@ -131,17 +130,6 @@ impl CardinalityOps for HllHipAdapter { } } -struct PortableHllAdapter(HllSketch); - -impl CardinalityOps for PortableHllAdapter { - fn ingest(&mut self, key: &[u8]) { - self.0.update(key); - } - fn estimate(&self) -> f64 { - self.0.estimate() - } -} - struct KllAdapter(KLL); impl QuantileOps for KllAdapter { @@ -312,15 +300,6 @@ fn hll_variants_pass_cardinality_conformance() { spec, ) .assert_ok(); - - conformance::cardinality_battery( - "portable HllSketch", - || PortableHllAdapter(HllSketch::new(HllVariant::Regular, 14)), - &unique, - 100_000, - spec, - ) - .assert_ok(); } /// Compaction-coin seed for the KLL adapters. The unseeded `init_kll` diff --git a/tests/e2e/cardinality.rs b/tests/e2e/cardinality.rs index bf0d9636..01111149 100644 --- a/tests/e2e/cardinality.rs +++ b/tests/e2e/cardinality.rs @@ -26,7 +26,6 @@ use common::{ }; use common::{uniform_u64, zipf_u64}; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::sketches::hll::{HyperLogLogHIPImpl, HyperLogLogImpl}; use asap_sketchlib::{ Classic, DataInput, ErtlMLE, HyperLogLogHIPP12, HyperLogLogHIPP14, HyperLogLogHIPP16, @@ -248,77 +247,6 @@ hll_battery!( not_mergeable ); -/// The portable wire type across every variant tag and precision. -/// -/// `HllVariant` selects the *wire tag*, not the estimator: `HllSketch::estimate` -/// runs the same register-based Classic formula for `Regular`, `Datafusion` and -/// `Hip` alike. So all three are held to the register model — checking the -/// `Hip` tag against HIP's tighter constant would be asserting a property this -/// type does not implement. -#[test] -fn portable_hll_variants_and_precisions_satisfy_the_register_error_model() { - const N: usize = 200_000; - - for (v, variant) in [HllVariant::Regular, HllVariant::Datafusion, HllVariant::Hip] - .into_iter() - .enumerate() - { - for (p, precision) in [12u32, 14, 16].into_iter().enumerate() { - let spec = CardinalityConfidenceSpec::hll(precision, Z); - // A distinct stream per (variant, precision). Reusing one stream - // across the three variants would feed three estimators the *same* - // registers, whose errors are then near-perfectly correlated — nine - // readings of at most three independent experiments. - let seed = 2001 + (v * 3 + p) as u64; - let stream = uniform_u64(N, u64::MAX / 4, seed); - let truth: std::collections::HashSet = stream.iter().copied().collect(); - - let mut hll = HllSketch::new(variant, precision); - for k in &stream { - hll.update(k.to_be_bytes().as_slice()); - } - if let Err(detail) = spec.check(hll.estimate(), truth.len()) { - panic!("{variant:?}/p{precision} stream_seed={seed}: {detail}"); - } - - // Merging a second sketch built over the SAME identities is a - // register-wise max with itself: the estimate must not move at all. - let mut other = HllSketch::new(variant, precision); - for k in &truth { - other.update((*k).to_be_bytes().as_slice()); - } - let before = hll.estimate(); - hll.merge(&other).expect("merge"); - assert_eq!( - hll.estimate(), - before, - "{variant:?} p{precision}: merging identical identities moved the estimate" - ); - - // Disjoint shards over the same stream merge by register-wise max, - // so the result is the *same* sketch as the single pass and its - // estimate is the *same number*. That is an equality, not a second - // confidence-band reading. - let mut left = HllSketch::new(variant, precision); - let mut right = HllSketch::new(variant, precision); - for (i, k) in stream.iter().enumerate() { - if i % 2 == 0 { - left.update(k.to_be_bytes().as_slice()); - } else { - right.update(k.to_be_bytes().as_slice()); - } - } - left.merge(&right).expect("shard merge"); - assert_eq!( - left.estimate(), - before, - "{variant:?} p{precision}: an even/odd shard merge must reproduce the \ - single pass exactly (registers combine by maximum)" - ); - } - } -} - /// A larger precision must actually buy accuracy. A fixed percentage band /// cannot see this — it passes identically at p12 and p16, so it would not /// notice a precision parameter that never reached the register array. diff --git a/tests/msgpack_compat.rs b/tests/msgpack_compat.rs index 267a51c6..1d377266 100644 --- a/tests/msgpack_compat.rs +++ b/tests/msgpack_compat.rs @@ -31,8 +31,8 @@ use asap_sketchlib::message_pack_format::portable::delta_set_aggregator::DeltaRe use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketchWire; use asap_sketchlib::message_pack_format::portable::kll::KllSketchData; use asap_sketchlib::{ - CountMinSketch, CountMinSketchWithHeap, CountSketch, DdSketch, HllSketch, HllVariant, - HydraKllSketch, KllSketch, SetAggregator, + CountMinSketch, CountMinSketchWithHeap, CountSketch, DdSketch, HydraKllSketch, KllSketch, + SetAggregator, }; // ===== round-trip: every wire-format-aligned type ===== @@ -84,17 +84,6 @@ fn dd_sketch_round_trip() { assert_eq!(restored.total_count(), 3); } -#[test] -fn hll_sketch_round_trip() { - let mut s = HllSketch::new(HllVariant::Regular, 8); - s.update(b"a"); - s.update(b"b"); - s.update(b"c"); - let bytes = s.to_msgpack().expect("encode"); - let restored = HllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.registers.len(), s.registers.len()); -} - #[test] fn kll_sketch_round_trip() { let mut s = KllSketch::with_seed(200, 0x5EED_0800); @@ -241,10 +230,6 @@ fn count_min_decodes_go_bytes() { #[test] fn dd_sketch_decodes_go_bytes() {} -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn hll_sketch_decodes_go_bytes() {} - #[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] #[test] fn kll_sketch_decodes_go_bytes() {} diff --git a/tests/pbt/hll.rs b/tests/pbt/hll.rs index e362f129..7159350a 100644 --- a/tests/pbt/hll.rs +++ b/tests/pbt/hll.rs @@ -20,15 +20,11 @@ //! `impl` block, so the two variants share one //! monomorphization. Estimation expands over both. //! -//! The merge laws cover both representations: `HyperLogLogImpl` itself and -//! `HllSketch`, the portable one with a register array of its own. -//! //! Flajolet, Fusy, Gandouet, Meunier, AofA '07. use crate::support::keys; -use asap_sketchlib::message_pack_format::MessagePackCodec; use asap_sketchlib::sketches::hll::{HyperLogLogHIPImpl, HyperLogLogImpl}; -use asap_sketchlib::{Classic, DataInput, ErtlMLE, HllSketch, HllVariant, HyperLogLog}; +use asap_sketchlib::{Classic, DataInput, ErtlMLE, HyperLogLog, HyperLogLogHIP}; use proptest::prelude::*; asap_sketchlib::impl_hll_bucket_list!(BucketsP4, 4, 1_usize << 4); @@ -446,14 +442,20 @@ macro_rules! hip_laws { const M: usize = <$buckets>::NUM_REGISTERS; const P: u32 = <$buckets>::PRECISION as u32; - /// The registers, read back through the wire format. The struct + /// The registers, read back out of the ASAPv1 payload. The struct /// holds them privately behind the `kxq0`/`kxq1` accumulators, and /// those accumulators are what the increment law checks, so the /// envelope is the one view of the registers that stays independent /// of it. fn registers_of(sketch: &S) -> Vec { - let bytes = sketch.to_msgpack().expect("encode"); - HllSketch::from_msgpack(&bytes).expect("decode").registers + let bytes = sketch.serialize_to_bytes().expect("encode"); + let lengths_at = 8 + bytes[7] as usize; + let metadata_len = + u32::from_be_bytes(bytes[lengths_at..lengths_at + 4].try_into().unwrap()); + let payload = &bytes[lengths_at + 8 + metadata_len as usize..]; + let (registers, ..): (serde_bytes::ByteBuf, f64, f64, f64) = + rmp_serde::from_slice(payload).expect("decode"); + registers.into_vec() } #[test] @@ -565,102 +567,28 @@ hip_laws!(hip_p6, BucketsP6); hip_laws!(hip_p8, BucketsP8); hip_laws!(hip_p10, BucketsP10); -fn byte_items(max: usize) -> impl Strategy>> { - prop::collection::vec(prop::collection::vec(any::(), 1..16), 0..max) -} - -fn portable_hll_of(precision: u32, items: &[Vec]) -> HllSketch { - let mut s = HllSketch::new(HllVariant::Regular, precision); - for i in items { - s.update(i); - } - s -} - -proptest! { - // ===== Merge ===== - - #[test] - fn hll_register_merge_is_commutative_and_idempotent( - precision in 4u32..12, - a in byte_items(50), - b in byte_items(50), - ) { - let mut ab = portable_hll_of(precision, &a); - ab.merge(&portable_hll_of(precision, &b)).expect("merge"); - let mut ba = portable_hll_of(precision, &b); - ba.merge(&portable_hll_of(precision, &a)).expect("merge"); - - prop_assert_eq!(&ab.registers, &ba.registers); - - let before = ab.registers.clone(); - let again = ab.clone(); - ab.merge(&again).expect("merge"); - prop_assert_eq!(&ab.registers, &before); - } - - #[test] - fn hll_merge_equals_streaming_the_concatenation( - precision in 4u32..12, - a in byte_items(50), - b in byte_items(50), - ) { - let mut merged = portable_hll_of(precision, &a); - merged.merge(&portable_hll_of(precision, &b)).expect("merge"); - - let concatenated: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); - let streamed = portable_hll_of(precision, &concatenated); - - prop_assert_eq!(&merged.registers, &streamed.registers); - } - - #[test] - fn hll_empty_is_a_merge_identity( - precision in 4u32..12, - a in byte_items(50), - ) { - let base = portable_hll_of(precision, &a); - let mut merged = base.clone(); - merged.merge(&HllSketch::new(HllVariant::Regular, precision)).expect("merge"); - - prop_assert_eq!(&merged.registers, &base.registers); - } -} - -fn hll_variant() -> impl Strategy { - prop_oneof![ - Just(HllVariant::Regular), - Just(HllVariant::Datafusion), - Just(HllVariant::Hip), - ] -} - proptest! { // ===== Wire ===== #[test] - fn hll_round_trip_preserves_registers( - variant in hll_variant(), - precision in 4u32..14, - items in byte_items(64), - ) { - let mut s = HllSketch::new(variant, precision); - for item in &items { - s.update(item); + fn hyperloglog_round_trips_including_the_empty_sketch(stream in keys(400)) { + type Hll = HyperLogLog; + let mut sketch = Hll::new(); + for k in &stream { + sketch.insert(&DataInput::U64(*k)); } - let bytes = s.to_msgpack().expect("encode"); - let restored = HllSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.variant, s.variant); - prop_assert_eq!(restored.precision, s.precision); - prop_assert_eq!(&restored.registers, &s.registers); - prop_assert_eq!(restored.estimate(), s.estimate()); + round_trip!( + Hll, + sketch, + |s: &Hll| s.registers_as_slice().to_vec(), + |s: &Hll| s.estimate(), + ); } #[test] - fn hyperloglog_round_trips_including_the_empty_sketch(stream in keys(400)) { - type Hll = HyperLogLog; + fn classic_round_trips_including_the_empty_sketch(stream in keys(400)) { + type Hll = HyperLogLog; let mut sketch = Hll::new(); for k in &stream { sketch.insert(&DataInput::U64(*k)); @@ -673,4 +601,14 @@ proptest! { |s: &Hll| s.estimate(), ); } + + #[test] + fn hip_round_trips_including_the_empty_sketch(stream in keys(400)) { + let mut sketch = HyperLogLogHIP::new(); + for k in &stream { + sketch.insert(&DataInput::U64(*k)); + } + + round_trip!(HyperLogLogHIP, sketch, |s: &HyperLogLogHIP| s.estimate()); + } } diff --git a/tests/proptest-regressions/hll.txt b/tests/proptest-regressions/hll.txt new file mode 100644 index 00000000..cb8fb6e2 --- /dev/null +++ b/tests/proptest-regressions/hll.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 115f567853acb20cbde319427f1d20dbe1769471bdccb8f153dab74ff382dc7a # shrinks to stream = [] diff --git a/tests/sketches_go_parity_probe.rs b/tests/sketches_go_parity_probe.rs index 2997e486..485bf01d 100644 --- a/tests/sketches_go_parity_probe.rs +++ b/tests/sketches_go_parity_probe.rs @@ -9,16 +9,13 @@ //! math is sufficient — no hashspec bypass needed. use asap_sketchlib::common::DataInput; -use asap_sketchlib::common::hash::CANONICAL_HASH_SEED; -use asap_sketchlib::common::structures::HllBucketListP14; use asap_sketchlib::proto::sketchlib::{ - CountMinState, CountSketchState, CounterType, DdSketchState, HyperLogLogState, SketchEnvelope, + CountMinState, CountSketchState, CounterType, DdSketchState, SketchEnvelope, sketch_envelope::SketchState, }; use asap_sketchlib::sketches::countminsketch::CountMin; use asap_sketchlib::sketches::countsketch::Count; use asap_sketchlib::sketches::ddsketch::DDSketch; -use asap_sketchlib::sketches::hll::{ErtlMLE, HyperLogLogImpl}; use asap_sketchlib::{DefaultXxHasher, FastPath, Vector2D}; use prost::Message; @@ -34,11 +31,6 @@ const COUNTSKETCH_GOLDEN_HEX: &str = include_str!("cs_envelope_golden.hex"); /// each repeated 5×). Same file the wrapper test already uses. const COUNTMIN_GOLDEN_HEX: &str = include_str!("../src/sketches/testdata/cms_envelope_golden.hex"); -/// 16 398-byte envelope captured from `sketchlib-go::HyperLogLog.SerializePortable` -/// for precision=14, `(1..=50)` IEEE-754-LE byte-key input. Same file -/// the wrapper test already uses. -const HLL_GOLDEN_HEX: &str = include_str!("../src/sketches/testdata/hll_envelope_golden.hex"); - /// 403-byte DDSketch envelope for `alpha=0.01`, `(1..=50)` integer-as-f64 /// input, without the DataPoint-level METRIC scalars (count/sum/min/max → /// proto tags 4-7 reserved). @@ -167,57 +159,6 @@ fn sketches_countmin_fastpath_matches_go_count_min_envelope() { ); } -#[test] -fn sketches_hll_classic_matches_go_envelope() { - // Go's `HyperLogLog.SerializePortable` emits variant=DATAFUSION=2 - // (ErtlMLE). Use the same variant + precision=14 + same input here. - let mut sk: HyperLogLogImpl = - >::new(); - for i in 1..=50i32 { - let v = i as f64; - let bytes = v.to_le_bytes(); - let hashed = ::hash64_seeded( - CANONICAL_HASH_SEED, - &DataInput::Bytes(&bytes), - ); - sk.insert_with_hash(hashed); - } - - let state = HyperLogLogState { - // 2 = DATAFUSION on the Go wire (matches ErtlMLE) - variant: 2, - precision: 14, - registers: sk.registers_as_slice().to_vec(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - // This parity probe pins the DENSE wire form; leave sparse unset. - registers_sparse: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Hll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - let want = decode_hex(HLL_GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "HLL envelope length: got {} want {}", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "sketches::HyperLogLogImpl envelope diverges from Go golden", - ); -} - #[test] fn sketches_ddsketch_matches_go_envelope() { let mut sk = DDSketch::new(0.01); diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs index 0113ff45..58dfddfe 100644 --- a/tests/xtest_consumer.rs +++ b/tests/xtest_consumer.rs @@ -8,7 +8,6 @@ //! countmin.pb — CountMinState with float64 counters //! kll.pb — KLLState with quantile items and coin RNG state //! ddsketch.pb — DDSketchState with alpha + bucket array -//! hll.pb — HyperLogLogState (ErtlMLE variant) //! countsketch.pb — CountSketchState with float64 signed counters //! coco.pb — CocoSketchState (hash+val+hasKey buckets) //! elastic.pb — ElasticState (heavy buckets + light CountMin) @@ -210,41 +209,6 @@ fn cross_language_proto() { all_ok = false; } - // ----------------------------------------------------------------------- - // HLL (ErtlMLE estimator) - // ----------------------------------------------------------------------- - println!(); - println!("[HLL] Step 1/3 — Read hll.pb"); - let bytes = read_file(in_dir.join("hll.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode hll envelope"); - - println!( - "[HLL] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let hll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {other:?}"), - }; - - println!( - "[HLL] variant={} precision={} registers={}", - hll_state.variant, - hll_state.precision, - hll_state.registers.len() - ); - - let hll_card = hll_ertl_mle_estimate(&hll_state); - println!("[HLL] Step 3/3 — cardinality ≈ {hll_card} (expect ~50000)"); - if (40_000..=65_000).contains(&hll_card) { - println!("[HLL] PASS"); - } else { - eprintln!("[HLL] FAIL: cardinality {hll_card} not in [40000, 65000]"); - all_ok = false; - } - // ----------------------------------------------------------------------- // CountSketch // ----------------------------------------------------------------------- @@ -491,48 +455,6 @@ fn cross_language_proto() { } } - // ----------------------------------------------------------------------- - // Sampled HLL (hash-threshold sampling, p=0.1) — cardinality ×1/p rescale. - // ----------------------------------------------------------------------- - let sampled_hll_path = in_dir.join("hll_sampled.pb"); - if sampled_hll_path.exists() { - println!(); - println!("[HLL/sampled] Step 1/3 — Read hll_sampled.pb"); - let bytes = read_file(&sampled_hll_path); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode sampled hll"); - - let p = effective_sample_p(&env); - println!("[HLL/sampled] Step 2/3 — envelope sample_p = {p}"); - if (p - 0.1).abs() > 1e-9 { - eprintln!("[HLL/sampled] FAIL: sample_p {p} != 0.1"); - all_ok = false; - } - - let mut hll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {other:?}"), - }; - // Dual-read the registers (sampling thins them → may be sparse-encoded). - let regs = - asap_sketchlib::message_pack_format::portable::hll::registers_from_state(&hll_state) - .expect("decode sampled hll registers"); - hll_state.registers = regs; - hll_state.registers_sparse = None; - - let raw = hll_ertl_mle_estimate(&hll_state) as f64; - let rescaled = rescale_count(raw, p); - let rel_err = (rescaled - 200_000.0).abs() / 200_000.0; - println!( - "[HLL/sampled] Step 3/3 — raw≈{raw:.0} rescaled(×1/p)≈{rescaled:.0} truth=200000 relErr={rel_err:.4}" - ); - if rel_err <= 0.06 { - println!("[HLL/sampled] PASS"); - } else { - eprintln!("[HLL/sampled] FAIL: rescaled rel err {rel_err:.4} > 0.06"); - all_ok = false; - } - } - // ----------------------------------------------------------------------- // Final summary // ----------------------------------------------------------------------- @@ -718,78 +640,6 @@ fn xxh3_64_seeded(seed: u64, data: &[u8]) -> u64 { XxHash3_64::oneshot_with_seed(seed, data) } -// --------------------------------------------------------------------------- -// HLL ErtlMLE estimator (Ertl 2017) -// --------------------------------------------------------------------------- -// Mirrors Go's HyperLogLog.Estimate() which uses HLLRegisterBits = 50, -// HLLPrecision = 14, HLLRegisterCount = 16384. - -fn hll_sigma(mut x: f64) -> f64 { - if x == 1.0 { - return f64::INFINITY; - } - let mut y = 1.0f64; - let mut z = x; - loop { - x *= x; - let z_prev = z; - z += x * y; - y += y; - if z_prev == z { - break; - } - } - z -} - -fn hll_tau(mut x: f64) -> f64 { - if x == 0.0 || x == 1.0 { - return 0.0; - } - let mut y = 1.0f64; - let mut z = 1.0 - x; - loop { - x = x.sqrt(); - let z_prev = z; - y *= 0.5; - z -= (1.0 - x).powi(2) * y; - if z_prev == z { - break; - } - } - z / 3.0 -} - -fn hll_ertl_mle_estimate(state: &HyperLogLogState) -> u64 { - let precision = state.precision as usize; - let register_bits = 64 - precision; // Q = 50 - let m = (1usize << precision) as f64; // 16384 - - // Build histogram C[v] for v in 0..=(register_bits+1) - let hist_len = register_bits + 2; - let mut hist = vec![0u32; hist_len]; - for &r in &state.registers { - let v = r as usize; - let capped = v.min(hist_len - 1); - hist[capped] += 1; - } - - // z = m * tau((m - C[register_bits+1]) / m) - let mut z = m * hll_tau((m - hist[register_bits + 1] as f64) / m); - - // for i from register_bits down to 1: z = (z + C[i]) * 0.5 - for i in (1..=register_bits).rev() { - z += hist[i] as f64; - z *= 0.5; - } - - // z += m * sigma(C[0] / m) - z += m * hll_sigma(hist[0] as f64 / m); - - // estimate = round(0.5 / ln(2) * m^2 / z) - (0.5 / std::f64::consts::LN_2 * m * m / z).round() as u64 -} - // --------------------------------------------------------------------------- // CountSketch median frequency query (float64 counters, packed hash) // --------------------------------------------------------------------------- From e0705a4a05eea726b66c6ba5228d606dbc9a7e46 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:51:21 -0600 Subject: [PATCH 2/3] refactor(hll): tighten wire visibility and proto comments Make the HLL wire DTOs, kind_id constants and metadata builder private to wire.rs, state only what the remaining HLL proto messages are, shorten the HIP registers_of doc, and drop a stray proptest regression file. Co-Authored-By: Claude Opus 5.5 --- proto/hll/hll.proto | 38 +++++---------------------- src/proto/generated/sketchlib.v1.rs | 38 +++++---------------------- src/sketches/hll/wire.rs | 40 ++++++++++++++--------------- tests/pbt/hll.rs | 8 +++--- tests/proptest-regressions/hll.txt | 7 ----- 5 files changed, 37 insertions(+), 94 deletions(-) delete mode 100644 tests/proptest-regressions/hll.txt diff --git a/proto/hll/hll.proto b/proto/hll/hll.proto index eba61943..6a8f0793 100644 --- a/proto/hll/hll.proto +++ b/proto/hll/hll.proto @@ -10,14 +10,8 @@ option go_package = "github.com/ProjectASAP/sketchlib-go/proto/hll;hllpb"; // HyperLogLog // ============================================================================ -// HyperLogLogState is the portable state of a HyperLogLog cardinality sketch. -// -// Precision p determines the register count: num_registers = 2^precision. -// Both libraries default to precision = 14 (16 384 registers). -// -// The variant field is required because the three estimator algorithms are -// not interchangeable: loading an ErtlMLE state into a Regular estimator -// (or vice versa) produces incorrect cardinality estimates. +// HyperLogLogState is the state of one HyperLogLog cell. +// num_registers = 2^precision; variant selects the estimator. message HyperLogLogState { // HLL estimator variant. Required. HLLVariant variant = 1; @@ -26,14 +20,8 @@ message HyperLogLogState { // Default = 14 → 16 384 registers. uint32 precision = 2; - // DENSE register encoding: raw register values, length = 2^precision. - // Each byte stores the maximum (leading-zeros + 1) seen for that bucket. - // Stored as a raw byte string for compact encoding (1 byte per register). - // - // Exactly one of `registers` (dense) or `registers_sparse` (sparse) is - // populated by a producer. A decoder MUST read whichever is present; if - // both are empty the sketch is all-zero. This field stays at tag 3 so old - // dense-only readers keep working unchanged. + // Dense registers, one byte each, length 2^precision. Exactly one of + // `registers` and `registers_sparse` is set; both empty means all zero. bytes registers = 3; // ---- HIP-variant extra fields (populated only when variant = HIP) -------- @@ -47,13 +35,7 @@ message HyperLogLogState { // HIP running cardinality estimate. double hip_est = 6; - // SPARSE register encoding (additive, tag 7). Populated instead of - // `registers` (tag 3) when the number of non-zero registers is below the - // producer's dense/sparse crossover. Old readers that predate this field - // ignore tag 7 (proto skips unknown fields), so adding it is wire-backward- - // compatible. New readers decode BOTH representations — see - // HLLSparseRegisters. Reconstruction yields the identical 2^precision-byte - // dense register array, so cardinality estimation is unaffected. + // Sparse registers, set instead of `registers`; see HLLSparseRegisters. HLLSparseRegisters registers_sparse = 7; reserved 8 to 15; @@ -67,13 +49,9 @@ message HyperLogLogState { // uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 // uvarint(value) // 1..=Q+1, always 1 byte in practice // -// For low/medium cardinality (most registers zero) this is far smaller than -// the dense 2^precision-byte array. Decoding allocates a zero-filled dense -// array of length `num_registers` and writes each decoded (index, value). +// Registers absent from `packed` are zero. message HLLSparseRegisters { - // Total number of registers in the reconstructed dense array (= 2^precision). - // Carried explicitly so the decoder can size the dense buffer without - // re-deriving it from precision. + // Length of the dense register array (= 2^precision). uint32 num_registers = 1; // Varint-packed (index_delta, value) pairs for non-zero registers, in @@ -89,8 +67,6 @@ enum HLLVariant { HLL_VARIANT_REGULAR = 1; // Otmar Ertl's MLE estimator (arXiv:1702.01284). - // Originally derived from Apache DataFusion's HLL implementation. - // Not compatible with REGULAR registers. HLL_VARIANT_ERTL_MLE = 2; // Kevin Lang's HyperLogLog++ Improved Proposal (HIP). diff --git a/src/proto/generated/sketchlib.v1.rs b/src/proto/generated/sketchlib.v1.rs index 8b1303a4..0383e092 100644 --- a/src/proto/generated/sketchlib.v1.rs +++ b/src/proto/generated/sketchlib.v1.rs @@ -392,14 +392,8 @@ pub struct CountSketchCell { #[prost(double, tag = "3")] pub d_count: f64, } -/// HyperLogLogState is the portable state of a HyperLogLog cardinality sketch. -/// -/// Precision p determines the register count: num_registers = 2^precision. -/// Both libraries default to precision = 14 (16 384 registers). -/// -/// The variant field is required because the three estimator algorithms are -/// not interchangeable: loading an ErtlMLE state into a Regular estimator -/// (or vice versa) produces incorrect cardinality estimates. +/// HyperLogLogState is the state of one HyperLogLog cell. +/// num_registers = 2^precision; variant selects the estimator. #[derive(Clone, PartialEq, ::prost::Message)] pub struct HyperLogLogState { /// HLL estimator variant. Required. @@ -409,14 +403,8 @@ pub struct HyperLogLogState { /// Default = 14 → 16 384 registers. #[prost(uint32, tag = "2")] pub precision: u32, - /// DENSE register encoding: raw register values, length = 2^precision. - /// Each byte stores the maximum (leading-zeros + 1) seen for that bucket. - /// Stored as a raw byte string for compact encoding (1 byte per register). - /// - /// Exactly one of `registers` (dense) or `registers_sparse` (sparse) is - /// populated by a producer. A decoder MUST read whichever is present; if - /// both are empty the sketch is all-zero. This field stays at tag 3 so old - /// dense-only readers keep working unchanged. + /// Dense registers, one byte each, length 2^precision. Exactly one of + /// `registers` and `registers_sparse` is set; both empty means all zero. #[prost(bytes = "vec", tag = "3")] pub registers: ::prost::alloc::vec::Vec, /// HIP accumulator component kxq0. @@ -428,13 +416,7 @@ pub struct HyperLogLogState { /// HIP running cardinality estimate. #[prost(double, tag = "6")] pub hip_est: f64, - /// SPARSE register encoding (additive, tag 7). Populated instead of - /// `registers` (tag 3) when the number of non-zero registers is below the - /// producer's dense/sparse crossover. Old readers that predate this field - /// ignore tag 7 (proto skips unknown fields), so adding it is wire-backward- - /// compatible. New readers decode BOTH representations — see - /// HLLSparseRegisters. Reconstruction yields the identical 2^precision-byte - /// dense register array, so cardinality estimation is unaffected. + /// Sparse registers, set instead of `registers`; see HLLSparseRegisters. #[prost(message, optional, tag = "7")] pub registers_sparse: ::core::option::Option, } @@ -446,14 +428,10 @@ pub struct HyperLogLogState { /// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 /// uvarint(value) // 1..=Q+1, always 1 byte in practice /// -/// For low/medium cardinality (most registers zero) this is far smaller than -/// the dense 2^precision-byte array. Decoding allocates a zero-filled dense -/// array of length `num_registers` and writes each decoded (index, value). +/// Registers absent from `packed` are zero. #[derive(Clone, PartialEq, ::prost::Message)] pub struct HllSparseRegisters { - /// Total number of registers in the reconstructed dense array (= 2^precision). - /// Carried explicitly so the decoder can size the dense buffer without - /// re-deriving it from precision. + /// Length of the dense register array (= 2^precision). #[prost(uint32, tag = "1")] pub num_registers: u32, /// Varint-packed (index_delta, value) pairs for non-zero registers, in @@ -469,8 +447,6 @@ pub enum HllVariant { /// Classic HLL with bias-correction tables (Flajolet et al. 2007). Regular = 1, /// Otmar Ertl's MLE estimator (arXiv:1702.01284). - /// Originally derived from Apache DataFusion's HLL implementation. - /// Not compatible with REGULAR registers. ErtlMle = 2, /// Kevin Lang's HyperLogLog++ Improved Proposal (HIP). /// Requires hip_kxq0, hip_kxq1, hip_est to be populated. diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index bb4e0f5e..9a5f93f2 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -36,25 +36,25 @@ impl HllWireVariant for ErtlMLE { /// positional). `registers` is a msgpack `bin` (one byte per register, matching /// Go's `[]byte`) via `serde_bytes` rather than serde's default `array`. #[derive(Debug, Serialize, Deserialize)] -pub(crate) struct HllPayloadPlain { +struct HllPayloadPlain { #[serde(with = "serde_bytes")] - pub(crate) registers: Vec, + registers: Vec, } /// HIP payload: the register bin plus the three HIP running scalars. #[derive(Debug, Serialize, Deserialize)] -pub(crate) struct HllPayloadHip { +struct HllPayloadHip { #[serde(with = "serde_bytes")] - pub(crate) registers: Vec, - pub(crate) hip_kxq0: f64, - pub(crate) hip_kxq1: f64, - pub(crate) hip_est: f64, + registers: Vec, + hip_kxq0: f64, + hip_kxq1: f64, + hip_est: f64, } const HLL_KIND_FAMILY: u8 = 0x01; -pub(crate) const HLL_KIND_CLASSIC: &[u8] = &[HLL_KIND_FAMILY, 0x01]; -pub(crate) const HLL_KIND_ERTL_MLE: &[u8] = &[HLL_KIND_FAMILY, 0x02]; -pub(crate) const HLL_KIND_HIP: &[u8] = &[HLL_KIND_FAMILY, 0x03]; +const HLL_KIND_CLASSIC: &[u8] = &[HLL_KIND_FAMILY, 0x01]; +const HLL_KIND_ERTL_MLE: &[u8] = &[HLL_KIND_FAMILY, 0x02]; +const HLL_KIND_HIP: &[u8] = &[HLL_KIND_FAMILY, 0x03]; /// Descriptor metadata for an HLL sketch (ASAPv1 §2), serialized as a msgpack /// **map** (`to_vec_named`) with keys in this declaration order — the canonical @@ -64,21 +64,21 @@ pub(crate) const HLL_KIND_HIP: &[u8] = &[HLL_KIND_FAMILY, 0x03]; /// decode fail closed on any unexpected key rather than silently dropping it. #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct HllMetadata { - pub(crate) metadata_version: u8, - pub(crate) hash_profile_id: String, - pub(crate) hash_algorithm: String, - pub(crate) seed_derivation: String, - pub(crate) input_encoding: String, - pub(crate) seed_list: Vec, - pub(crate) canonical_seed_index: u32, - pub(crate) precision: u32, +struct HllMetadata { + metadata_version: u8, + hash_profile_id: String, + hash_algorithm: String, + seed_derivation: String, + input_encoding: String, + seed_list: Vec, + canonical_seed_index: u32, + precision: u32, } /// Builds the HLL descriptor metadata from the hasher's [`HashProfile`], so the /// wire bytes truthfully describe how the sketch was hashed (rather than /// hardcoding the standard profile). -pub(crate) fn hll_metadata(precision: u32) -> HllMetadata { +fn hll_metadata(precision: u32) -> HllMetadata { HllMetadata { metadata_version: 1, hash_profile_id: H::PROFILE_ID.to_string(), diff --git a/tests/pbt/hll.rs b/tests/pbt/hll.rs index 7159350a..373bf3c3 100644 --- a/tests/pbt/hll.rs +++ b/tests/pbt/hll.rs @@ -442,11 +442,9 @@ macro_rules! hip_laws { const M: usize = <$buckets>::NUM_REGISTERS; const P: u32 = <$buckets>::PRECISION as u32; - /// The registers, read back out of the ASAPv1 payload. The struct - /// holds them privately behind the `kxq0`/`kxq1` accumulators, and - /// those accumulators are what the increment law checks, so the - /// envelope is the one view of the registers that stays independent - /// of it. + /// The registers, read out of the ASAPv1 payload: the struct keeps + /// them private, and this view is independent of the `kxq0`/`kxq1` + /// accumulators the increment law checks. fn registers_of(sketch: &S) -> Vec { let bytes = sketch.serialize_to_bytes().expect("encode"); let lengths_at = 8 + bytes[7] as usize; diff --git a/tests/proptest-regressions/hll.txt b/tests/proptest-regressions/hll.txt deleted file mode 100644 index cb8fb6e2..00000000 --- a/tests/proptest-regressions/hll.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 115f567853acb20cbde319427f1d20dbe1769471bdccb8f153dab74ff382dc7a # shrinks to stream = [] From 0c206994cfa5a853ecd01bd631cb11cb96928af9 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 01:09:52 -0600 Subject: [PATCH 3/3] test(hll): check the P14 ASAPv1 goldens for all three estimators Bump asapv1_golden to the commit adding hll_classic_p14, hll_ertl_mle_p14 and hll_hip_p14, built from fixed registers holding the maximum P14 rank, and check each in both directions. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++ asapv1_golden | 2 +- docs/tests.md | 3 ++ tests/asapv1_golden.rs | 64 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5266204e..7ce2ef4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ signals a backwards-compatible change. - **HyperLogLog `from_storage` constructors.** `HyperLogLogImpl::from_storage(registers)` and `HyperLogLogHIPImpl::from_storage(registers, kxq0, kxq1, est)` build a sketch from known register storage (and HIP running state). +- **HLL P14 golden fixtures** `hll_classic_p14`, `hll_ertl_mle_p14` and + `hll_hip_p14` in `asapv1_golden/`, checked in both directions by + `tests/asapv1_golden.rs`. Their registers touch the first, a middle and the + last index and hold 51, the largest P14 rank. ### Changed diff --git a/asapv1_golden b/asapv1_golden index 8287a4ae..43b6cd2d 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit 8287a4ae04e6cfbfb3ed33dafbf1dac77d8f7fc1 +Subproject commit 43b6cd2d3f14688f56edda5e8c987c16fbd4f919 diff --git a/docs/tests.md b/docs/tests.md index 66513dab..9b3227b4 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -2095,8 +2095,11 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `cs_i64_fast_2x4_matches_golden` | The fast-path mode reaches the bytes, so the same matrix encodes differently per path. | Verifies `Count::, FastPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` serializes to `asapv1_golden/cs_i64_fast_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, and that the fixture differs from `cs_i64_regular_2x4.hex` built from the same matrix. | | `cs_i64_regular_2x4_matches_golden` | A known signed Count-Sketch matrix serializes to its checked-in golden bytes. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` — sweeping the msgpack integer widths in both directions — serializes to `asapv1_golden/cs_i64_regular_2x4.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `4`, the eight cells, and a byte-identical re-serialization. | | `hll_classic_p12_matches_golden` | A known P12 Classic register pattern serializes to its golden bytes, which round-trip unchanged. | Builds `HyperLogLogP12::::from_storage` over 4,096 registers that are zero except `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_classic_p12.hex`, and that `deserialize_from_bytes` on the fixture recovers those registers and re-serializes identically. | +| `hll_classic_p14_matches_golden` | A known P14 Classic register pattern, holding the maximum rank, serializes to its golden bytes, which round-trip unchanged. | Builds `HyperLogLogP14::::from_storage` over 16,384 registers that are zero except `[0] = 1`, `[1] = 7`, `[8192] = 42` and `[16383] = 51`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_classic_p14.hex`, and that `deserialize_from_bytes` on the fixture recovers those registers and re-serializes identically. | | `hll_ertl_mle_p12_matches_golden` | The ErtlMLE kind_id reaches the bytes for the same P12 register pattern. | Builds `HyperLogLogP12::::from_storage` over the shared P12 pattern, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_ertl_mle_p12.hex`, and that `deserialize_from_bytes` recovers the registers and re-serializes identically. | +| `hll_ertl_mle_p14_matches_golden` | The ErtlMLE kind_id reaches the bytes for the same P14 register pattern. | Builds `HyperLogLogP14::::from_storage` over the shared P14 pattern, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_ertl_mle_p14.hex`, and that `deserialize_from_bytes` recovers the registers and re-serializes identically. | | `hll_hip_p12_matches_golden` | The HIP variant's three running scalars travel on the wire beside its registers. | Builds `HyperLogLogHIPP12::from_storage` over the shared P12 pattern with `kxq0` `1.5`, `kxq1` `2.5` and `est` `3.0`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_hip_p12.hex`, that the decode's `Debug` form (registers and all three scalars) equals the built sketch's, and that it re-serializes identically. | +| `hll_hip_p14_matches_golden` | The P14 HIP registers and running scalars match their golden bytes in both directions. | Builds `HyperLogLogHIPP14::from_storage` over the shared P14 pattern with `kxq0` `16380.5`, `kxq1` `0.25` and `est` `4.125`, verifies `serialize_to_bytes()` equals `asapv1_golden/hll_hip_p14.hex`, that the decode's `Debug` form equals the built sketch's, and that it re-serializes identically. | | `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `KLL::::init_kll_with_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index b820c978..a1710e33 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -9,8 +9,9 @@ //! See `asapv1_golden/README.md`. use asap_sketchlib::{ - Classic, Count, CountMin, ErtlMLE, FastPath, HllBucketListP12, HllRegisterStorage, - HyperLogLogHIPP12, HyperLogLogP12, KLL, RegularPath, Vector2D, + Classic, Count, CountMin, ErtlMLE, FastPath, HllBucketListP12, HllBucketListP14, + HllRegisterStorage, HyperLogLogHIPP12, HyperLogLogHIPP14, HyperLogLogP12, HyperLogLogP14, KLL, + RegularPath, Vector2D, }; fn decode_hex(s: &str) -> Vec { @@ -25,6 +26,9 @@ fn decode_hex(s: &str) -> Vec { const GOLDEN_CLASSIC: &str = include_str!("../asapv1_golden/hll_classic_p12.hex"); const GOLDEN_ERTL: &str = include_str!("../asapv1_golden/hll_ertl_mle_p12.hex"); const GOLDEN_HIP: &str = include_str!("../asapv1_golden/hll_hip_p12.hex"); +const GOLDEN_CLASSIC_P14: &str = include_str!("../asapv1_golden/hll_classic_p14.hex"); +const GOLDEN_ERTL_P14: &str = include_str!("../asapv1_golden/hll_ertl_mle_p14.hex"); +const GOLDEN_HIP_P14: &str = include_str!("../asapv1_golden/hll_hip_p14.hex"); const GOLDEN_CMS_I64: &str = include_str!("../asapv1_golden/cms_i64_regular_2x3.hex"); const GOLDEN_CMS_F64: &str = include_str!("../asapv1_golden/cms_f64_fast_2x3.hex"); const GOLDEN_CS_REGULAR: &str = include_str!("../asapv1_golden/cs_i64_regular_2x4.hex"); @@ -49,6 +53,23 @@ fn p12_storage() -> HllBucketListP12 { storage } +/// The known P14 register pattern: the first, a middle and the last index, +/// up to 51, the largest rank a P14 register holds. +fn p14_registers() -> Vec { + let mut r = vec![0u8; 16384]; + r[0] = 1; + r[1] = 7; + r[8192] = 42; + r[16383] = 51; + r +} + +fn p14_storage() -> HllBucketListP14 { + let mut storage = HllBucketListP14::default(); + storage.as_mut_slice().copy_from_slice(&p14_registers()); + storage +} + const I64_VALS: [[i64; 3]; 2] = [[0, 1, 127], [128, 300, 65536]]; const F64_VALS: [[f64; 3]; 2] = [[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]; @@ -102,6 +123,45 @@ fn hll_hip_p12_matches_golden() { assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); } +#[test] +fn hll_classic_p14_matches_golden() { + let want = decode_hex(GOLDEN_CLASSIC_P14); + + let sketch = HyperLogLogP14::::from_storage(p14_storage()); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "Classic P14 bytes diverge from golden"); + + let decoded = HyperLogLogP14::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.registers_as_slice(), p14_registers().as_slice()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} + +#[test] +fn hll_ertl_mle_p14_matches_golden() { + let want = decode_hex(GOLDEN_ERTL_P14); + + let sketch = HyperLogLogP14::::from_storage(p14_storage()); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "Ertl-MLE P14 bytes diverge from golden"); + + let decoded = HyperLogLogP14::::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(decoded.registers_as_slice(), p14_registers().as_slice()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} + +#[test] +fn hll_hip_p14_matches_golden() { + let want = decode_hex(GOLDEN_HIP_P14); + + let sketch = HyperLogLogHIPP14::from_storage(p14_storage(), 16380.5, 0.25, 4.125); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "HIP P14 bytes diverge from golden"); + + let decoded = HyperLogLogHIPP14::deserialize_from_bytes(&want).expect("decode"); + assert_eq!(format!("{decoded:?}"), format!("{sketch:?}")); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} + // --------------------------------------------------------------------------- // Count-Min: build known matrix state -> serialize == golden, round-trips. // ---------------------------------------------------------------------------