From a64c435c674ca5cbb84a5719bf3ab34cc87a8ef2 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:13:41 +0000 Subject: [PATCH 1/4] refactor!: remove the protobuf format (#206) Delete `proto/`, the `proto` module and its vendored generated code, `tools/gen-proto`, the `prost` and `bytes` dependencies, the CI job that checked the vendored code, and the proto-only tests and fixtures (`tests/xtest_consumer.rs`, `tests/sketches_go_parity_probe.rs`, the envelope goldens). The KLL `KLLState` value-offset encoding goes with it. Closes #206 Closes #188 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 28 - .github/workflows/docs.yml | 4 - CHANGELOG.md | 3 + Cargo.toml | 7 - README.md | 25 - docs/ddsketch_signed_values.md | 1 - docs/development_guide.md | 10 +- docs/features.md | 2 - docs/library_map.md | 16 - docs/tests.md | 26 - proto/cocosketch/cocosketch.proto | 35 - proto/common/common.proto | 98 -- proto/countminsketch/countminsketch.proto | 116 -- proto/countsketch/countsketch.proto | 123 -- proto/ddsketch/ddsketch.proto | 75 - proto/elasticsketch/elasticsketch.proto | 41 - proto/hll/hll.proto | 134 -- proto/hydra/hydra.proto | 80 -- proto/kll/kll.proto | 105 -- proto/sketchlib.proto | 100 -- proto/univmon/univmon.proto | 50 - src/lib.rs | 3 - src/message_pack_format/portable/kll.rs | 455 +----- src/proto.rs | 20 - src/proto/generated/sketchlib.v1.rs | 923 ------------ src/sketches/testdata/cms_envelope_golden.hex | 1 - src/sketches/testdata/hll_envelope_golden.hex | 1 - tests/cs_envelope_golden.hex | 1 - tests/ddsketch_signed.rs | 31 - tests/sketches_go_parity_probe.rs | 279 ---- tests/xtest_consumer.rs | 1246 ----------------- tools/gen-proto/Cargo.lock | 646 --------- tools/gen-proto/Cargo.toml | 15 - tools/gen-proto/src/main.rs | 94 -- 34 files changed, 6 insertions(+), 4788 deletions(-) delete mode 100644 proto/cocosketch/cocosketch.proto delete mode 100644 proto/common/common.proto delete mode 100644 proto/countminsketch/countminsketch.proto delete mode 100644 proto/countsketch/countsketch.proto delete mode 100644 proto/ddsketch/ddsketch.proto delete mode 100644 proto/elasticsketch/elasticsketch.proto delete mode 100644 proto/hll/hll.proto delete mode 100644 proto/hydra/hydra.proto delete mode 100644 proto/kll/kll.proto delete mode 100644 proto/sketchlib.proto delete mode 100644 proto/univmon/univmon.proto delete mode 100644 src/proto.rs delete mode 100644 src/proto/generated/sketchlib.v1.rs delete mode 100644 src/sketches/testdata/cms_envelope_golden.hex delete mode 100644 src/sketches/testdata/hll_envelope_golden.hex delete mode 100644 tests/cs_envelope_golden.hex delete mode 100644 tests/sketches_go_parity_probe.rs delete mode 100644 tests/xtest_consumer.rs delete mode 100644 tools/gen-proto/Cargo.lock delete mode 100644 tools/gen-proto/Cargo.toml delete mode 100644 tools/gen-proto/src/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baeb15c8..f75ceb1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,31 +111,3 @@ jobs: - name: Run tests run: cargo test --features experimental --locked --verbose - - proto-vendored-up-to-date: - name: Vendored proto code is up to date - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo dependencies and build artifacts - uses: Swatinem/rust-cache@v2 - with: - workspaces: tools/gen-proto -> tools/gen-proto/target - - - name: Regenerate vendored proto code - run: cargo run --manifest-path tools/gen-proto/Cargo.toml --locked - - - name: Verify src/proto/generated/ has no drift - run: | - if ! git diff --exit-code -- src/proto/generated; then - echo "::error::src/proto/generated/ is out of date. Run:" >&2 - echo " cargo run --manifest-path tools/gen-proto/Cargo.toml" >&2 - echo "and commit the result." >&2 - exit 1 - fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3458961f..b31ab278 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,8 +5,6 @@ on: branches: [ "main" ] paths: - "src/**" - - "proto/**" - - "tools/gen-proto/**" - "Cargo.toml" - "Cargo.lock" - "README.md" @@ -16,8 +14,6 @@ on: branches: [ "main" ] paths: - "src/**" - - "proto/**" - - "tools/gen-proto/**" - "Cargo.toml" - "Cargo.lock" - "README.md" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cbd1fc9..93ed568e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -331,6 +331,9 @@ signals a backwards-compatible change. ### Removed +- **Protobuf** (breaking). Module `proto`, the `.proto` sources, + `tools/gen-proto` and the `prost`/`bytes` dependencies; with them the KLL + value-offset sample encoding. - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); `PRECOMPUTED_SAMPLE_RATE_1PERCENT` and module `common::precompute_sample2`; module `common::hashspec` (`HashSpec`, `derive_index`, `derive_sign`, diff --git a/Cargo.toml b/Cargo.toml index d9314dc9..fa89c4e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,6 @@ serde_bytes = "0.11" rmp-serde = "1.3.0" twox-hash = "2.1.2" smallvec = "1.13.2" -prost = "0.13" -bytes = "1" xxhash-rust = { version = "0.8", features = ["xxh32"] } core_affinity = { version = "0.8", optional = true } crossbeam-channel = { version = "0.5", optional = true } @@ -39,11 +37,6 @@ default = [] experimental = [] octo-runtime = ["core_affinity", "crossbeam-channel"] -# Note: there are intentionally no `[build-dependencies]`. The prost output -# under `src/proto/generated/` is checked in and refreshed by the maintainer -# tool at `tools/gen-proto/`. This keeps the published crate pure-Rust with -# no requirement on `protoc` for downstream consumers. - [lib] crate-type = ["rlib"] name = "asap_sketchlib" diff --git a/README.md b/README.md index 76d99348..dce7176d 100644 --- a/README.md +++ b/README.md @@ -220,31 +220,6 @@ cargo build --features experimental cargo test --features "experimental octo-runtime" ``` -## Protobuf code generation - -`asap_sketchlib` is a pure-Rust crate: building it does **not** require -`protoc` or any build script. The Rust types generated from -`proto/**/*.proto` are vendored into `src/proto/generated/` and refreshed -manually by maintainers using the in-repo tool at `tools/gen-proto/`. - -Downstream users can therefore simply add the crate to their `Cargo.toml` and -build it like any other pure-Rust dependency. - -### For maintainers - -After editing any `.proto` file, regenerate the vendored output and commit -the result: - -```bash -cargo run --manifest-path tools/gen-proto/Cargo.toml -git add src/proto/generated -``` - -The tool uses `prost-build` together with the `protoc-bin-vendored` binary, -so no system `protoc` is required to regenerate either. CI rejects any pull -request whose committed `src/proto/generated/` does not match the result of -running this command on its current `.proto` sources. - ## FAQ ### When is Apache DataSketches a better fit? diff --git a/docs/ddsketch_signed_values.md b/docs/ddsketch_signed_values.md index 67350cf2..d9d0c5ff 100644 --- a/docs/ddsketch_signed_values.md +++ b/docs/ddsketch_signed_values.md @@ -12,7 +12,6 @@ If interpolation combines endpoints x and y with weight t, the absolute error is - Native ASAPv1: every state is metadata version 1 with the eight-field payload, negative counts, negative offset and zero count included. `DdSketchDelta` (`0x05 0x80`) carries both stores and a zero-count increment. - Portable MessagePack: positive-only snapshots keep their three fields; signed snapshots append the same three fields. Positive-only deltas keep their two arrays; signed deltas append negative indices, negative counts, and zero-count increment. New readers accept both layouts. -- Protobuf: `DdSketchState` adds fields 16–18; `DdSketchDelta` adds fields 16–17. `DdSketch::to_proto` / `from_proto` preserve all stores. Old protobuf readers silently ignore unknown fields: upgrade Collector, backend, and Go producer schema/mapping together before emitting signed protobuf. This is not DataDog's binary wire format. - Octo: `DdSketchBucketDelta` now requires a `sign: DdSign` discriminator. Existing positive delta constructors should use `DdSign::Positive`. Worker residual keys are `(DdSign, i32)`. Both promotion and flush preserve all three categories; partial counts still need flushing before complete queries. Go implementations of the ASAP wire formats and downstream dependency pins are not changed by this Rust library patch. diff --git a/docs/development_guide.md b/docs/development_guide.md index 459617e1..55a66c12 100644 --- a/docs/development_guide.md +++ b/docs/development_guide.md @@ -36,14 +36,6 @@ cargo test --features octo-runtime --locked cargo test --features experimental --locked ``` -Vendored proto code must have no drift — regenerate, then confirm the tree is -unchanged: - -```bash -cargo run --manifest-path tools/gen-proto/Cargo.toml --locked -git diff --exit-code -- src/proto/generated -``` - Rustdoc, warnings denied: ```bash @@ -51,5 +43,5 @@ RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features --locked ``` The Rust CI workflow skips `**/*.md` and `docs/**`; the docs workflow runs on -`docs/**`, `README.md`, `src/**`, `proto/**`, `tools/gen-proto/**` and the +`docs/**`, `README.md`, `src/**` and the manifests. A documentation-only change is therefore gated by rustdoc alone. diff --git a/docs/features.md b/docs/features.md index 2f5f5d63..b831e285 100644 --- a/docs/features.md +++ b/docs/features.md @@ -113,8 +113,6 @@ MessagePack (`rmp-serde`) support. **serde support** means the type derives `Ser | EhBucketSketch | Yes | Yes | | ExponentialHistogram | No | Yes | -Protobuf (prost): `.proto` definitions exist for CountMinSketch, Count, HLL, DDSketch, KLL, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. - ### API Stability - Public APIs are stabilizing but may still change in naming and structure diff --git a/docs/library_map.md b/docs/library_map.md index 3c6a6dfa..00e8134e 100644 --- a/docs/library_map.md +++ b/docs/library_map.md @@ -41,19 +41,3 @@ - The large precomputed hash/sample tables are built lazily at runtime via `std::sync::LazyLock` in `src/common/precompute_hash.rs` and `src/common/precompute_sample.rs`. - -## Proto code generation - -- `proto/**/*.proto` is the cross-language wire-format source of truth shared - with `sketchlib-go`. -- The corresponding Rust types are **vendored** under - `src/proto/generated/sketchlib.v1.rs` and re-exported by `src/proto.rs` as - `crate::proto::sketchlib`. Downstream users therefore build the crate as - pure Rust without needing `protoc` or any build script. -- To regenerate after editing any `.proto` file, run from the repository root: - - ```bash - cargo run --manifest-path tools/gen-proto/Cargo.toml - ``` - - CI enforces that the committed file matches the result of regeneration. diff --git a/docs/tests.md b/docs/tests.md index 5468b62d..9686d50e 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -1503,13 +1503,6 @@ Test file: [`src/message_pack_format/portable/kll.rs`](../src/message_pack_forma | `test_msgpack_round_trip` | The round trip preserves `k`, the count, and the extremes. | Round-trips a `k` `200` sketch fed `1.0..=5.0` and verifies the decode reports `k` `200`, `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | | `test_aggregate_kll` | `aggregate_kll` builds a decodable sketch from a value array. | For `aggregate_kll(200, [1.0, 2.0, 3.0, 4.0, 5.0])`, verifies the returned bytes decode to a sketch with `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | | `test_aggregate_kll_empty` | An empty value array produces nothing. | Verifies `aggregate_kll(200, &[])` returns `None`. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The raw-`f64` proto envelope is byte-identical to the golden. | For `Kll::with_k_and_seed(200, 42)` fed `1.0..=50.0`, builds a `KllState` from the `wire_k`/`m`/`num_levels`/`level_bounds_top_down`/`items_top_down` accessors plus the `CoinState` from `coin_state()`, leaving `offset` `0.0`, `value_scale` `0`, and `residuals` empty, wraps it in a `SketchEnvelope` with `format_version` `1`, and verifies the prost encoding equals a 423-byte inline golden. | -| `value_offset_round_trip_exact_fixed_point` | The fixed-point form round-trips integer and fixed-decimal series bit-exactly. | Verifies the 200 values `1_000_000 + 7i` encode at `value_scale` `0` and decode back equal, and that the 128 values `100.0 + 0.001i` encode at a scale in `-3..=0` and decode back equal. | -| `value_offset_guard_rejects_irrational` | A value with no exact decimal scale falls back to raw `f64`. | Verifies `encode_value_offset` returns `None` for `[0.0, 1.0, PI, 3.0]`, for the empty slice, for `[1.0, NAN]`, and for `[1.0, INFINITY]`. | -| `proto_dual_read_round_trip_both_forms` | The decoder reads the raw-`f64` and the value-offset form to the same answer. | For `Kll::with_k_and_seed(200, 7)` fed `1.0..=5000.0`, builds one `KllState` carrying `items_top_down()` and one carrying the `encode_value_offset` triple with `items` empty, and verifies `KllProtoItems::from_state` yields identical items and identical quantiles at `0.0`, `0.1`, `0.25`, `0.5`, `0.75`, `0.9`, and `1.0`. | -| `proto_rejects_both_forms_populated` | A state carrying both forms at once is refused. | Verifies a `KllState` with `k` `200`, `m` `8`, `num_levels` `1`, `levels` `[0, 2]`, `items` `[1.0, 2.0]`, and `residuals` `[1, 2]` fails to decode. | -| `golden_value_offset_envelope_from_go` | The value-offset envelope the Go side emits decodes to the right samples. | Decodes an 80-byte golden and verifies the `KllState` leaves `items` empty, carries `value_scale` `0`, `offset` `1.0`, and 50 residuals, and that the dual-read recovers `1.0..=50.0` with `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | -| `golden_value_offset_envelope_for_go` | The value-offset envelope this encoder emits is pinned byte for byte. | For `1.0..=50.0`, verifies `encode_value_offset` gives `offset` `1.0` and `value_scale` `0`, that the `SketchEnvelope` around a `KllState` with `k` `200`, `m` `8`, `num_levels` `1`, `levels` `[0, 50]`, and no coin encodes to the 76-byte `RUST_GOLDEN_HEX`, and that the dual-read recovers `1.0..=50.0`. | ### Portable Sampling Rescale @@ -2362,25 +2355,6 @@ Test file: [`tests/spec_self_tests.rs`](../tests/spec_self_tests.rs) | `simultaneous_kappa_reflects_the_corrected_threshold` | The simultaneous `kappa` search reaches its target and is wider than a `d/2 + 1` threshold demands. | For `CountSketchSpec::new(4, 2048)`, verifies `simultaneous_kappa(512, SIMULTANEOUS_LEVEL)` at a level of `1e-3` yields a `kappa` whose `key_failure_at(kappa)` is at most `1e-3 / 512`, and that this `kappa` is strictly larger than the one a geometric-then-bisection search over `binomial_tail_ge(4, 3, 1.0 / kappa)` finds for the same target. | | `two_same_direction_bad_rows_move_an_averaged_four_row_median_out_of_band` | Two same-direction bad rows break a four-row averaged median, and one does not. | With an inline median-of-four `(v[1] + v[2]) / 2`, a true value of `100.0` and an error scale of `10.0` (band `[90, 110]`), verifies `[95, 105, 100, 10_000]` stays in band, `[95, 105, 10_000, 10_000]` leaves it, and `[95, 105, -10_000, 10_000]` stays in band, asserting the bad-row count of each fixture first. | -### Proto Envelope Parity Probe - -Test file: [`tests/sketches_go_parity_probe.rs`](../tests/sketches_go_parity_probe.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `sketches_count_sketch_fastpath_matches_go_count_sketch_envelope` | A `sketches::CountSketch` fast-path matrix encodes to the checked-in Count-Sketch envelope bytes. | Fills `CountSketch, FastPath, DefaultXxHasher>` at `3x512` with 25 `update_weighted` calls over the five keys `k-a`..`k-e`, packs the row-major cells and per-row L2 sums into a `CountSketchState` with `counter_type` `INT64` inside a `SketchEnvelope { format_version: 1 }` with no producer or hash_spec, and verifies the prost encoding equals the 1,577-byte fixture `tests/cs_envelope_golden.hex`. | -| `sketches_countmin_fastpath_matches_go_count_min_envelope` | A `sketches::CountMinSketch` fast-path matrix encodes to the checked-in Count-Min envelope bytes. | Fills `CountMinSketch, FastPath, DefaultXxHasher>` at `4x2048` with 50 `update_weighted` calls over the ten keys `flow-0`..`flow-9`, packs the cells cast to `i64` plus per-row L1 and L2 sums into a `CountMinState` with `counter_type` `INT64`, and verifies the prost encoding matches the 8,275-byte fixture `src/sketches/testdata/cms_envelope_golden.hex` in length and then byte for byte. | -| `sketches_ddsketch_matches_go_envelope` | A `sketches::DdSketch` encodes to the DDSketch envelope hex constant held in the test file. | Fills `DdSketch::new(0.01)` with `1..=50` as `f64`, encodes a `DdSketchState` carrying `positive_store_counts`, `positive_store_offset` and an alpha round-tripped through gamma (`gamma = (1 + a) / (1 - a)`, `alpha_wire = (gamma - 1) / (gamma + 1)`), and verifies the envelope matches the 403-byte inline hex constant in length and then byte for byte; the comment on that constant records it as this crate's own output rather than a fixture from another implementation, so no cross-language parity is pinned. | -| `sketches_hll_classic_matches_go_envelope` | An Ertl P14 register set encodes to the checked-in HyperLogLog envelope bytes. | Feeds `HyperLogLogImpl` the `hash64_seeded(DEFAULT_CANONICAL_SEED_INDEX, ...)` digests of the IEEE-754 little-endian bytes of `1.0..=50.0` through `update_with_hash`, encodes a `HyperLogLogState` with `variant` `2`, `precision` `14`, the dense registers, zeroed HIP scalars and no sparse form, and verifies the envelope matches the 16,398-byte fixture `src/sketches/testdata/hll_envelope_golden.hex` in length and then byte for byte; despite the name, the variant under test is `Ertl`, not `Classic`. | - -### Cross-Language Proto Consumer - -Test file: [`tests/xtest_consumer.rs`](../tests/xtest_consumer.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes nine `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `hll.pb` an Ertl cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | - ### HLL Custom Precision Test file: [`tests/hll_custom_precision.rs`](../tests/hll_custom_precision.rs) diff --git a/proto/cocosketch/cocosketch.proto b/proto/cocosketch/cocosketch.proto deleted file mode 100644 index 041b3c04..00000000 --- a/proto/cocosketch/cocosketch.proto +++ /dev/null @@ -1,35 +0,0 @@ -// cocosketch.proto — Portable wire format for CocoSketch flow-size estimator state. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/cocosketch;cocosketchpb"; - -// ============================================================================ -// CocoSketch -// ============================================================================ - -// CocoSketchState is the portable state of a CocoSketch flow-size estimator. -// -// The bucket table is stored flat in row-major order (d rows × width cols). -// Element [r][c] is at index r * width + c. -message CocoSketchState { - // Number of hash rows. - uint32 d = 1; - - // Number of buckets per row. - uint32 width = 2; - - // Full hash stored in each bucket, length = d * width. - // Zero means the bucket is empty (has_keys[i] == false). - repeated uint64 hashes = 3 [packed = true]; - - // Cumulative count value per bucket, length = d * width. - repeated uint64 vals = 4 [packed = true]; - - // Whether each bucket holds a valid key entry, length = d * width. - repeated bool has_keys = 5 [packed = true]; - - reserved 6 to 15; -} diff --git a/proto/common/common.proto b/proto/common/common.proto deleted file mode 100644 index 3776f520..00000000 --- a/proto/common/common.proto +++ /dev/null @@ -1,98 +0,0 @@ -// common.proto — Shared types used across all sketch-specific proto files. -// -// Contains: ProducerInfo, HashSpec, HashAlgorithm, SeedDerivation, CounterType. -// These are referenced by the per-sketch protos and the top-level envelope. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/common;commonpb"; - -// ============================================================================ -// Producer Info -// ============================================================================ - -// ProducerInfo records which library version created this envelope. -message ProducerInfo { - // One of "sketchlib-go" or "asap_sketchlib". - string library = 1; - - // Semantic version string of the producing library, e.g. "0.1.0". - string version = 2; -} - -// ============================================================================ -// Hash Specification -// ============================================================================ - -// HashSpec fully describes the hashing configuration so both sides can confirm -// they will compute identical bucket indices. -// -// Both libraries share: -// canonical_seed_index = 5 (CanonicalHashSeed / CANONICAL_HASH_SEED) -// seed_list = the 20-entry table in hash.go / hash.rs -// seed_derivation = ADDITIVE_OFFSET for per-row seeds -message HashSpec { - // Hash algorithm used for all sketch operations. - HashAlgorithm algorithm = 1; - - // Index into seed_list used as the primary (canonical) hash seed. - // Both libraries default to 5. - uint32 canonical_seed_index = 2; - - // Complete seed table. Must be identical on both sides for correct results. - // Both libraries currently use exactly 20 seeds; producers always emit all 20. - repeated uint64 seed_list = 3 [packed = true]; - - // How per-row seeds are derived from the base seed index. - SeedDerivation seed_derivation = 4; -} - -// HashAlgorithm enumerates supported hash algorithms. -// Only XXH3_64 is cross-language stable for the current implementation. -enum HashAlgorithm { - HASH_ALGORITHM_UNSPECIFIED = 0; - - // xxHash3 64-bit with seed (zeebo/xxh3 in Go; twox-hash XxHash3_64 in Rust). - // This is the standard algorithm for both libraries. - HASH_ALGORITHM_XXH3_64 = 1; - - // xxHash3 128-bit with seed. Used for HLL register indexing and packed matrix - // hashes that exceed 64 bits. - HASH_ALGORITHM_XXH3_128 = 2; - - // xxHash 64-bit (non-XXH3). Reserved for legacy data; not produced by either - // current library. - HASH_ALGORITHM_XXHASH64 = 3; -} - -// SeedDerivation describes how per-row seed indices are produced. -enum SeedDerivation { - SEED_DERIVATION_UNSPECIFIED = 0; - - // Each row r uses seed_list[(base + r) % len(seed_list)]. - // Standard derivation for multi-row sketches (CountMin, CountSketch). - SEED_DERIVATION_ADDITIVE_OFFSET = 1; - - // All rows share a single hash (Packed64 or Packed128 fast-path). - // Applicable only when the total bit-width fits in 64 or 128 bits. - SEED_DERIVATION_PACKED = 2; -} - -// ============================================================================ -// Counter Type -// ============================================================================ - -// CounterType identifies the numeric type used to store matrix counters. -// Go defaults to FLOAT64; Rust defaults to INT32. -// Determines which counts_* array is populated in CountMinState and -// CountSketchState. -enum CounterType { - COUNTER_TYPE_UNSPECIFIED = 0; - COUNTER_TYPE_INT32 = 1; // Rust default for CountMin / CountSketch. - COUNTER_TYPE_INT64 = 2; - // INT128 is stored as pairs of int64 (hi, lo) interleaved in counts_int. - COUNTER_TYPE_INT128 = 3; - COUNTER_TYPE_FLOAT64 = 4; // Go default for CountMin / CountSketch. -} diff --git a/proto/countminsketch/countminsketch.proto b/proto/countminsketch/countminsketch.proto deleted file mode 100644 index 27e09d3b..00000000 --- a/proto/countminsketch/countminsketch.proto +++ /dev/null @@ -1,116 +0,0 @@ -// countminsketch.proto — Portable wire format for Count-Min Sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -import "common/common.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/countminsketch;countminsketchpb"; - -// ============================================================================ -// Count-Min Sketch -// ============================================================================ - -// CountMinState is the portable state of a Count-Min Sketch. -// -// The counter matrix is stored flat in row-major order: -// element [r][c] is at index r * cols + c. -// -// The Go implementation tracks auxiliary sum/L1/L2 arrays that the Rust -// implementation does not produce. These fields are optional; consumers -// that need them must recompute from raw insertions if absent. -message CountMinState { - // Matrix dimensions. Both must be > 0. - uint32 rows = 1; - uint32 cols = 2; - - // Indicates which counts_* field below is populated. - CounterType counter_type = 3; - - // Integer counters in row-major order, length = rows * cols. - // Populated when counter_type ∈ {INT32, INT64}. - // Use sint64 for efficient varint encoding of near-zero values. - // For INT128 counters, values are interleaved as (hi_i, lo_i) pairs, - // making the effective length 2 * rows * cols. - repeated sint64 counts_int = 4 [packed = true]; - - // Floating-point counters in row-major order, length = rows * cols. - // Populated when counter_type = FLOAT64 (Go producer). - repeated double counts_float = 5 [packed = true]; - - // ---- Optional Go-extension fields ---------------------------------------- - // These are produced by sketchlib-go and absent from asap_sketchlib producers. - // Consumers should treat absence as "unknown; recompute if needed". - - // Sum-weighted counter matrix (same layout as counts_*), length = rows * cols. - repeated double sum_counts = 6 [packed = true]; - - // Sum-of-squares counter matrix, length = rows * cols. - repeated double sum2_counts = 7 [packed = true]; - - // Per-row L1 norms, length = rows. - repeated double l1 = 8 [packed = true]; - - // Per-row L2 norms, length = rows. Recompute if absent. - repeated double l2 = 9 [packed = true]; - - reserved 10 to 15; -} - -// ============================================================================ -// Count-Min Sketch Delta -// ============================================================================ - -// CountMinDelta carries only the matrix cells that changed by at least the -// threshold between two consecutive snapshots, plus the full per-row L1/L2 -// norm deltas (one entry per row, negligible size). -// -// Cells apply additively on the receiver: matrix[row][col] += d_count. CMS -// counters only ever grow, but the delta encodes a signed count (sint64) so -// the same wire form covers weighted/decay variants. -// -// The packed cell encoding (cell_rows / cell_cols / d_counts, tags 9-11) is -// the canonical form; the legacy per-cell message (tag 3) is retained only so -// payloads from older producers still decode. This message is byte-identical -// to the Go reference implementation's CountMinDelta so the two runtimes emit -// byte-identical delta frames for identical window state (cross-language byte -// parity). -// -// CountMinDelta is structurally identical to CountSketchDelta: rows, cols, the -// packed cell encoding, per-row norm deltas, and an optional repeated hh_keys -// at the SAME tag number (6). Both sketches can track heavy hitters; whether -// hh_keys is populated is a control-plane decision (it is empty/omitted when -// heavy-hitter tracking is not enabled for this sketch). -message CountMinDelta { - uint32 rows = 1; - uint32 cols = 2; - // Deprecated: use cell_rows/cell_cols/d_counts (tags 9-11). Retained for - // backward-compatible decode of payloads from older producers. - repeated CountMinCell cells_legacy = 3; - // Per-row L1 norm deltas, length = rows. - repeated double l1 = 4 [packed = true]; - // Per-row L2 norm deltas, length = rows. - repeated double l2 = 5 [packed = true]; - // Heavy-hitter candidate keys from an upstream tracker, mirroring - // CountSketchDelta.hh_keys (same tag number 6, same wire shape). Downstream - // queries the merged Count-Min matrix for each key to (re)build its Top-K - // with globally-merged estimates. Empty/omitted when heavy-hitter tracking - // is not enabled. - repeated string hh_keys = 6; - // Packed cell encoding (canonical form): - repeated uint32 cell_rows = 9 [packed = true]; // row index of each changed cell - repeated uint32 cell_cols = 10 [packed = true]; // col index of each changed cell - repeated sint64 d_counts = 11 [packed = true]; // signed integer count delta -} - -// CountMinCell is the deprecated per-cell delta record. Producers emit the -// packed cell arrays on CountMinDelta instead; this message exists only to -// decode payloads from older producers. -message CountMinCell { - uint32 row = 1; - uint32 col = 2; - double d_count = 3; // deprecated: use CountMinDelta.d_counts - double d_sum = 4; // deprecated: omitted in the packed encoding - double d_sum2 = 5; // deprecated: omitted in the packed encoding -} diff --git a/proto/countsketch/countsketch.proto b/proto/countsketch/countsketch.proto deleted file mode 100644 index ae921100..00000000 --- a/proto/countsketch/countsketch.proto +++ /dev/null @@ -1,123 +0,0 @@ -// countsketch.proto — Portable wire format for Count Sketch state. -// Also contains TopKState and HeapEntry used by CountSketch and UnivMon. - -syntax = "proto3"; - -package sketchlib.v1; - -import "common/common.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/countsketch;countsketchpb"; - -// ============================================================================ -// Count Sketch -// ============================================================================ - -// CountSketchState is the portable state of a Count (±1) Sketch. -// -// The counter matrix is signed because Count Sketch uses ±1 increments. -// The matrix layout is identical to CountMinState (row-major, flat). -// -// L2 norms are derived values and must be recomputed on load: -// l2[r] = sqrt( sum_c counts[r][c]^2 ) -// Producers may include them as a convenience for consumers that only query. -message CountSketchState { - // Matrix dimensions. Both must be > 0. - uint32 rows = 1; - uint32 cols = 2; - - // Indicates which counts_* field below is populated. - CounterType counter_type = 3; - - // Signed integer counters in row-major order, length = rows * cols. - // Populated when counter_type ∈ {INT32, INT64}. - repeated sint64 counts_int = 4 [packed = true]; - - // Floating-point counters in row-major order, length = rows * cols. - // Populated when counter_type = FLOAT64 (Go producer). - repeated double counts_float = 5 [packed = true]; - - // Per-row L2 norms, length = rows. Derived — recompute on load if absent. - repeated double l2 = 6 [packed = true]; - - // Optional heavy-hitter (TopK min-heap) state. - // Absent when TopK tracking is disabled. - TopKState topk = 7; - - reserved 8 to 15; -} - -// ============================================================================ -// TopK / Heavy-Hitter State -// ============================================================================ - -// TopKState is the serialized state of a min-heap heavy-hitter tracker. -message TopKState { - // Maximum number of entries retained. - uint32 k = 1; - - // All retained entries. Order is not guaranteed (heap order is not preserved). - repeated HeapEntry entries = 2; -} - -// HeapEntry is one heavy-hitter record inside a TopKState. -message HeapEntry { - // String key identifying the heavy hitter (e.g. flow ID, IP address). - // For binary keys, producers should use base64 or hex encoding and document - // their choice; a future bytes_key field is reserved at number 3. - string key = 1; - - // Approximate frequency or weight accumulated for this key. - double count = 2; - - reserved 3; // Reserved for bytes_key alternative. -} - -// ============================================================================ -// Count Sketch Delta -// ============================================================================ - -// CountSketchDelta carries only the matrix cells that changed by at least the -// threshold between two consecutive snapshots, plus the full per-row L2 norm -// deltas and any heavy-hitter candidate keys. -// -// Cells apply additively on the receiver: matrix[row][col] += d_count. Count -// Sketch counters are signed (±1 increments), so d_count is a signed integer -// (sint64). Heavy-hitter candidate keys (hh_keys) are re-queried against the -// merged matrix downstream to rebuild the receiver's Top-K with accurate, -// globally-merged estimates. -// -// The packed cell encoding (cell_rows / cell_cols / d_counts, tags 9-11) plus -// l2 (tag 12) is the canonical form; the legacy per-cell message (tag 3), -// l2_legacy (tag 4) and topk (tag 5) are retained only so payloads from older -// producers still decode. This message is byte-identical to the Go reference -// implementation's CountSketchDelta so the two runtimes emit byte-identical -// delta frames for identical window state (cross-language byte parity). -message CountSketchDelta { - uint32 rows = 1; - uint32 cols = 2; - // Deprecated: use cell_rows/cell_cols/d_counts (tags 9-11). Retained for - // backward-compatible decode of payloads from older producers. - repeated CountSketchCell cells_legacy = 3; - // Deprecated: use l2 (tag 12) instead. - repeated double l2_legacy = 4 [packed = true]; - // Deprecated: use hh_keys (tag 6) instead. - TopKState topk = 5; - // Heavy-hitter candidate keys from the upstream tracker. Downstream queries - // the merged Count Sketch matrix for each key to build its Top-K. - repeated string hh_keys = 6; - // Packed cell encoding (canonical form): - repeated uint32 cell_rows = 9 [packed = true]; // row index of each changed cell - repeated uint32 cell_cols = 10 [packed = true]; // col index of each changed cell - repeated sint64 d_counts = 11 [packed = true]; // signed integer count delta - repeated double l2 = 12 [packed = true]; // per-row L2 norm deltas -} - -// CountSketchCell is the deprecated per-cell delta record. Producers emit the -// packed cell arrays on CountSketchDelta instead; this message exists only to -// decode payloads from older producers. -message CountSketchCell { - uint32 row = 1; - uint32 col = 2; - double d_count = 3; // deprecated: use CountSketchDelta.d_counts -} diff --git a/proto/ddsketch/ddsketch.proto b/proto/ddsketch/ddsketch.proto deleted file mode 100644 index 02b8beff..00000000 --- a/proto/ddsketch/ddsketch.proto +++ /dev/null @@ -1,75 +0,0 @@ -// ddsketch.proto — Portable wire format for DDSketch quantile sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/ddsketch;ddsketchpb"; - -// ============================================================================ -// DDSketch -// ============================================================================ - -// DDSketchState is the portable state of a DDSketch quantile sketch. -// -// Only alpha is required to reconstruct the index mapping. Consumers must -// recompute all derived mapping values on load: -// gamma = (1.0 + alpha) / (1.0 - alpha) -// log_gamma = ln(gamma) -// inv_log_gamma = 1.0 / log_gamma -// -// Bucket indices: a value v maps to bucket floor(ln(v) * inv_log_gamma). -// Stored bucket absolute_index = array_index + store_offset. -message DDSketchState { - // Relative accuracy guarantee. Required. Must satisfy 0 < alpha < 1. - double alpha = 1; - - // Bucket count array in index order (ascending absolute bucket index). - // Absolute index of store_counts[i] is i + store_offset. - repeated uint64 store_counts = 2 [packed = true]; - - // Absolute bucket index corresponding to store_counts[0]. - // May be negative (values < 1.0 map to negative bucket indices). - sint32 store_offset = 3; - - // Tags 4-7 previously carried the DataPoint-level METRIC scalars - // (count, sum, min, max). They were dropped from the wire format: - // the count is recoverable by summing store_counts and the remaining - // aggregates are carried elsewhere. - reserved 4, 5, 6, 7; - - reserved 8 to 15; - // Signed extension: negative magnitudes use the same mapping as positives. - repeated uint64 negative_store_counts = 16 [packed = true]; - sint32 negative_store_offset = 17; - uint64 zero_count = 18; -} - -// ============================================================================ -// DDSketch Delta -// ============================================================================ - -// DDSketchDelta carries only the buckets that changed above threshold T. -// -// DataPoint-level metric scalars (d_count/d_sum/new_min/new_max/min_changed/ -// max_changed) are intentionally NOT carried on the wire: the count delta is -// recoverable by summing bucket deltas, and min/max/quantiles are derived from -// the bucket distribution within the relative-accuracy guarantee. This message -// is byte-identical to the Go reference implementation's DDSketchDelta so the -// Go and Rust runtimes emit byte-identical delta frames for identical window -// state (cross-language byte parity). -message DDSketchDelta { - repeated DDSketchBucketDelta buckets = 1; - repeated DDSketchBucketDelta negative_buckets = 16; - uint64 zero_count = 17; - - // Fields 2-7 (d_count/d_sum/new_min/new_max/min_changed/max_changed) - // removed: DataPoint-level metric scalars are no longer carried on the wire. - reserved 2, 3, 4, 5, 6, 7; -} - -// DDSketchBucketDelta is the delta for one log-scale bucket. -message DDSketchBucketDelta { - sint32 index = 1; // signed bucket index (negative for values < 1) - uint64 d_count = 2; // Δcount ≥ threshold T -} diff --git a/proto/elasticsketch/elasticsketch.proto b/proto/elasticsketch/elasticsketch.proto deleted file mode 100644 index 94f1f465..00000000 --- a/proto/elasticsketch/elasticsketch.proto +++ /dev/null @@ -1,41 +0,0 @@ -// elasticsketch.proto — Portable wire format for Elastic Sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -import "countminsketch/countminsketch.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/elasticsketch;elasticsketchpb"; - -// ============================================================================ -// ElasticSketch -// ============================================================================ - -// ElasticState is the portable state of an Elastic Sketch. -// -// The Elastic Sketch has a heavy part (one bucket per index) and a light -// Count-Min layer. The heavy part is serialized as parallel arrays. -// The light layer is stored as a CountMinState with FLOAT64 counters. -message ElasticState { - // Number of heavy buckets. - uint32 bucket_count = 1; - - // Per-bucket flow identifiers (strings), length = bucket_count. - // Empty string means the bucket is unoccupied. - repeated string flow_ids = 2; - - // Positive vote counts per bucket, length = bucket_count. - repeated sint32 vote_pos = 3 [packed = true]; - - // Negative vote counts per bucket, length = bucket_count. - repeated sint32 vote_neg = 4 [packed = true]; - - // Whether each bucket is in eviction state, length = bucket_count. - repeated bool evictions = 5 [packed = true]; - - // Light Count-Min layer (FLOAT64 counters). - CountMinState light = 6; - - reserved 7 to 15; -} diff --git a/proto/hll/hll.proto b/proto/hll/hll.proto deleted file mode 100644 index 1cdf6124..00000000 --- a/proto/hll/hll.proto +++ /dev/null @@ -1,134 +0,0 @@ -// hll.proto — Portable wire format for HyperLogLog sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/hll;hllpb"; - -// ============================================================================ -// HyperLogLog -// ============================================================================ - -// HyperLogLogState is the portable state of a HyperLogLog cardinality sketch. -// -// Precision p determines the register count: num_registers = 2^precision. -// Both libraries default to precision = 14 (16 384 registers). -// -// The variant field is required because the three estimator algorithms are -// not interchangeable: loading an ErtlMLE state into a Regular estimator -// (or vice versa) produces incorrect cardinality estimates. -message HyperLogLogState { - // HLL estimator variant. Required. - HLLVariant variant = 1; - - // Log₂ of the number of registers (precision parameter p). - // Default = 14 → 16 384 registers. - uint32 precision = 2; - - // DENSE register encoding: raw register values, length = 2^precision. - // Each byte stores the maximum (leading-zeros + 1) seen for that bucket. - // Stored as a raw byte string for compact encoding (1 byte per register). - // - // Exactly one of `registers` (dense) or `registers_sparse` (sparse) is - // populated by a producer. A decoder MUST read whichever is present; if - // both are empty the sketch is all-zero. This field stays at tag 3 so old - // dense-only readers keep working unchanged. - bytes registers = 3; - - // ---- HIP-variant extra fields (populated only when variant = HIP) -------- - - // HIP accumulator component kxq0. - double hip_kxq0 = 4; - - // HIP accumulator component kxq1. - double hip_kxq1 = 5; - - // HIP running cardinality estimate. - double hip_est = 6; - - // SPARSE register encoding (additive, tag 7). Populated instead of - // `registers` (tag 3) when the number of non-zero registers is below the - // producer's dense/sparse crossover. Old readers that predate this field - // ignore tag 7 (proto skips unknown fields), so adding it is wire-backward- - // compatible. New readers decode BOTH representations — see - // HLLSparseRegisters. Reconstruction yields the identical 2^precision-byte - // dense register array, so cardinality estimation is unaffected. - HLLSparseRegisters registers_sparse = 7; - - reserved 8 to 15; -} - -// HLLSparseRegisters is the sparse (HLL++ style) full-state encoding of the -// register array. Only non-zero registers are stored, sorted by ascending -// index and varint-packed as (index_delta, value) pairs: -// -// for each non-zero register, in ascending index order: -// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -// uvarint(value) // 1..=Q+1, always 1 byte in practice -// -// For low/medium cardinality (most registers zero) this is far smaller than -// the dense 2^precision-byte array. Decoding allocates a zero-filled dense -// array of length `num_registers` and writes each decoded (index, value). -message HLLSparseRegisters { - // Total number of registers in the reconstructed dense array (= 2^precision). - // Carried explicitly so the decoder can size the dense buffer without - // re-deriving it from precision. - uint32 num_registers = 1; - - // Varint-packed (index_delta, value) pairs for non-zero registers, in - // ascending index order. See message-level comment for the exact layout. - bytes packed = 2; -} - -// HLLVariant identifies which HLL estimator algorithm the registers belong to. -enum HLLVariant { - HLL_VARIANT_UNSPECIFIED = 0; - - // Classic HLL with bias-correction tables (Flajolet et al. 2007). - HLL_VARIANT_REGULAR = 1; - - // Otmar Ertl's MLE estimator (arXiv:1702.01284). - // Originally derived from Apache DataFusion's HLL implementation. - // Not compatible with REGULAR registers. - HLL_VARIANT_ERTL_MLE = 2; - - // Kevin Lang's HyperLogLog++ Improved Proposal (HIP). - // Requires hip_kxq0, hip_kxq1, hip_est to be populated. - HLL_VARIANT_HIP = 3; -} - -// ============================================================================ -// HyperLogLog Delta -// ============================================================================ - -// HLLDelta carries the registers that increased between two consecutive -// snapshots. HLL uses max semantics, so only increases are meaningful; -// at a fixed precision a register can never decrease. Each update is applied -// losslessly on the receiver via register[index] = max(register[index], value) -// — no register update is ever dropped (there is no threshold to apply). -// -// The increased registers are varint-packed exactly like -// HLLSparseRegisters.packed: sorted ascending by index, each register emitted -// as (index_delta, value) where index_delta = index - prev_index (prev_index -// starts at 0) and value is the new (larger) register value: -// -// for each increased register, in ascending index order: -// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -// uvarint(value) // 1..=Q+1, always 1 byte in practice -// -// A single-emit delta against the all-zero snapshot is therefore the same size -// as the full sparse frame, and a sub-window delta is strictly smaller (it -// packs only the registers that actually grew). This replaces the previous -// per-register sub-message encoding, eliminating ~6–8 bytes of tag/length -// overhead per register. -// -// This message is byte-identical to the Go reference implementation's HLLDelta -// so the two runtimes emit byte-identical delta frames for identical window -// state (cross-language byte parity). -message HLLDelta { - // Varint-packed (index_delta, value) pairs for the registers that increased, - // in ascending index order. Same layout as HLLSparseRegisters.packed. Empty - // when no register changed. - bytes packed_updates = 1; -} diff --git a/proto/hydra/hydra.proto b/proto/hydra/hydra.proto deleted file mode 100644 index 116bb439..00000000 --- a/proto/hydra/hydra.proto +++ /dev/null @@ -1,80 +0,0 @@ -// hydra.proto — Portable wire format for Hydra multi-sketch framework state. - -syntax = "proto3"; - -package sketchlib.v1; - -import "countminsketch/countminsketch.proto"; -import "countsketch/countsketch.proto"; -import "hll/hll.proto"; -import "kll/kll.proto"; -import "univmon/univmon.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/hydra;hydrapb"; - -// ============================================================================ -// Hydra Sketch -// ============================================================================ - -// HydraState is the portable state of a Hydra multi-sketch framework. -// -// Hydra maintains a grid of row_num child sketches (cells), each of the -// same type (counter_type). Items are routed to a specific row cell using a -// position hash derived from the item's subkey. An optional global "big -// counter" sketch aggregates the full stream. -// -// Current implementations set col_num = 1 (one sketch per row). The field is -// included for schema completeness and future extensions. -message HydraState { - // Number of sketch rows (D / number of subpopulations). - uint32 row_num = 1; - - // Number of sketch columns per row. Currently always 1. - uint32 col_num = 2; - - // The sketch type used for every cell. - HydraCounterType counter_type = 3; - - // Serialized cells in row-major order, length = row_num * col_num. - // Each cell's oneof field must match counter_type. - repeated HydraCell cells = 4; - - // Optional global sketch covering the entire stream. - // Absent when global counter is disabled (enable_global_counter = false). - HydraCell big_counter = 5; - - // Seed index used for subpopulation routing (Go: seedHydra, Rust: HYDRA_SEED). - // Both libraries default to 6. - uint32 seed_index = 6; - - // True when per-cell TopK tracking is enabled. - bool enable_topk = 7; - - // True when fanout subkey expansion is active. - bool fanout_subkeys = 8; - - reserved 9 to 15; -} - -// HydraCounterType identifies the sketch algorithm used inside Hydra cells. -enum HydraCounterType { - HYDRA_COUNTER_TYPE_UNSPECIFIED = 0; - HYDRA_COUNTER_TYPE_COUNT_MIN = 1; - HYDRA_COUNTER_TYPE_COUNT_SKETCH = 2; - HYDRA_COUNTER_TYPE_HLL = 3; - HYDRA_COUNTER_TYPE_KLL = 4; - HYDRA_COUNTER_TYPE_UNIVMON = 5; -} - -// HydraCell holds the serialized state of one Hydra grid cell. -// Exactly one field in the oneof must be set, and it must match the parent -// HydraState.counter_type. -message HydraCell { - oneof sketch { - CountMinState count_min = 1; - CountSketchState count_sketch = 2; - HyperLogLogState hll = 3; - KLLState kll = 4; - UnivMonState univmon = 5; - } -} diff --git a/proto/kll/kll.proto b/proto/kll/kll.proto deleted file mode 100644 index bd307a14..00000000 --- a/proto/kll/kll.proto +++ /dev/null @@ -1,105 +0,0 @@ -// kll.proto — Portable wire format for KLL quantile sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/kll;kllpb"; - -// ============================================================================ -// KLL Sketch -// ============================================================================ - -// KLLState is the portable state of a KLL quantile sketch. -// -// The items array is partitioned into compaction levels by the levels[] array: -// items[ levels[i] .. levels[i+1] ) are the retained items at level i. -// -// Derived state that must be recomputed on load (not serialized): -// capacity_cache — computed from k, m, and level boundaries. -// level0_capacity — computed from k and num_levels. -// top_height — equals num_levels - 1. -message KLLState { - // Target compactor capacity k. Must be >= 8. - uint32 k = 1; - - // Minimum compactor size m. Must satisfy 2 <= m <= k. - uint32 m = 2; - - // Number of levels currently in use. - uint32 num_levels = 3; - - // Level boundary indices into items[], length = num_levels + 1. - // levels[0] is always 0; levels[num_levels] = len(items). - repeated uint32 levels = 4 [packed = true]; - - // All retained samples in level order, length = levels[num_levels]. - // - // RAW-F64 representation. Populated when the value-offset fixed-point - // representation (offset/value_scale/residuals, fields 7–9) is NOT used — - // i.e. when the producer could not represent the retained set exactly at - // any candidate decimal scale and fell back to raw f64. When residuals - // (field 9) is non-empty, items MUST be empty and decoders MUST reconstruct - // from the fixed-point fields instead. - repeated double items = 5 [packed = true]; - - // Random bit generator state for deterministic compaction continuation. - // Required for sketches that will receive further updates after loading. - // Consumers that only query may ignore this field. - CoinState coin = 6; - - // ---- Value-offset fixed-point representation (additive, optional) ------- - // - // KLL retained samples are typically integer-ish or low-cardinality decimal - // values whose raw f64 encoding (8 bytes/item) dominates the wire size. - // When every retained value is exactly representable as - // - // value = residual * 10^value_scale + offset - // - // the producer stores the per-sample residual integers (field 9, zigzag - // varint) plus a single per-sketch `offset` (field 7) and decimal exponent - // `value_scale` (field 8) instead of raw f64 items. This is exact for the - // stored sample SET (the producer applies an exactness guard and falls back - // to raw `items` when any value is not exactly representable), and quantiles - // are shift-equivariant (q(X - c) = q(X) - c) so query results are - // unchanged. residuals[] is in the SAME level order as items[]. - // - // Backward compatibility: when residuals (field 9) is absent/empty, decoders - // read raw f64 from items (field 5) — the original v1 behavior. Decoders on - // BOTH languages read both forms; producers gate emission of the fixed-point - // form behind the exactness guard. - // - // NOTE: this is the self-contained per-SKETCH offset carried in the sketch - // header. The epoch-bound offset for the delta-transmission path is a - // separate, later concern and is intentionally NOT modeled here. - - // Per-sketch subtraction constant. value = residual * 10^value_scale + offset. - double offset = 7; - - // Decimal scale exponent. A value v maps to residual = round((v - offset) * - // 10^(-value_scale)); reconstruction multiplies by 10^value_scale. scale 0 - // means residuals are integers; negative scales carry fractional precision - // (e.g. value_scale = -3 -> millis). sint32 so negative scales varint-pack. - sint32 value_scale = 8; - - // Per-sample residual integers, same level order and length as a raw items[] - // would have. Zigzag varint (sint64) keeps small magnitudes compact. When - // present and non-empty, this is the authoritative item source and items[] - // (field 5) MUST be empty. - repeated sint64 residuals = 9 [packed = true]; - - reserved 10 to 15; -} - -// CoinState is the RNG used by KLL's probabilistic compaction. -// Both Go and Rust use an identical xorshift-based generator. -message CoinState { - // Xorshift generator internal state word. - uint64 state = 1; - - // Up to 64 random bits buffered for single-bit coin flips. - uint64 bit_cache = 2; - - // Number of valid bits remaining in bit_cache (0–64). - uint32 remaining_bits = 3; -} diff --git a/proto/sketchlib.proto b/proto/sketchlib.proto deleted file mode 100644 index 0a6d8193..00000000 --- a/proto/sketchlib.proto +++ /dev/null @@ -1,100 +0,0 @@ -// sketchlib.proto — Top-level envelope for sketchlib portable wire format. -// -// This file defines only the SketchEnvelope message. All sketch-specific -// message types live in their per-sketch proto files: -// -// common/common.proto — HashSpec, ProducerInfo, CounterType, ... -// countminsketch/countminsketch.proto — CountMinState -// countsketch/countsketch.proto — CountSketchState, TopKState, HeapEntry -// hll/hll.proto — HyperLogLogState, HLLVariant -// kll/kll.proto — KLLState, CoinState -// ddsketch/ddsketch.proto — DDSketchState -// univmon/univmon.proto — UnivMonState, UnivMonLayer -// hydra/hydra.proto — HydraState, HydraCounterType, HydraCell -// -// Design goals: -// - Cross-language: Go (sketchlib-go) ↔ Rust (asap_sketchlib) round-trips. -// - Forward-compatible: new fields use reserved numbers; old readers skip them. -// - Logical state only: no pointers, no runtime caches, no gob/msgpack artefacts. -// - Derived values (gamma, L2 norms, capacity caches) are always recomputed on load. -// - Flat row-major arrays for all 2-D matrices. -// - Hash specification is explicit so consumers can detect configuration mismatches. -// -// Version history: -// 1 — initial portable format (CountMin, Count, HLL, KLL, DDSketch) -// 2 — extended to UnivMon and Hydra (reserved, not yet stable) - -syntax = "proto3"; - -package sketchlib.v1; - -import "common/common.proto"; -import "countminsketch/countminsketch.proto"; -import "countsketch/countsketch.proto"; -import "hll/hll.proto"; -import "kll/kll.proto"; -import "ddsketch/ddsketch.proto"; -import "univmon/univmon.proto"; -import "hydra/hydra.proto"; -import "cocosketch/cocosketch.proto"; -import "elasticsketch/elasticsketch.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/sketch_envelope;sketch_envelopepb"; - -// ============================================================================ -// Top-Level Envelope -// ============================================================================ - -// SketchEnvelope is the portable container for all sketch transfers. -// Producers fill exactly one field inside sketch_state. -// Consumers check format_version first, then dispatch on the oneof. -message SketchEnvelope { - // Schema version. Consumers should reject envelopes whose version exceeds - // the highest version they understand. - // 1 = CountMin, Count, HLL, KLL, DDSketch - // 2 = UnivMon, Hydra (reserved) - // 3 = CocoSketch, ElasticSketch (reserved) - uint32 format_version = 1; - - // Library and build info of the producing side. - ProducerInfo producer = 2; - - // Complete hashing configuration. Consumers must validate this against their - // own configuration before performing updates or queries on the loaded sketch. - // Mismatches do not prevent loading but will cause diverging results. - HashSpec hash_spec = 3; - - // The sketch payload. Exactly one field must be set. - oneof sketch_state { - CountMinState count_min = 10; - CountSketchState count_sketch = 11; - HyperLogLogState hll = 12; - KLLState kll = 13; - DDSketchState ddsketch = 14; - UnivMonState univmon = 15; - HydraState hydra = 16; - CocoSketchState coco = 17; - ElasticState elastic = 18; - } - - reserved 19 to 31; // Reserved for future sketch types. - - // Geometric / NitroSketch sampling probability for the producing sketch. - // - // The producer admits each stream update with probability `sample_p` (using - // geometric skip-sampling, statistically identical to per-update Bernoulli(p)) - // and stores the RAW SAMPLED sketch state — never the rescaled state. The - // consumer applies the `× 1/sample_p` rescale at QUERY time on the count-like - // estimators (HLL cardinality, CountMin / CountSketch frequency, SUM / COUNT); - // quantile estimators (KLL, DDSketch) are scale-invariant under uniform - // sampling so they carry `sample_p` but need no rescale. - // - // Lives on the envelope (not per-sketch state) so downstream consumers that - // construct per-sketch state structs by literal are unaffected. - // - // Dual-read default: a value of 0.0 (the proto3 default for an unset field) - // is interpreted as 1.0 (exact, no sampling) for backward compatibility with - // producers that predate this field. With sample_p = 1.0 a 0.0/unset envelope - // is byte-identical to the pre-sampling format. - double sample_p = 4; -} diff --git a/proto/univmon/univmon.proto b/proto/univmon/univmon.proto deleted file mode 100644 index f22828f7..00000000 --- a/proto/univmon/univmon.proto +++ /dev/null @@ -1,50 +0,0 @@ -// univmon.proto — Portable wire format for Universal Monitoring (UnivMon) sketch state. - -syntax = "proto3"; - -package sketchlib.v1; - -import "countsketch/countsketch.proto"; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/univmon;univmonpb"; - -// ============================================================================ -// UnivMon -// ============================================================================ - -// UnivMonState is the portable state of a Universal Monitoring (UnivMon) sketch. -// -// UnivMon is a hierarchy of L Count Sketches paired with L heavy-hitter heaps. -// An item's hash trailing-zero count determines which layers receive it, giving -// each layer a geometrically decreasing sample of the stream. -// -// All layers share the same sketch dimensions and heap capacity. -message UnivMonState { - // Number of layers L in the hierarchy (inclusive of all levels). - uint32 layer_size = 1; - - // Count Sketch dimensions shared by all layers. - uint32 sketch_rows = 2; - uint32 sketch_cols = 3; - - // Heavy-hitter heap capacity shared by all layers. - uint32 heap_size = 4; - - // Total items inserted across all layers (the "bucket size" counter). - uint64 bucket_size = 5; - - // Per-layer state, in order from layer 0 (finest) to layer_size-1 (coarsest). - // Length must equal layer_size. - repeated UnivMonLayer layers = 6; - - reserved 7 to 15; -} - -// UnivMonLayer is the joint state of one UnivMon layer. -message UnivMonLayer { - // The Count Sketch accumulating items at this layer. - CountSketchState sketch = 1; - - // The heavy-hitter heap for this layer. - TopKState heap = 2; -} diff --git a/src/lib.rs b/src/lib.rs index e14c1f95..d2952725 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ //! KLL, and DDSketch. //! - [`common`]: shared input types, hashing abstractions, storage backends, and //! reusable utilities used across sketches. -//! - [`proto`]: portable protobuf message types for sketch interchange. //! - [`sketch_framework`]: higher-level composition layers such as Hydra, //! UnivMon, tumbling windows, and batch/parallel execution helpers. //! - [`message_pack_format`]: MessagePack/proto wire format shared with @@ -24,8 +23,6 @@ /// storage backends, and reusable utilities. pub mod common; pub mod message_pack_format; -/// Portable protobuf message types for sketch interchange. -pub mod proto; /// Higher-level composition layers such as Hydra, UnivMon, tumbling windows, /// and batch/parallel execution helpers. pub mod sketch_framework; diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index c61fb54f..ab686338 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -1,9 +1,7 @@ //! Wire-format-aligned KLL sketch type for cross-language interop. //! -//! `KllSketch` is a thin facade over the pure-Rust [`crate::sketches::kll::Kll`] -//! (parity already locked in by the existing -//! `test_update_then_envelope_matches_sketchlib_go_bytes` in this file -//! against `sketchlib-go::KLLSketch.SerializePortable`). The wire shape +//! `KllSketch` is a thin facade over the pure-Rust [`crate::sketches::kll::Kll`]. +//! The wire shape //! also serves as a nested field for //! [`crate::message_pack_format::portable::hydra_kll`]. @@ -196,162 +194,6 @@ impl std::fmt::Debug for KllSketch { unsafe impl Send for KllSketch {} unsafe impl Sync for KllSketch {} -// ----- Value-offset fixed-point representation (cross-language) ----- -// -// KLL `KLLState` (proto/kll/kll.proto) carries the retained samples in one of -// two forms: -// -// * RAW F64 — `items` (field 5), 8 bytes/sample. The original v1 form. -// * VALUE-OFFSET — `offset` (7) + `value_scale` (8) + `residuals` (9), -// where `value = residual * 10^value_scale + offset`. -// -// PR2 makes BOTH languages read both forms (this is the decode-first half of -// the rollout). `sketchlib-go` gates emission of the value-offset form behind -// an exactness guard; this crate currently only reads it but the -// encode/decode helpers below are symmetric so the Rust side can produce it -// too (used by the cross-language golden test and any future Rust producer). - -/// Per-sketch scale exponents tried by [`encode_value_offset`], in priority -/// order. Mirrors `kllScaleSweep` in `sketchlib-go`. -pub const KLL_SCALE_SWEEP: [i32; 7] = [0, -1, -2, -3, -4, -5, -6]; - -/// Reconstructs retained samples from the value-offset fixed-point fields: -/// `value = residual * 10^value_scale + offset`. Caller invokes this only when -/// `residuals` is non-empty. -pub fn decode_value_offset(offset: f64, value_scale: i32, residuals: &[i64]) -> Vec { - let mul = 10f64.powi(value_scale); - residuals.iter().map(|&r| r as f64 * mul + offset).collect() -} - -/// Attempts to represent `items` exactly as `residual * 10^scale + offset` -/// using the finite [`KLL_SCALE_SWEEP`]. `offset` is the minimum value. Returns -/// `None` (the exactness-guard fallback) when no candidate scale round-trips -/// every sample bit-exactly, or when the slice is empty / contains a non-finite -/// value. Symmetric with `sketchlib-go::encodeValueOffset`. -pub fn encode_value_offset(items: &[f64]) -> Option<(f64, i32, Vec)> { - if items.is_empty() { - return None; - } - let mut offset = items[0]; - for &v in items { - if !v.is_finite() { - return None; - } - if v < offset { - offset = v; - } - } - for &sc in &KLL_SCALE_SWEEP { - let mul = 10f64.powi(-sc); - let mut out = Vec::with_capacity(items.len()); - let mut good = true; - for &v in items { - let r = ((v - offset) * mul).round(); - if !r.is_finite() || r > i64::MAX as f64 || r < i64::MIN as f64 { - good = false; - break; - } - let ri = r as i64; - // Exactness guard: reconstruct and require bit-identical recovery. - let recon = ri as f64 * 10f64.powi(sc) + offset; - if recon != v { - good = false; - break; - } - out.push(ri); - } - if good { - return Some((offset, sc, out)); - } - } - None -} - -/// Materialized retained samples + level layout decoded from a proto -/// `KLLState`, dual-reading the raw-f64 and value-offset forms. -#[derive(Debug, Clone)] -pub struct KllProtoItems { - /// Retained samples in level order (length == `levels[num_levels]`). - pub items: Vec, - /// Level boundary indices, length == `num_levels + 1`. - pub levels: Vec, - pub num_levels: usize, - pub k: u32, - pub m: u32, -} - -impl KllProtoItems { - /// Decodes a proto `KLLState`, dual-reading: when `residuals` is non-empty - /// it reconstructs from (offset, value_scale, residuals); otherwise it - /// reads raw f64 `items`. Returns an error if both forms are populated or - /// the level layout is inconsistent. - pub fn from_state( - s: &crate::proto::sketchlib::KllState, - ) -> Result> { - let items = if !s.residuals.is_empty() { - if !s.items.is_empty() { - return Err("kll: both residuals and raw items[] set".into()); - } - decode_value_offset(s.offset, s.value_scale, &s.residuals) - } else { - s.items.clone() - }; - let levels: Vec = s.levels.iter().map(|&v| v as usize).collect(); - if levels.len() < 2 { - return Err(format!("kll: invalid levels length {}", levels.len()).into()); - } - if *levels.last().unwrap() != items.len() { - return Err("kll: invalid item layout".into()); - } - Ok(Self { - items, - levels, - num_levels: s.num_levels as usize, - k: s.k, - m: s.m, - }) - } - - /// Weighted (value, weight) pairs across all levels. - pub fn weighted_samples(&self) -> Vec<(f64, u64)> { - let mut out = Vec::with_capacity(self.items.len()); - for h in 0..self.num_levels { - let weight: u64 = 1 << h; - let idx = self.num_levels - 1 - h; - if idx + 1 >= self.levels.len() { - continue; - } - let start = self.levels[idx]; - let end = self.levels[idx + 1]; - for &v in &self.items[start..end] { - out.push((v, weight)); - } - } - out - } - - /// Estimated value at quantile `q ∈ [0, 1]` from the weighted samples. - pub fn quantile(&self, q: f64) -> f64 { - let mut samples = self.weighted_samples(); - if samples.is_empty() { - return 0.0; - } - samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - let total: u64 = samples.iter().map(|(_, w)| w).sum(); - let target = (q * total as f64).ceil() as u64; - let mut acc = 0u64; - for (v, w) in &samples { - acc += w; - if acc >= target { - return *v; - } - } - samples.last().unwrap().0 - } -} - -// ----- Wire format ----- - /// Wire DTO for [`KllSketch`]. Also referenced as a nested field by /// [`crate::message_pack_format::portable::hydra_kll::HydraKllSketchWire`]. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -495,297 +337,4 @@ mod tests { fn test_aggregate_kll_empty() { assert!(KllSketch::aggregate_kll(200, &[]).is_none()); } - - /// Cross-language byte-parity guard against `sketchlib-go`'s - /// `KLLSketch.SerializePortable` output for the deterministic - /// input `(1..=50)` with `seed=42` and `k=200`. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - CoinState, KllState, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let mut sk: Kll = Kll::with_k_and_seed(200, 42); - for i in 1..=50 { - sk.update(&(i as f64)); - } - - let (state, bit_cache, remaining_bits) = sk.coin_state(); - let kll_state = KllState { - k: sk.k() as u32, - m: sk.m(), - num_levels: sk.num_levels(), - levels: sk.level_bounds_top_down(), - // This golden pins the RAW-F64 wire form; the value-offset fields - // stay at their proto defaults so the bytes are unchanged. - items: sk.items_top_down(), - coin: Some(CoinState { - state, - bit_cache, - remaining_bits, - }), - offset: 0.0, - value_scale: 0, - residuals: vec![], - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Kll(kll_state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - const GOLDEN_HEX: &str = "08016aa20308c80110081801220200322a9003000000000000f03f000000000000004000000000000008400000000000001040000000000000144000000000000018400000000000001c40000000000000204000000000000022400000000000002440000000000000264000000000000028400000000000002a400000000000002c400000000000002e4000000000000030400000000000003140000000000000324000000000000033400000000000003440000000000000354000000000000036400000000000003740000000000000384000000000000039400000000000003a400000000000003b400000000000003c400000000000003d400000000000003e400000000000003f4000000000000040400000000000804040000000000000414000000000008041400000000000004240000000000080424000000000000043400000000000804340000000000000444000000000008044400000000000004540000000000080454000000000000046400000000000804640000000000000474000000000008047400000000000004840000000000080484000000000000049403202082a"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "KLL envelope bytes diverge from sketchlib-go golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - fn decode_hex(s: &str) -> Vec { - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - // ----- Value-offset fixed-point representation ----- - - use crate::proto::sketchlib::{KllState, SketchEnvelope, sketch_envelope::SketchState}; - use prost::Message; - - /// Round-trip exactness for the value-offset (fixed-point) form: integer - /// and fixed-decimal series must encode then decode back bit-exactly. - #[test] - fn value_offset_round_trip_exact_fixed_point() { - // Integer-ish series (scale 0). - let ints: Vec = (0..200).map(|i| (1_000_000 + i * 7) as f64).collect(); - let (offset, scale, residuals) = - encode_value_offset(&ints).expect("integer series must be representable"); - assert_eq!(scale, 0, "integer series should pick scale 0"); - let back = decode_value_offset(offset, scale, &residuals); - assert_eq!(ints, back, "fixed-point round-trip changed integer items"); - - // Fixed-decimal series (milli resolution → scale -3). - let decis: Vec = (0..128).map(|i| 100.0 + (i as f64) * 0.001).collect(); - let (o2, s2, r2) = - encode_value_offset(&decis).expect("milli-decimal series must be representable"); - assert!( - (-3..=0).contains(&s2), - "expected a small negative scale, got {s2}" - ); - let back2 = decode_value_offset(o2, s2, &r2); - assert_eq!(decis, back2, "fixed-point round-trip changed decimal items"); - } - - /// The exactness guard must reject a value not representable at any - /// candidate scale, so the caller falls back to raw f64. - #[test] - fn value_offset_guard_rejects_irrational() { - let items = vec![0.0, 1.0, std::f64::consts::PI, 3.0]; - assert!( - encode_value_offset(&items).is_none(), - "pi is not exactly representable at any decimal scale; guard must reject" - ); - // Empty slice falls back too. - assert!(encode_value_offset(&[]).is_none()); - // Non-finite values are rejected. - assert!(encode_value_offset(&[1.0, f64::NAN]).is_none()); - assert!(encode_value_offset(&[1.0, f64::INFINITY]).is_none()); - } - - /// `KllProtoItems::from_state` must dual-read both the raw-f64 and the - /// value-offset forms and yield identical items + quantiles. - #[test] - fn proto_dual_read_round_trip_both_forms() { - let mut sk: Kll = Kll::with_k_and_seed(200, 7); - for i in 1..=5000 { - sk.update(&(i as f64)); - } - let raw_items = sk.items_top_down(); - let levels = sk.level_bounds_top_down(); - let num_levels = sk.num_levels(); - - // Raw-f64 form. - let raw_state = KllState { - k: sk.k() as u32, - m: sk.m(), - num_levels, - levels: levels.clone(), - items: raw_items.clone(), - coin: None, - offset: 0.0, - value_scale: 0, - residuals: vec![], - }; - let from_raw = KllProtoItems::from_state(&raw_state).expect("decode raw-f64"); - - // Value-offset form (items empty, residuals populated). - let (offset, scale, residuals) = - encode_value_offset(&raw_items).expect("integer items representable"); - let fp_state = KllState { - k: sk.k() as u32, - m: sk.m(), - num_levels, - levels, - items: vec![], - coin: None, - offset, - value_scale: scale, - residuals, - }; - let from_fp = KllProtoItems::from_state(&fp_state).expect("decode value-offset"); - - assert_eq!( - from_raw.items, from_fp.items, - "raw-f64 and value-offset decode to different items" - ); - for &q in &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0] { - assert_eq!( - from_raw.quantile(q), - from_fp.quantile(q), - "quantile mismatch at p={q} between the two decode paths" - ); - } - } - - /// Decoding must error if a malformed state sets BOTH raw items and - /// residuals. - #[test] - fn proto_rejects_both_forms_populated() { - let state = KllState { - k: 200, - m: 8, - num_levels: 1, - levels: vec![0, 2], - items: vec![1.0, 2.0], - coin: None, - offset: 0.0, - value_scale: 0, - residuals: vec![1, 2], - }; - assert!(KllProtoItems::from_state(&state).is_err()); - } - - /// Cross-language golden: decode the exact envelope bytes that - /// `sketchlib-go` emits for the deterministic `(1..=50)` integer input via - /// its value-offset encoder, and confirm the retained samples + quantiles - /// match. The Go side produces this with `SerializePortable` (fixed-point, - /// scale 0, offset = 1.0). See `sketches/KLL/portable_test.go:: - /// TestGoldenValueOffsetEnvelopeForRust` in sketchlib-go, which prints this - /// hex and asserts the identical bytes. - #[test] - fn golden_value_offset_envelope_from_go() { - // SketchEnvelope { format_version: 1, kll: KLLState { k: 200, m: 8, - // num_levels: 1, levels: [0, 50], offset: 1.0, value_scale: 0, - // residuals: [0..=49] } }, producer/hash_spec cleared. - const GOLDEN_HEX: &str = GO_VALUE_OFFSET_GOLDEN_HEX; - let bytes = decode_hex(GOLDEN_HEX); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode envelope"); - let state = match env.sketch_state { - Some(SketchState::Kll(s)) => s, - _ => panic!("envelope did not contain a KLLState"), - }; - assert!( - state.items.is_empty(), - "value-offset form must leave items[] empty" - ); - assert_eq!(state.value_scale, 0); - assert_eq!(state.offset, 1.0); - assert_eq!(state.residuals.len(), 50); - - let decoded = KllProtoItems::from_state(&state).expect("dual-read decode"); - let expected: Vec = (1..=50).map(|i| i as f64).collect(); - assert_eq!(decoded.items, expected, "value-offset decode != (1..=50)"); - // Quantile sanity: with k=200 and 50 inputs, no compaction occurred so - // these are exact order statistics. - assert_eq!(decoded.quantile(0.0), 1.0); - assert_eq!(decoded.quantile(1.0), 50.0); - } - - /// Reverse-direction cross-language golden: build the value-offset envelope - /// for (1..=50) with the Rust encoder and assert exact bytes. The hex - /// printed here is consumed by sketchlib-go `TestRustGoldenDecodesInGo`. - /// Run with `-- --nocapture` to (re)print when regenerating. - #[test] - fn golden_value_offset_envelope_for_go() { - let items: Vec = (1..=50).map(|i| i as f64).collect(); - let (offset, value_scale, residuals) = - encode_value_offset(&items).expect("integers representable"); - assert_eq!(offset, 1.0); - assert_eq!(value_scale, 0); - - let state = KllState { - k: 200, - m: 8, - num_levels: 1, - levels: vec![0, 50], - items: vec![], - // No coin: the reverse-decode in Go only checks the retained set, - // and an absent coin keeps the fixture minimal/stable. - coin: None, - offset, - value_scale, - residuals, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Kll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - let got_hex: String = got.iter().map(|b| format!("{b:02x}")).collect(); - println!( - "RUST_VALUE_OFFSET_GOLDEN_HEX ({} bytes):\n{got_hex}", - got.len() - ); - - const RUST_GOLDEN_HEX: &str = "08016a4808c801100818012202003239000000000000f03f4a3200020406080a0c0e10121416181a1c1e20222426282a2c2e30323436383a3c3e40424446484a4c4e50525456585a5c5e6062"; - assert_eq!( - got_hex, RUST_GOLDEN_HEX, - "Rust value-offset envelope bytes drifted; update both this constant \ - and sketchlib-go's RustValueOffsetGoldenHex" - ); - - // Self-consistency: our own dual-reader recovers (1..=50). - let decoded = KllProtoItems::from_state(match &envelope.sketch_state { - Some(SketchState::Kll(s)) => s, - _ => unreachable!(), - }) - .expect("decode"); - assert_eq!(decoded.items, items); - } - - // Captured from sketchlib-go's value-offset `SerializePortable` for input - // (1..=50), k=200, producer/hash_spec cleared. Generated and asserted by - // sketchlib-go `TestGoldenValueOffsetEnvelopeForRust` (run with `-v` to - // re-print). 80 bytes: SketchEnvelope{ format_version:1, kll: KLLState{ - // k:200, m:8, num_levels:1, levels:[0,50], coin:{state:42}, offset:1.0, - // value_scale:0, residuals:[0..=49] } }. - const GO_VALUE_OFFSET_GOLDEN_HEX: &str = "08016a4c08c80110081801220200323202082a39000000000000f03f4a3200020406080a0c0e10121416181a1c1e20222426282a2c2e30323436383a3c3e40424446484a4c4e50525456585a5c5e6062"; } diff --git a/src/proto.rs b/src/proto.rs deleted file mode 100644 index 933fe26d..00000000 --- a/src/proto.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Proto-generated types for cross-language sketch serialization. -// -// The contents of `sketchlib` are vendored: the file under -// `src/proto/generated/sketchlib.v1.rs` is produced by `tools/gen-proto` -// and committed to the repository. End-users of this crate therefore do -// not need `protoc` or any build-script machinery to build it. -// -// To regenerate after editing any `proto/**/*.proto`: -// -// cargo run --manifest-path tools/gen-proto/Cargo.toml -// -// CI enforces that the committed file matches the result of running the -// tool against the current `.proto` sources. -#[allow(rustdoc::broken_intra_doc_links)] -#[allow(missing_docs)] -#[allow(clippy::all)] -#[rustfmt::skip] -pub mod sketchlib { - include!("proto/generated/sketchlib.v1.rs"); -} diff --git a/src/proto/generated/sketchlib.v1.rs b/src/proto/generated/sketchlib.v1.rs deleted file mode 100644 index 2c098d3f..00000000 --- a/src/proto/generated/sketchlib.v1.rs +++ /dev/null @@ -1,923 +0,0 @@ -// @generated by tools/gen-proto -- DO NOT EDIT. -// -// Regenerate with: -// cargo run --manifest-path tools/gen-proto/Cargo.toml -// -// Source of truth: proto/**/*.proto - -// This file is @generated by prost-build. -/// ProducerInfo records which library version created this envelope. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProducerInfo { - /// One of "sketchlib-go" or "asap_sketchlib". - #[prost(string, tag = "1")] - pub library: ::prost::alloc::string::String, - /// Semantic version string of the producing library, e.g. "0.1.0". - #[prost(string, tag = "2")] - pub version: ::prost::alloc::string::String, -} -/// HashSpec fully describes the hashing configuration so both sides can confirm -/// they will compute identical bucket indices. -/// -/// Both libraries share: -/// canonical_seed_index = 5 (CanonicalHashSeed / CANONICAL_HASH_SEED) -/// seed_list = the 20-entry table in hash.go / hash.rs -/// seed_derivation = ADDITIVE_OFFSET for per-row seeds -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HashSpec { - /// Hash algorithm used for all sketch operations. - #[prost(enumeration = "HashAlgorithm", tag = "1")] - pub algorithm: i32, - /// Index into seed_list used as the primary (canonical) hash seed. - /// Both libraries default to 5. - #[prost(uint32, tag = "2")] - pub canonical_seed_index: u32, - /// Complete seed table. Must be identical on both sides for correct results. - /// Both libraries currently use exactly 20 seeds; producers always emit all 20. - #[prost(uint64, repeated, tag = "3")] - pub seed_list: ::prost::alloc::vec::Vec, - /// How per-row seeds are derived from the base seed index. - #[prost(enumeration = "SeedDerivation", tag = "4")] - pub seed_derivation: i32, -} -/// HashAlgorithm enumerates supported hash algorithms. -/// Only XXH3_64 is cross-language stable for the current implementation. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum HashAlgorithm { - Unspecified = 0, - /// xxHash3 64-bit with seed (zeebo/xxh3 in Go; twox-hash XxHash3_64 in Rust). - /// This is the standard algorithm for both libraries. - Xxh364 = 1, - /// xxHash3 128-bit with seed. Used for HLL register indexing and packed matrix - /// hashes that exceed 64 bits. - Xxh3128 = 2, - /// xxHash 64-bit (non-XXH3). Reserved for legacy data; not produced by either - /// current library. - Xxhash64 = 3, -} -impl HashAlgorithm { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unspecified => "HASH_ALGORITHM_UNSPECIFIED", - Self::Xxh364 => "HASH_ALGORITHM_XXH3_64", - Self::Xxh3128 => "HASH_ALGORITHM_XXH3_128", - Self::Xxhash64 => "HASH_ALGORITHM_XXHASH64", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "HASH_ALGORITHM_UNSPECIFIED" => Some(Self::Unspecified), - "HASH_ALGORITHM_XXH3_64" => Some(Self::Xxh364), - "HASH_ALGORITHM_XXH3_128" => Some(Self::Xxh3128), - "HASH_ALGORITHM_XXHASH64" => Some(Self::Xxhash64), - _ => None, - } - } -} -/// SeedDerivation describes how per-row seed indices are produced. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SeedDerivation { - Unspecified = 0, - /// Each row r uses seed_list\[(base + r) % len(seed_list)\]. - /// Standard derivation for multi-row sketches (CountMin, CountSketch). - AdditiveOffset = 1, - /// All rows share a single hash (Packed64 or Packed128 fast-path). - /// Applicable only when the total bit-width fits in 64 or 128 bits. - Packed = 2, -} -impl SeedDerivation { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unspecified => "SEED_DERIVATION_UNSPECIFIED", - Self::AdditiveOffset => "SEED_DERIVATION_ADDITIVE_OFFSET", - Self::Packed => "SEED_DERIVATION_PACKED", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SEED_DERIVATION_UNSPECIFIED" => Some(Self::Unspecified), - "SEED_DERIVATION_ADDITIVE_OFFSET" => Some(Self::AdditiveOffset), - "SEED_DERIVATION_PACKED" => Some(Self::Packed), - _ => None, - } - } -} -/// CounterType identifies the numeric type used to store matrix counters. -/// Go defaults to FLOAT64; Rust defaults to INT32. -/// Determines which counts_* array is populated in CountMinState and -/// CountSketchState. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum CounterType { - Unspecified = 0, - /// Rust default for CountMin / CountSketch. - Int32 = 1, - Int64 = 2, - /// INT128 is stored as pairs of int64 (hi, lo) interleaved in counts_int. - Int128 = 3, - /// Go default for CountMin / CountSketch. - Float64 = 4, -} -impl CounterType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unspecified => "COUNTER_TYPE_UNSPECIFIED", - Self::Int32 => "COUNTER_TYPE_INT32", - Self::Int64 => "COUNTER_TYPE_INT64", - Self::Int128 => "COUNTER_TYPE_INT128", - Self::Float64 => "COUNTER_TYPE_FLOAT64", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "COUNTER_TYPE_UNSPECIFIED" => Some(Self::Unspecified), - "COUNTER_TYPE_INT32" => Some(Self::Int32), - "COUNTER_TYPE_INT64" => Some(Self::Int64), - "COUNTER_TYPE_INT128" => Some(Self::Int128), - "COUNTER_TYPE_FLOAT64" => Some(Self::Float64), - _ => None, - } - } -} -/// CountMinState is the portable state of a Count-Min Sketch. -/// -/// The counter matrix is stored flat in row-major order: -/// element [r][c] is at index r * cols + c. -/// -/// The Go implementation tracks auxiliary sum/L1/L2 arrays that the Rust -/// implementation does not produce. These fields are optional; consumers -/// that need them must recompute from raw insertions if absent. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CountMinState { - /// Matrix dimensions. Both must be > 0. - #[prost(uint32, tag = "1")] - pub rows: u32, - #[prost(uint32, tag = "2")] - pub cols: u32, - /// Indicates which counts_* field below is populated. - #[prost(enumeration = "CounterType", tag = "3")] - pub counter_type: i32, - /// Integer counters in row-major order, length = rows * cols. - /// Populated when counter_type ∈ {INT32, INT64}. - /// Use sint64 for efficient varint encoding of near-zero values. - /// For INT128 counters, values are interleaved as (hi_i, lo_i) pairs, - /// making the effective length 2 * rows * cols. - #[prost(sint64, repeated, tag = "4")] - pub counts_int: ::prost::alloc::vec::Vec, - /// Floating-point counters in row-major order, length = rows * cols. - /// Populated when counter_type = FLOAT64 (Go producer). - #[prost(double, repeated, tag = "5")] - pub counts_float: ::prost::alloc::vec::Vec, - /// Sum-weighted counter matrix (same layout as counts_*), length = rows * cols. - #[prost(double, repeated, tag = "6")] - pub sum_counts: ::prost::alloc::vec::Vec, - /// Sum-of-squares counter matrix, length = rows * cols. - #[prost(double, repeated, tag = "7")] - pub sum2_counts: ::prost::alloc::vec::Vec, - /// Per-row L1 norms, length = rows. - #[prost(double, repeated, tag = "8")] - pub l1: ::prost::alloc::vec::Vec, - /// Per-row L2 norms, length = rows. Recompute if absent. - #[prost(double, repeated, tag = "9")] - pub l2: ::prost::alloc::vec::Vec, -} -/// CountMinDelta carries only the matrix cells that changed by at least the -/// threshold between two consecutive snapshots, plus the full per-row L1/L2 -/// norm deltas (one entry per row, negligible size). -/// -/// Cells apply additively on the receiver: matrix[row][col] += d_count. CMS -/// counters only ever grow, but the delta encodes a signed count (sint64) so -/// the same wire form covers weighted/decay variants. -/// -/// The packed cell encoding (cell_rows / cell_cols / d_counts, tags 9-11) is -/// the canonical form; the legacy per-cell message (tag 3) is retained only so -/// payloads from older producers still decode. This message is byte-identical -/// to the Go reference implementation's CountMinDelta so the two runtimes emit -/// byte-identical delta frames for identical window state (cross-language byte -/// parity). -/// -/// CountMinDelta is structurally identical to CountSketchDelta: rows, cols, the -/// packed cell encoding, per-row norm deltas, and an optional repeated hh_keys -/// at the SAME tag number (6). Both sketches can track heavy hitters; whether -/// hh_keys is populated is a control-plane decision (it is empty/omitted when -/// heavy-hitter tracking is not enabled for this sketch). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CountMinDelta { - #[prost(uint32, tag = "1")] - pub rows: u32, - #[prost(uint32, tag = "2")] - pub cols: u32, - /// Deprecated: use cell_rows/cell_cols/d_counts (tags 9-11). Retained for - /// backward-compatible decode of payloads from older producers. - #[prost(message, repeated, tag = "3")] - pub cells_legacy: ::prost::alloc::vec::Vec, - /// Per-row L1 norm deltas, length = rows. - #[prost(double, repeated, tag = "4")] - pub l1: ::prost::alloc::vec::Vec, - /// Per-row L2 norm deltas, length = rows. - #[prost(double, repeated, tag = "5")] - pub l2: ::prost::alloc::vec::Vec, - /// Heavy-hitter candidate keys from an upstream tracker, mirroring - /// CountSketchDelta.hh_keys (same tag number 6, same wire shape). Downstream - /// queries the merged Count-Min matrix for each key to (re)build its Top-K - /// with globally-merged estimates. Empty/omitted when heavy-hitter tracking - /// is not enabled. - #[prost(string, repeated, tag = "6")] - pub hh_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// Packed cell encoding (canonical form): - /// - /// row index of each changed cell - #[prost(uint32, repeated, tag = "9")] - pub cell_rows: ::prost::alloc::vec::Vec, - /// col index of each changed cell - #[prost(uint32, repeated, tag = "10")] - pub cell_cols: ::prost::alloc::vec::Vec, - /// signed integer count delta - #[prost(sint64, repeated, tag = "11")] - pub d_counts: ::prost::alloc::vec::Vec, -} -/// CountMinCell is the deprecated per-cell delta record. Producers emit the -/// packed cell arrays on CountMinDelta instead; this message exists only to -/// decode payloads from older producers. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct CountMinCell { - #[prost(uint32, tag = "1")] - pub row: u32, - #[prost(uint32, tag = "2")] - pub col: u32, - /// deprecated: use CountMinDelta.d_counts - #[prost(double, tag = "3")] - pub d_count: f64, - /// deprecated: omitted in the packed encoding - #[prost(double, tag = "4")] - pub d_sum: f64, - /// deprecated: omitted in the packed encoding - #[prost(double, tag = "5")] - pub d_sum2: f64, -} -/// CountSketchState is the portable state of a Count (±1) Sketch. -/// -/// The counter matrix is signed because Count Sketch uses ±1 increments. -/// The matrix layout is identical to CountMinState (row-major, flat). -/// -/// L2 norms are derived values and must be recomputed on load: -/// l2\[r\] = sqrt( sum_c counts[r][c]^2 ) -/// Producers may include them as a convenience for consumers that only query. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CountSketchState { - /// Matrix dimensions. Both must be > 0. - #[prost(uint32, tag = "1")] - pub rows: u32, - #[prost(uint32, tag = "2")] - pub cols: u32, - /// Indicates which counts_* field below is populated. - #[prost(enumeration = "CounterType", tag = "3")] - pub counter_type: i32, - /// Signed integer counters in row-major order, length = rows * cols. - /// Populated when counter_type ∈ {INT32, INT64}. - #[prost(sint64, repeated, tag = "4")] - pub counts_int: ::prost::alloc::vec::Vec, - /// Floating-point counters in row-major order, length = rows * cols. - /// Populated when counter_type = FLOAT64 (Go producer). - #[prost(double, repeated, tag = "5")] - pub counts_float: ::prost::alloc::vec::Vec, - /// Per-row L2 norms, length = rows. Derived — recompute on load if absent. - #[prost(double, repeated, tag = "6")] - pub l2: ::prost::alloc::vec::Vec, - /// Optional heavy-hitter (TopK min-heap) state. - /// Absent when TopK tracking is disabled. - #[prost(message, optional, tag = "7")] - pub topk: ::core::option::Option, -} -/// TopKState is the serialized state of a min-heap heavy-hitter tracker. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TopKState { - /// Maximum number of entries retained. - #[prost(uint32, tag = "1")] - pub k: u32, - /// All retained entries. Order is not guaranteed (heap order is not preserved). - #[prost(message, repeated, tag = "2")] - pub entries: ::prost::alloc::vec::Vec, -} -/// HeapEntry is one heavy-hitter record inside a TopKState. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeapEntry { - /// String key identifying the heavy hitter (e.g. flow ID, IP address). - /// For binary keys, producers should use base64 or hex encoding and document - /// their choice; a future bytes_key field is reserved at number 3. - #[prost(string, tag = "1")] - pub key: ::prost::alloc::string::String, - /// Approximate frequency or weight accumulated for this key. - #[prost(double, tag = "2")] - pub count: f64, -} -/// CountSketchDelta carries only the matrix cells that changed by at least the -/// threshold between two consecutive snapshots, plus the full per-row L2 norm -/// deltas and any heavy-hitter candidate keys. -/// -/// Cells apply additively on the receiver: matrix[row][col] += d_count. Count -/// Sketch counters are signed (±1 increments), so d_count is a signed integer -/// (sint64). Heavy-hitter candidate keys (hh_keys) are re-queried against the -/// merged matrix downstream to rebuild the receiver's Top-K with accurate, -/// globally-merged estimates. -/// -/// The packed cell encoding (cell_rows / cell_cols / d_counts, tags 9-11) plus -/// l2 (tag 12) is the canonical form; the legacy per-cell message (tag 3), -/// l2_legacy (tag 4) and topk (tag 5) are retained only so payloads from older -/// producers still decode. This message is byte-identical to the Go reference -/// implementation's CountSketchDelta so the two runtimes emit byte-identical -/// delta frames for identical window state (cross-language byte parity). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CountSketchDelta { - #[prost(uint32, tag = "1")] - pub rows: u32, - #[prost(uint32, tag = "2")] - pub cols: u32, - /// Deprecated: use cell_rows/cell_cols/d_counts (tags 9-11). Retained for - /// backward-compatible decode of payloads from older producers. - #[prost(message, repeated, tag = "3")] - pub cells_legacy: ::prost::alloc::vec::Vec, - /// Deprecated: use l2 (tag 12) instead. - #[prost(double, repeated, tag = "4")] - pub l2_legacy: ::prost::alloc::vec::Vec, - /// Deprecated: use hh_keys (tag 6) instead. - #[prost(message, optional, tag = "5")] - pub topk: ::core::option::Option, - /// Heavy-hitter candidate keys from the upstream tracker. Downstream queries - /// the merged Count Sketch matrix for each key to build its Top-K. - #[prost(string, repeated, tag = "6")] - pub hh_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// Packed cell encoding (canonical form): - /// - /// row index of each changed cell - #[prost(uint32, repeated, tag = "9")] - pub cell_rows: ::prost::alloc::vec::Vec, - /// col index of each changed cell - #[prost(uint32, repeated, tag = "10")] - pub cell_cols: ::prost::alloc::vec::Vec, - /// signed integer count delta - #[prost(sint64, repeated, tag = "11")] - pub d_counts: ::prost::alloc::vec::Vec, - /// per-row L2 norm deltas - #[prost(double, repeated, tag = "12")] - pub l2: ::prost::alloc::vec::Vec, -} -/// CountSketchCell is the deprecated per-cell delta record. Producers emit the -/// packed cell arrays on CountSketchDelta instead; this message exists only to -/// decode payloads from older producers. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct CountSketchCell { - #[prost(uint32, tag = "1")] - pub row: u32, - #[prost(uint32, tag = "2")] - pub col: u32, - /// deprecated: use CountSketchDelta.d_counts - #[prost(double, tag = "3")] - pub d_count: f64, -} -/// HyperLogLogState is the portable state of a HyperLogLog cardinality sketch. -/// -/// Precision p determines the register count: num_registers = 2^precision. -/// Both libraries default to precision = 14 (16 384 registers). -/// -/// The variant field is required because the three estimator algorithms are -/// not interchangeable: loading an ErtlMLE state into a Regular estimator -/// (or vice versa) produces incorrect cardinality estimates. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HyperLogLogState { - /// HLL estimator variant. Required. - #[prost(enumeration = "HllVariant", tag = "1")] - pub variant: i32, - /// Log₂ of the number of registers (precision parameter p). - /// Default = 14 → 16 384 registers. - #[prost(uint32, tag = "2")] - pub precision: u32, - /// DENSE register encoding: raw register values, length = 2^precision. - /// Each byte stores the maximum (leading-zeros + 1) seen for that bucket. - /// Stored as a raw byte string for compact encoding (1 byte per register). - /// - /// Exactly one of `registers` (dense) or `registers_sparse` (sparse) is - /// populated by a producer. A decoder MUST read whichever is present; if - /// both are empty the sketch is all-zero. This field stays at tag 3 so old - /// dense-only readers keep working unchanged. - #[prost(bytes = "vec", tag = "3")] - pub registers: ::prost::alloc::vec::Vec, - /// HIP accumulator component kxq0. - #[prost(double, tag = "4")] - pub hip_kxq0: f64, - /// HIP accumulator component kxq1. - #[prost(double, tag = "5")] - pub hip_kxq1: f64, - /// HIP running cardinality estimate. - #[prost(double, tag = "6")] - pub hip_est: f64, - /// SPARSE register encoding (additive, tag 7). Populated instead of - /// `registers` (tag 3) when the number of non-zero registers is below the - /// producer's dense/sparse crossover. Old readers that predate this field - /// ignore tag 7 (proto skips unknown fields), so adding it is wire-backward- - /// compatible. New readers decode BOTH representations — see - /// HLLSparseRegisters. Reconstruction yields the identical 2^precision-byte - /// dense register array, so cardinality estimation is unaffected. - #[prost(message, optional, tag = "7")] - pub registers_sparse: ::core::option::Option, -} -/// HLLSparseRegisters is the sparse (HLL++ style) full-state encoding of the -/// register array. Only non-zero registers are stored, sorted by ascending -/// index and varint-packed as (index_delta, value) pairs: -/// -/// for each non-zero register, in ascending index order: -/// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -/// uvarint(value) // 1..=Q+1, always 1 byte in practice -/// -/// For low/medium cardinality (most registers zero) this is far smaller than -/// the dense 2^precision-byte array. Decoding allocates a zero-filled dense -/// array of length `num_registers` and writes each decoded (index, value). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HllSparseRegisters { - /// Total number of registers in the reconstructed dense array (= 2^precision). - /// Carried explicitly so the decoder can size the dense buffer without - /// re-deriving it from precision. - #[prost(uint32, tag = "1")] - pub num_registers: u32, - /// Varint-packed (index_delta, value) pairs for non-zero registers, in - /// ascending index order. See message-level comment for the exact layout. - #[prost(bytes = "vec", tag = "2")] - pub packed: ::prost::alloc::vec::Vec, -} -/// HLLDelta carries the registers that increased between two consecutive -/// snapshots. HLL uses max semantics, so only increases are meaningful; -/// at a fixed precision a register can never decrease. Each update is applied -/// losslessly on the receiver via register\[index\] = max(register\[index\], value) -/// — no register update is ever dropped (there is no threshold to apply). -/// -/// The increased registers are varint-packed exactly like -/// HLLSparseRegisters.packed: sorted ascending by index, each register emitted -/// as (index_delta, value) where index_delta = index - prev_index (prev_index -/// starts at 0) and value is the new (larger) register value: -/// -/// for each increased register, in ascending index order: -/// uvarint(index - prev_index) // prev_index starts at 0; deltas are >= 0 -/// uvarint(value) // 1..=Q+1, always 1 byte in practice -/// -/// A single-emit delta against the all-zero snapshot is therefore the same size -/// as the full sparse frame, and a sub-window delta is strictly smaller (it -/// packs only the registers that actually grew). This replaces the previous -/// per-register sub-message encoding, eliminating ~6–8 bytes of tag/length -/// overhead per register. -/// -/// This message is byte-identical to the Go reference implementation's HLLDelta -/// so the two runtimes emit byte-identical delta frames for identical window -/// state (cross-language byte parity). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HllDelta { - /// Varint-packed (index_delta, value) pairs for the registers that increased, - /// in ascending index order. Same layout as HLLSparseRegisters.packed. Empty - /// when no register changed. - #[prost(bytes = "vec", tag = "1")] - pub packed_updates: ::prost::alloc::vec::Vec, -} -/// HLLVariant identifies which HLL estimator algorithm the registers belong to. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum HllVariant { - Unspecified = 0, - /// Classic HLL with bias-correction tables (Flajolet et al. 2007). - Regular = 1, - /// Otmar Ertl's MLE estimator (arXiv:1702.01284). - /// Originally derived from Apache DataFusion's HLL implementation. - /// Not compatible with REGULAR registers. - ErtlMle = 2, - /// Kevin Lang's HyperLogLog++ Improved Proposal (HIP). - /// Requires hip_kxq0, hip_kxq1, hip_est to be populated. - Hip = 3, -} -impl HllVariant { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unspecified => "HLL_VARIANT_UNSPECIFIED", - Self::Regular => "HLL_VARIANT_REGULAR", - Self::ErtlMle => "HLL_VARIANT_ERTL_MLE", - Self::Hip => "HLL_VARIANT_HIP", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "HLL_VARIANT_UNSPECIFIED" => Some(Self::Unspecified), - "HLL_VARIANT_REGULAR" => Some(Self::Regular), - "HLL_VARIANT_ERTL_MLE" => Some(Self::ErtlMle), - "HLL_VARIANT_HIP" => Some(Self::Hip), - _ => None, - } - } -} -/// KLLState is the portable state of a KLL quantile sketch. -/// -/// The items array is partitioned into compaction levels by the levels\[\] array: -/// items\[ levels[i\] .. levels\[i+1\] ) are the retained items at level i. -/// -/// Derived state that must be recomputed on load (not serialized): -/// capacity_cache — computed from k, m, and level boundaries. -/// level0_capacity — computed from k and num_levels. -/// top_height — equals num_levels - 1. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct KllState { - /// Target compactor capacity k. Must be >= 8. - #[prost(uint32, tag = "1")] - pub k: u32, - /// Minimum compactor size m. Must satisfy 2 <= m <= k. - #[prost(uint32, tag = "2")] - pub m: u32, - /// Number of levels currently in use. - #[prost(uint32, tag = "3")] - pub num_levels: u32, - /// Level boundary indices into items\[\], length = num_levels + 1. - /// levels\[0\] is always 0; levels\[num_levels\] = len(items). - #[prost(uint32, repeated, tag = "4")] - pub levels: ::prost::alloc::vec::Vec, - /// All retained samples in level order, length = levels\[num_levels\]. - /// - /// RAW-F64 representation. Populated when the value-offset fixed-point - /// representation (offset/value_scale/residuals, fields 7–9) is NOT used — - /// i.e. when the producer could not represent the retained set exactly at - /// any candidate decimal scale and fell back to raw f64. When residuals - /// (field 9) is non-empty, items MUST be empty and decoders MUST reconstruct - /// from the fixed-point fields instead. - #[prost(double, repeated, tag = "5")] - pub items: ::prost::alloc::vec::Vec, - /// Random bit generator state for deterministic compaction continuation. - /// Required for sketches that will receive further updates after loading. - /// Consumers that only query may ignore this field. - #[prost(message, optional, tag = "6")] - pub coin: ::core::option::Option, - /// Per-sketch subtraction constant. value = residual * 10^value_scale + offset. - #[prost(double, tag = "7")] - pub offset: f64, - /// Decimal scale exponent. A value v maps to residual = round((v - offset) * - /// 10^(-value_scale)); reconstruction multiplies by 10^value_scale. scale 0 - /// means residuals are integers; negative scales carry fractional precision - /// (e.g. value_scale = -3 -> millis). sint32 so negative scales varint-pack. - #[prost(sint32, tag = "8")] - pub value_scale: i32, - /// Per-sample residual integers, same level order and length as a raw items\[\] - /// would have. Zigzag varint (sint64) keeps small magnitudes compact. When - /// present and non-empty, this is the authoritative item source and items\[\] - /// (field 5) MUST be empty. - #[prost(sint64, repeated, tag = "9")] - pub residuals: ::prost::alloc::vec::Vec, -} -/// CoinState is the RNG used by KLL's probabilistic compaction. -/// Both Go and Rust use an identical xorshift-based generator. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct CoinState { - /// Xorshift generator internal state word. - #[prost(uint64, tag = "1")] - pub state: u64, - /// Up to 64 random bits buffered for single-bit coin flips. - #[prost(uint64, tag = "2")] - pub bit_cache: u64, - /// Number of valid bits remaining in bit_cache (0–64). - #[prost(uint32, tag = "3")] - pub remaining_bits: u32, -} -/// DDSketchState is the portable state of a DDSketch quantile sketch. -/// -/// Only alpha is required to reconstruct the index mapping. Consumers must -/// recompute all derived mapping values on load: -/// gamma = (1.0 + alpha) / (1.0 - alpha) -/// log_gamma = ln(gamma) -/// inv_log_gamma = 1.0 / log_gamma -/// -/// Bucket indices: a value v maps to bucket floor(ln(v) * inv_log_gamma). -/// Stored bucket absolute_index = array_index + store_offset. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DdSketchState { - /// Relative accuracy guarantee. Required. Must satisfy 0 < alpha < 1. - #[prost(double, tag = "1")] - pub alpha: f64, - /// Bucket count array in index order (ascending absolute bucket index). - /// Absolute index of store_counts\[i\] is i + store_offset. - #[prost(uint64, repeated, tag = "2")] - pub store_counts: ::prost::alloc::vec::Vec, - /// Absolute bucket index corresponding to store_counts\[0\]. - /// May be negative (values < 1.0 map to negative bucket indices). - #[prost(sint32, tag = "3")] - pub store_offset: i32, - /// Signed extension: negative magnitudes use the same mapping as positives. - #[prost(uint64, repeated, tag = "16")] - pub negative_store_counts: ::prost::alloc::vec::Vec, - #[prost(sint32, tag = "17")] - pub negative_store_offset: i32, - #[prost(uint64, tag = "18")] - pub zero_count: u64, -} -/// DDSketchDelta carries only the buckets that changed above threshold T. -/// -/// DataPoint-level metric scalars (d_count/d_sum/new_min/new_max/min_changed/ -/// max_changed) are intentionally NOT carried on the wire: the count delta is -/// recoverable by summing bucket deltas, and min/max/quantiles are derived from -/// the bucket distribution within the relative-accuracy guarantee. This message -/// is byte-identical to the Go reference implementation's DDSketchDelta so the -/// Go and Rust runtimes emit byte-identical delta frames for identical window -/// state (cross-language byte parity). -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DdSketchDelta { - #[prost(message, repeated, tag = "1")] - pub buckets: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "16")] - pub negative_buckets: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "17")] - pub zero_count: u64, -} -/// DDSketchBucketDelta is the delta for one log-scale bucket. -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct DdSketchBucketDelta { - /// signed bucket index (negative for values < 1) - #[prost(sint32, tag = "1")] - pub index: i32, - /// Δcount ≥ threshold T - #[prost(uint64, tag = "2")] - pub d_count: u64, -} -/// UnivMonState is the portable state of a Universal Monitoring (UnivMon) sketch. -/// -/// UnivMon is a hierarchy of L Count Sketches paired with L heavy-hitter heaps. -/// An item's hash trailing-zero count determines which layers receive it, giving -/// each layer a geometrically decreasing sample of the stream. -/// -/// All layers share the same sketch dimensions and heap capacity. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnivMonState { - /// Number of layers L in the hierarchy (inclusive of all levels). - #[prost(uint32, tag = "1")] - pub layer_size: u32, - /// Count Sketch dimensions shared by all layers. - #[prost(uint32, tag = "2")] - pub sketch_rows: u32, - #[prost(uint32, tag = "3")] - pub sketch_cols: u32, - /// Heavy-hitter heap capacity shared by all layers. - #[prost(uint32, tag = "4")] - pub heap_size: u32, - /// Total items inserted across all layers (the "bucket size" counter). - #[prost(uint64, tag = "5")] - pub bucket_size: u64, - /// Per-layer state, in order from layer 0 (finest) to layer_size-1 (coarsest). - /// Length must equal layer_size. - #[prost(message, repeated, tag = "6")] - pub layers: ::prost::alloc::vec::Vec, -} -/// UnivMonLayer is the joint state of one UnivMon layer. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnivMonLayer { - /// The Count Sketch accumulating items at this layer. - #[prost(message, optional, tag = "1")] - pub sketch: ::core::option::Option, - /// The heavy-hitter heap for this layer. - #[prost(message, optional, tag = "2")] - pub heap: ::core::option::Option, -} -/// HydraState is the portable state of a Hydra multi-sketch framework. -/// -/// Hydra maintains a grid of row_num child sketches (cells), each of the -/// same type (counter_type). Items are routed to a specific row cell using a -/// position hash derived from the item's subkey. An optional global "big -/// counter" sketch aggregates the full stream. -/// -/// Current implementations set col_num = 1 (one sketch per row). The field is -/// included for schema completeness and future extensions. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HydraState { - /// Number of sketch rows (D / number of subpopulations). - #[prost(uint32, tag = "1")] - pub row_num: u32, - /// Number of sketch columns per row. Currently always 1. - #[prost(uint32, tag = "2")] - pub col_num: u32, - /// The sketch type used for every cell. - #[prost(enumeration = "HydraCounterType", tag = "3")] - pub counter_type: i32, - /// Serialized cells in row-major order, length = row_num * col_num. - /// Each cell's oneof field must match counter_type. - #[prost(message, repeated, tag = "4")] - pub cells: ::prost::alloc::vec::Vec, - /// Optional global sketch covering the entire stream. - /// Absent when global counter is disabled (enable_global_counter = false). - #[prost(message, optional, tag = "5")] - pub big_counter: ::core::option::Option, - /// Seed index used for subpopulation routing (Go: seedHydra, Rust: HYDRA_SEED). - /// Both libraries default to 6. - #[prost(uint32, tag = "6")] - pub seed_index: u32, - /// True when per-cell TopK tracking is enabled. - #[prost(bool, tag = "7")] - pub enable_topk: bool, - /// True when fanout subkey expansion is active. - #[prost(bool, tag = "8")] - pub fanout_subkeys: bool, -} -/// HydraCell holds the serialized state of one Hydra grid cell. -/// Exactly one field in the oneof must be set, and it must match the parent -/// HydraState.counter_type. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HydraCell { - #[prost(oneof = "hydra_cell::Sketch", tags = "1, 2, 3, 4, 5")] - pub sketch: ::core::option::Option, -} -/// Nested message and enum types in `HydraCell`. -pub mod hydra_cell { - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sketch { - #[prost(message, tag = "1")] - CountMin(super::CountMinState), - #[prost(message, tag = "2")] - CountSketch(super::CountSketchState), - #[prost(message, tag = "3")] - Hll(super::HyperLogLogState), - #[prost(message, tag = "4")] - Kll(super::KllState), - #[prost(message, tag = "5")] - Univmon(super::UnivMonState), - } -} -/// HydraCounterType identifies the sketch algorithm used inside Hydra cells. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum HydraCounterType { - Unspecified = 0, - CountMin = 1, - CountSketch = 2, - Hll = 3, - Kll = 4, - Univmon = 5, -} -impl HydraCounterType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Self::Unspecified => "HYDRA_COUNTER_TYPE_UNSPECIFIED", - Self::CountMin => "HYDRA_COUNTER_TYPE_COUNT_MIN", - Self::CountSketch => "HYDRA_COUNTER_TYPE_COUNT_SKETCH", - Self::Hll => "HYDRA_COUNTER_TYPE_HLL", - Self::Kll => "HYDRA_COUNTER_TYPE_KLL", - Self::Univmon => "HYDRA_COUNTER_TYPE_UNIVMON", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "HYDRA_COUNTER_TYPE_UNSPECIFIED" => Some(Self::Unspecified), - "HYDRA_COUNTER_TYPE_COUNT_MIN" => Some(Self::CountMin), - "HYDRA_COUNTER_TYPE_COUNT_SKETCH" => Some(Self::CountSketch), - "HYDRA_COUNTER_TYPE_HLL" => Some(Self::Hll), - "HYDRA_COUNTER_TYPE_KLL" => Some(Self::Kll), - "HYDRA_COUNTER_TYPE_UNIVMON" => Some(Self::Univmon), - _ => None, - } - } -} -/// CocoSketchState is the portable state of a CocoSketch flow-size estimator. -/// -/// The bucket table is stored flat in row-major order (d rows × width cols). -/// Element [r][c] is at index r * width + c. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CocoSketchState { - /// Number of hash rows. - #[prost(uint32, tag = "1")] - pub d: u32, - /// Number of buckets per row. - #[prost(uint32, tag = "2")] - pub width: u32, - /// Full hash stored in each bucket, length = d * width. - /// Zero means the bucket is empty (has_keys\[i\] == false). - #[prost(uint64, repeated, tag = "3")] - pub hashes: ::prost::alloc::vec::Vec, - /// Cumulative count value per bucket, length = d * width. - #[prost(uint64, repeated, tag = "4")] - pub vals: ::prost::alloc::vec::Vec, - /// Whether each bucket holds a valid key entry, length = d * width. - #[prost(bool, repeated, tag = "5")] - pub has_keys: ::prost::alloc::vec::Vec, -} -/// ElasticState is the portable state of an Elastic Sketch. -/// -/// The Elastic Sketch has a heavy part (one bucket per index) and a light -/// Count-Min layer. The heavy part is serialized as parallel arrays. -/// The light layer is stored as a CountMinState with FLOAT64 counters. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ElasticState { - /// Number of heavy buckets. - #[prost(uint32, tag = "1")] - pub bucket_count: u32, - /// Per-bucket flow identifiers (strings), length = bucket_count. - /// Empty string means the bucket is unoccupied. - #[prost(string, repeated, tag = "2")] - pub flow_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// Positive vote counts per bucket, length = bucket_count. - #[prost(sint32, repeated, tag = "3")] - pub vote_pos: ::prost::alloc::vec::Vec, - /// Negative vote counts per bucket, length = bucket_count. - #[prost(sint32, repeated, tag = "4")] - pub vote_neg: ::prost::alloc::vec::Vec, - /// Whether each bucket is in eviction state, length = bucket_count. - #[prost(bool, repeated, tag = "5")] - pub evictions: ::prost::alloc::vec::Vec, - /// Light Count-Min layer (FLOAT64 counters). - #[prost(message, optional, tag = "6")] - pub light: ::core::option::Option, -} -/// SketchEnvelope is the portable container for all sketch transfers. -/// Producers fill exactly one field inside sketch_state. -/// Consumers check format_version first, then dispatch on the oneof. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SketchEnvelope { - /// Schema version. Consumers should reject envelopes whose version exceeds - /// the highest version they understand. - /// 1 = CountMin, Count, HLL, KLL, DDSketch - /// 2 = UnivMon, Hydra (reserved) - /// 3 = CocoSketch, ElasticSketch (reserved) - #[prost(uint32, tag = "1")] - pub format_version: u32, - /// Library and build info of the producing side. - #[prost(message, optional, tag = "2")] - pub producer: ::core::option::Option, - /// Complete hashing configuration. Consumers must validate this against their - /// own configuration before performing updates or queries on the loaded sketch. - /// Mismatches do not prevent loading but will cause diverging results. - #[prost(message, optional, tag = "3")] - pub hash_spec: ::core::option::Option, - /// Geometric / NitroSketch sampling probability for the producing sketch. - /// - /// The producer admits each stream update with probability `sample_p` (using - /// geometric skip-sampling, statistically identical to per-update Bernoulli(p)) - /// and stores the RAW SAMPLED sketch state — never the rescaled state. The - /// consumer applies the `× 1/sample_p` rescale at QUERY time on the count-like - /// estimators (HLL cardinality, CountMin / CountSketch frequency, SUM / COUNT); - /// quantile estimators (KLL, DDSketch) are scale-invariant under uniform - /// sampling so they carry `sample_p` but need no rescale. - /// - /// Lives on the envelope (not per-sketch state) so downstream consumers that - /// construct per-sketch state structs by literal are unaffected. - /// - /// Dual-read default: a value of 0.0 (the proto3 default for an unset field) - /// is interpreted as 1.0 (exact, no sampling) for backward compatibility with - /// producers that predate this field. With sample_p = 1.0 a 0.0/unset envelope - /// is byte-identical to the pre-sampling format. - #[prost(double, tag = "4")] - pub sample_p: f64, - /// The sketch payload. Exactly one field must be set. - #[prost( - oneof = "sketch_envelope::SketchState", - tags = "10, 11, 12, 13, 14, 15, 16, 17, 18" - )] - pub sketch_state: ::core::option::Option, -} -/// Nested message and enum types in `SketchEnvelope`. -pub mod sketch_envelope { - /// The sketch payload. Exactly one field must be set. - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum SketchState { - #[prost(message, tag = "10")] - CountMin(super::CountMinState), - #[prost(message, tag = "11")] - CountSketch(super::CountSketchState), - #[prost(message, tag = "12")] - Hll(super::HyperLogLogState), - #[prost(message, tag = "13")] - Kll(super::KllState), - #[prost(message, tag = "14")] - Ddsketch(super::DdSketchState), - #[prost(message, tag = "15")] - Univmon(super::UnivMonState), - #[prost(message, tag = "16")] - Hydra(super::HydraState), - #[prost(message, tag = "17")] - Coco(super::CocoSketchState), - #[prost(message, tag = "18")] - Elastic(super::ElasticState), - } -} diff --git a/src/sketches/testdata/cms_envelope_golden.hex b/src/sketches/testdata/cms_envelope_golden.hex deleted file mode 100644 index c859ecc5..00000000 --- a/src/sketches/testdata/cms_envelope_golden.hex +++ /dev/null @@ -1 +0,0 @@ -080152ce4008041080101802228040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00422000000000000049400000000000004940000000000000494000000000000049404a200000000000406f400000000000406f400000000000406f400000000000406f40 \ No newline at end of file diff --git a/src/sketches/testdata/hll_envelope_golden.hex b/src/sketches/testdata/hll_envelope_golden.hex deleted file mode 100644 index cd7760ec..00000000 --- a/src/sketches/testdata/hll_envelope_golden.hex +++ /dev/null @@ -1 +0,0 @@ -0801628880010802100e1a80800100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000006000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/tests/cs_envelope_golden.hex b/tests/cs_envelope_golden.hex deleted file mode 100644 index e1c665c2..00000000 --- a/tests/cs_envelope_golden.hex +++ /dev/null @@ -1 +0,0 @@ -08015aa40c0803108004180222800c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000090000000000000a000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032180000000000405f400000000000405f400000000000405f40 diff --git a/tests/ddsketch_signed.rs b/tests/ddsketch_signed.rs index 99bf8768..a14c8d25 100644 --- a/tests/ddsketch_signed.rs +++ b/tests/ddsketch_signed.rs @@ -38,37 +38,6 @@ fn signed_merge_and_serialization() { assert_eq!(c2.get_value_at_quantile(0.5), Some(0.0)); } -#[test] -fn signed_delta_and_proto_round_trips() { - use prost::Message; - let mut base = PortableDdSketch::new(0.01); - for v in [-100., -1., 0., 1., 100.] { - base.update(v); - } - let mut current = base.clone(); - for v in [-1000., -0.1, 0., 0., 0.1, 1000.] { - current.update(v); - } - let mut protobuf = base.clone(); - protobuf - .apply_delta_bytes(¤t.compute_delta(&base, 1)) - .unwrap(); - let mut msgpack = base; - msgpack - .apply_delta_msgpack_bytes(¤t.compute_delta_msgpack(&msgpack, 1)) - .unwrap(); - let bytes = current.to_proto().encode_to_vec(); - let snapshot = PortableDdSketch::from_proto( - asap_sketchlib::proto::sketchlib::DdSketchState::decode(bytes.as_slice()).unwrap(), - ); - for restored in [protobuf, msgpack, snapshot] { - assert_eq!(restored.total_count(), 11); - for q in [0., 0.1, 0.5, 0.9, 1.] { - assert!((restored.quantile(q).unwrap() - current.quantile(q).unwrap()).abs() < 1e-9); - } - } -} - #[test] fn signed_order_statistics_obey_relative_bound() { let mut values: Vec = (-300i32..=300) diff --git a/tests/sketches_go_parity_probe.rs b/tests/sketches_go_parity_probe.rs deleted file mode 100644 index 6848d7fc..00000000 --- a/tests/sketches_go_parity_probe.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! Probe: `sketches::*` defaults produce Go-byte-parity output. -//! -//! Approach: reuse the exact same Go-golden envelopes that the -//! `message_pack_format::portable` parity tests use, but build the -//! matrix via `sketches::CountSketch` / `sketches::CountMinSketch` with `FastPath` -//! and `DefaultXxHasher`. Identical bytes confirm the shared FastPath -//! math is sufficient. - -use asap_sketchlib::common::DataInput; -use asap_sketchlib::common::hash::DEFAULT_CANONICAL_SEED_INDEX; -use asap_sketchlib::common::structures::HllRegistersP14; -use asap_sketchlib::proto::sketchlib::{ - CountMinState, CountSketchState, CounterType, DdSketchState, HyperLogLogState, SketchEnvelope, - sketch_envelope::SketchState, -}; -use asap_sketchlib::sketches::countminsketch::CountMinSketch; -use asap_sketchlib::sketches::countsketch::CountSketch; -use asap_sketchlib::sketches::ddsketch::DdSketch; -use asap_sketchlib::sketches::hll::{Ertl, HyperLogLogImpl}; -use asap_sketchlib::{DefaultXxHasher, FastPath, Vector2D}; -use prost::Message; - -/// 1577-byte envelope captured from `sketchlib-go::CountSketch.SerializeProtoBytes` -/// for `(rows=3, cols=512)` × `goldenCsKeys()` (25 keys "k-a".."k-e", each -/// repeated 5×). Identical to the constant in -/// `src/message_pack_format/portable/countsketch.rs::test_update_then_envelope_matches_sketchlib_go_bytes`. -const COUNTSKETCH_GOLDEN_HEX: &str = include_str!("cs_envelope_golden.hex"); - -/// 8275-byte CMS Frequency-Only golden, captured from -/// `sketchlib-go::CountMinSketch.SerializeProtoBytesFO` for -/// `(rows=4, cols=2048)` × `goldenCmsKeys()` (10 keys "flow-0".."flow-9", -/// each repeated 5×). Same file the wrapper test already uses. -const COUNTMIN_GOLDEN_HEX: &str = include_str!("../src/sketches/testdata/cms_envelope_golden.hex"); - -/// 16 398-byte envelope captured from `sketchlib-go::HyperLogLog.SerializePortable` -/// for precision=14, `(1..=50)` IEEE-754-LE byte-key input. Same file -/// the wrapper test already uses. -const HLL_GOLDEN_HEX: &str = include_str!("../src/sketches/testdata/hll_envelope_golden.hex"); - -/// 403-byte DDSketch envelope for `alpha=0.01`, `(1..=50)` integer-as-f64 -/// input, without the DataPoint-level METRIC scalars (count/sum/min/max → -/// proto tags 4-7 reserved). -/// -/// NOTE: this is currently the RUST-produced value; it MUST be reconciled -/// against the parallel `sketchlib-go` golden regeneration before declaring -/// cross-language byte parity. -const DDSKETCH_GOLDEN_HEX: &str = "0801728e03096214ae47e17a843f128003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000100000000000000000000000000000100000000000000000000010000000000000000010000000000000001000000000001000000000001000000000001000000010000000001000000010000010000000100000100000100000100000100010000010001000100010001000100010001000100010100010100010100010101000101010100010101010101010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000187f"; - -#[test] -fn sketches_count_sketch_fastpath_matches_go_count_sketch_envelope() { - let rows = 3usize; - let cols = 512usize; - - let mut sk: CountSketch, FastPath, DefaultXxHasher> = - CountSketch::with_dimensions(rows, cols); - for i in 0..25 { - let key = format!("k-{}", (b'a' + (i % 5) as u8) as char); - sk.update_weighted(&DataInput::String(key), 1i64); - } - - let storage = sk.as_storage(); - let mut counts_int: Vec = Vec::with_capacity(rows * cols); - let mut l2: Vec = Vec::with_capacity(rows); - for r in 0..rows { - let mut row_l2 = 0.0f64; - for c in 0..cols { - let cell = *storage.get(r, c).expect("cell in range"); - counts_int.push(cell); - row_l2 += (cell as f64) * (cell as f64); - } - l2.push(row_l2); - } - - let state = CountSketchState { - rows: rows as u32, - cols: cols as u32, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - l2, - topk: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::CountSketch(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - let want = decode_hex(COUNTSKETCH_GOLDEN_HEX); - assert_eq!( - got, - want, - "sketches::CountSketch, FastPath> envelope diverges from Go golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); -} - -#[test] -fn sketches_countmin_fastpath_matches_go_count_min_envelope() { - let rows = 4usize; - let cols = 2048usize; - - let mut sk: CountMinSketch, FastPath, DefaultXxHasher> = - CountMinSketch::with_dimensions(rows, cols); - for i in 0..50u64 { - let key = format!("flow-{}", i % 10); - sk.update_weighted(&DataInput::String(key), 1.0f64); - } - - let storage = sk.as_storage(); - let mut counts_int: Vec = Vec::with_capacity(rows * cols); - let mut l1: Vec = Vec::with_capacity(rows); - let mut l2: Vec = Vec::with_capacity(rows); - for r in 0..rows { - let mut row_l1 = 0.0f64; - let mut row_l2 = 0.0f64; - for c in 0..cols { - let cell = *storage.get(r, c).expect("cell in range"); - counts_int.push(cell as i64); - row_l1 += cell; - row_l2 += cell * cell; - } - l1.push(row_l1); - l2.push(row_l2); - } - - let state = CountMinState { - rows: rows as u32, - cols: cols as u32, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - sum_counts: Vec::new(), - sum2_counts: Vec::new(), - l1, - l2, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::CountMin(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - let want = decode_hex(COUNTMIN_GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "CountMinSketch envelope length: got {} want {}", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "sketches::CountMinSketch, FastPath> envelope diverges from Go golden", - ); -} - -#[test] -fn sketches_hll_classic_matches_go_envelope() { - // Go's `HyperLogLog.SerializePortable` emits variant=DATAFUSION=2 - // (Ertl). Use the same variant + precision=14 + same input here. - let mut sk: HyperLogLogImpl = - >::new(); - for i in 1..=50i32 { - let v = i as f64; - let bytes = v.to_le_bytes(); - let hashed = ::hash64_seeded( - DEFAULT_CANONICAL_SEED_INDEX, - &DataInput::Bytes(&bytes), - ); - sk.update_with_hash(hashed); - } - - let state = HyperLogLogState { - // 2 = DATAFUSION on the Go wire (matches Ertl) - variant: 2, - precision: 14, - registers: sk.registers_as_slice().to_vec(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - // This parity probe pins the DENSE wire form; leave sparse unset. - registers_sparse: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Hll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - let want = decode_hex(HLL_GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "HLL envelope length: got {} want {}", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "sketches::HyperLogLogImpl envelope diverges from Go golden", - ); -} - -#[test] -fn sketches_ddsketch_matches_go_envelope() { - let mut sk = DdSketch::new(0.01); - for i in 1..=50i32 { - sk.add(&(i as f64)); - } - - // Go serializes `alpha` round-tripped through gamma to mirror its - // internal storage: `alpha_wire = (γ-1)/(γ+1)`, `γ = (1+α)/(1-α)`. - // This is a 25-ULP shift from the user-supplied α=0.01. - let alpha = sk.alpha(); - let gamma = (1.0 + alpha) / (1.0 - alpha); - let alpha_wire = (gamma - 1.0) / (gamma + 1.0); - // The DataPoint-level METRIC scalars (count/sum/min/max) are not on the - // wire; only alpha + the bucket array. - let state = DdSketchState { - alpha: alpha_wire, - store_counts: sk.positive_store_counts().to_vec(), - store_offset: sk.positive_store_offset(), - ..Default::default() - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Ddsketch(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - let want = decode_hex(DDSKETCH_GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "DDSketch envelope length: got {} want {}", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "sketches::DDSketch envelope diverges from Go golden", - ); -} - -fn decode_hex(s: &str) -> Vec { - s.trim() - .as_bytes() - .chunks(2) - .map(|pair| (hex_nibble(pair[0]) << 4) | hex_nibble(pair[1])) - .collect() -} - -fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } -} diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs deleted file mode 100644 index 1c916a2e..00000000 --- a/tests/xtest_consumer.rs +++ /dev/null @@ -1,1246 +0,0 @@ -//! xtest_consumer — Cross-language integration test: Rust consumer side. -//! -//! Reads protobuf-encoded SketchEnvelope files written by the Go xtest_producer, -//! deserialises each sketch from the portable wire format, and runs sanity queries -//! to confirm the data survived the language boundary intact. -//! -//! Files consumed (from $XTEST_DIR/): -//! countmin.pb — CountMinState with float64 counters -//! kll.pb — KLLState with quantile items and coin RNG state -//! ddsketch.pb — DdSketchState with alpha + bucket array -//! hll.pb — HyperLogLogState (Ertl variant) -//! countsketch.pb — CountSketchState with float64 signed counters -//! coco.pb — CocoSketchState (hash+val+hasKey buckets) -//! elastic.pb — ElasticState (heavy buckets + light CountMinSketch) -//! univmon.pb — UnivMonState (layered CountSketch + TopK heaps) -//! hydra.pb — HydraState (CM-cell grid) -//! -//! Usage: -//! XTEST_DIR= cargo test --test xtest_consumer -- --nocapture - -use asap_sketchlib::message_pack_format::portable::sampling::effective_sample_p; -use asap_sketchlib::message_pack_format::portable::sampling::rescale_count; -use asap_sketchlib::proto::sketchlib::*; -use prost::Message; -use std::{ - env, fs, - path::{Path, PathBuf}, - process::Command, - sync::atomic::{AtomicU64, Ordering}, - time::{SystemTime, UNIX_EPOCH}, -}; -use twox_hash::XxHash3_64; - -#[test] -fn cross_language_proto() { - let Some(xtest_dir) = XtestDir::prepare() else { - eprintln!( - "skipping cross_language_proto: no XTEST_DIR provided and Go xtest producer is unavailable" - ); - return; - }; - let in_dir = xtest_dir.path(); - - println!("======================================================="); - println!(" asap_sketchlib ← xtest_consumer"); - println!("======================================================="); - - let mut all_ok = true; - - // ----------------------------------------------------------------------- - // CountMinSketch - // ----------------------------------------------------------------------- - println!(); - println!("[CountMin] Step 1/3 — Read countmin.pb"); - let bytes = read_file(in_dir.join("countmin.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode countmin envelope"); - - println!( - "[CountMin] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let cm_state = match env.sketch_state { - Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {other:?}"), - }; - - println!( - "[CountMin] rows={} cols={} counter_type={:?}", - cm_state.rows, - cm_state.cols, - CounterType::try_from(cm_state.counter_type) - ); - - // Reconstruct the row-major matrix as f64. The producer populates EITHER - // counts_int (counter_type ∈ {INT32,INT64}) OR counts_float (FLOAT64), - // discriminated by counter_type — read whichever is present. - let rows = cm_state.rows as usize; - let cols = cm_state.cols as usize; - let counts: Vec = if !cm_state.counts_float.is_empty() { - cm_state.counts_float.clone() - } else { - cm_state.counts_int.iter().map(|&v| v as f64).collect() - }; - let counts = &counts; - if counts.len() != rows * cols { - eprintln!( - "[CountMin] FAIL: counts length {} != rows*cols {}", - counts.len(), - rows * cols - ); - all_ok = false; - } - - // Point query for hot key "item:42" using the same hash formula as Go. - let hot_key = b"item:42" as &[u8]; - let hot_hash = xxh3_64_seeded(SEED_0, hot_key); - - println!("[CountMin] Step 3/3 — Point query 'item:42' (hash=0x{hot_hash:016x})"); - - let bits_per_row = col_bits(cols); - let mask = (cols as u64) - 1; - let mut min_freq = f64::MAX; - for r in 0..rows { - let shift = (r as u64) * bits_per_row; - let c = ((hot_hash >> shift) & mask) as usize; - let v = counts[r * cols + c]; - if v < min_freq { - min_freq = v; - } - } - println!("[CountMin] min frequency estimate = {min_freq:.0} (expect ≥ 101)"); - if min_freq < 101.0 { - eprintln!("[CountMin] FAIL: frequency {min_freq:.0} < 101"); - all_ok = false; - } else { - println!("[CountMin] PASS"); - } - - // ----------------------------------------------------------------------- - // KLL - // ----------------------------------------------------------------------- - println!(); - println!("[KLL] Step 1/3 — Read kll.pb"); - let bytes = read_file(in_dir.join("kll.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode kll envelope"); - - println!( - "[KLL] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let kll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(ref s)) => s.clone(), - other => panic!("expected KLL sketch_state, got {other:?}"), - }; - - println!( - "[KLL] k={} m={} num_levels={} items={} levels={}", - kll_state.k, - kll_state.m, - kll_state.num_levels, - kll_state.items.len(), - kll_state.levels.len() - ); - - let kll = KllFromProto::from_state(&kll_state); - let p50 = kll.quantile(0.50); - let p99 = kll.quantile(0.99); - println!("[KLL] Step 3/3 — p50 ≈ {p50:.1} (expect ~5000) p99 ≈ {p99:.1} (expect ~9900)"); - - let p50_ok = (p50 - 5000.0).abs() / 5000.0 < 0.05; - let p99_ok = (p99 - 9900.0).abs() / 9900.0 < 0.05; - if p50_ok && p99_ok { - println!("[KLL] PASS"); - } else { - eprintln!("[KLL] FAIL: p50={p50:.1} p99={p99:.1}"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // DDSketch - // ----------------------------------------------------------------------- - println!(); - println!("[DDSketch] Step 1/3 — Read ddsketch.pb"); - let bytes = read_file(in_dir.join("ddsketch.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode ddsketch envelope"); - - println!( - "[DDSketch] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let dd_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Ddsketch(ref s)) => s.clone(), - other => panic!("expected DDSketch sketch_state, got {other:?}"), - }; - - // `count` is not on the wire; recover it by summing the bucket counts. - let dd_total_count: u64 = dd_state.store_counts.iter().copied().sum(); - println!( - "[DDSketch] alpha={:.4} count={} buckets={} offset={}", - dd_state.alpha, - dd_total_count, - dd_state.store_counts.len(), - dd_state.store_offset - ); - - let dd = DdFromProto::from_state(&dd_state); - let p50_dd = dd.quantile(0.50); - let p99_dd = dd.quantile(0.99); - println!( - "[DDSketch] Step 3/3 — p50 ≈ {:.2} (expect ~5000) p99 ≈ {:.2} (expect ~9900)", - p50_dd.unwrap_or(f64::NAN), - p99_dd.unwrap_or(f64::NAN) - ); - - let p50_ok = p50_dd - .map(|v| (v - 5000.0).abs() / 5000.0 < 0.02) - .unwrap_or(false); - let p99_ok = p99_dd - .map(|v| (v - 9900.0).abs() / 9900.0 < 0.02) - .unwrap_or(false); - if p50_ok && p99_ok { - println!("[DDSketch] PASS"); - } else { - eprintln!("[DDSketch] FAIL: p50={p50_dd:?} p99={p99_dd:?}"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // HLL (Ertl estimator) - // ----------------------------------------------------------------------- - println!(); - println!("[HLL] Step 1/3 — Read hll.pb"); - let bytes = read_file(in_dir.join("hll.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode hll envelope"); - - println!( - "[HLL] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let hll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {other:?}"), - }; - - println!( - "[HLL] variant={} precision={} registers={}", - hll_state.variant, - hll_state.precision, - hll_state.registers.len() - ); - - let hll_card = hll_ertl_mle_estimate(&hll_state); - println!("[HLL] Step 3/3 — cardinality ≈ {hll_card} (expect ~50000)"); - if (40_000..=65_000).contains(&hll_card) { - println!("[HLL] PASS"); - } else { - eprintln!("[HLL] FAIL: cardinality {hll_card} not in [40000, 65000]"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // CountSketch - // ----------------------------------------------------------------------- - println!(); - println!("[CountSketch] Step 1/3 — Read countsketch.pb"); - let bytes = read_file(in_dir.join("countsketch.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode countsketch envelope"); - - println!( - "[CountSketch] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let cs_state = match env.sketch_state { - Some(sketch_envelope::SketchState::CountSketch(ref s)) => s.clone(), - other => panic!("expected CountSketch sketch_state, got {other:?}"), - }; - - println!( - "[CountSketch] rows={} cols={} counter_type={:?}", - cs_state.rows, - cs_state.cols, - CounterType::try_from(cs_state.counter_type) - ); - - // Query "cs:hot" — inserted 200 extra times. - // Go hash: common.Hash64([]byte("cs:hot")) = xxh3_64_seeded(seedList[0], b"cs:hot") - let cs_hot_hash = xxh3_64_seeded(SEED_0, b"cs:hot"); - let cs_est = count_sketch_query_float(&cs_state, cs_hot_hash); - println!("[CountSketch] Step 3/3 — 'cs:hot' est = {cs_est:.0} (expect ≥ 200)"); - if cs_est >= 200.0 { - println!("[CountSketch] PASS"); - } else { - eprintln!("[CountSketch] FAIL: estimate {cs_est:.0} < 200"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // CocoSketch - // ----------------------------------------------------------------------- - println!(); - println!("[CocoSketch] Step 1/3 — Read coco.pb"); - let bytes = read_file(in_dir.join("coco.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode coco envelope"); - - println!( - "[CocoSketch] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let coco_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Coco(ref s)) => s.clone(), - other => panic!("expected CocoSketch sketch_state, got {other:?}"), - }; - - println!( - "[CocoSketch] d={} width={} buckets={}", - coco_state.d, - coco_state.width, - coco_state.hashes.len() - ); - - // Query "coco:hot" — inserted with val=500. - // Go uses: hash = common.Hash64([]byte("coco:hot")) = xxh3_64_seeded(seedList[0], ...) - // DeriveIndex(hash, row, width): col = (hash >> (row * maskBitsForWidth(width))) & (width-1) - let coco_hash = xxh3_64_seeded(SEED_0, b"coco:hot"); - let coco_est = coco_estimate(&coco_state, coco_hash); - println!("[CocoSketch] Step 3/3 — 'coco:hot' est = {coco_est} (expect ≥ 500)"); - if coco_est >= 500 { - println!("[CocoSketch] PASS"); - } else { - eprintln!("[CocoSketch] FAIL: estimate {coco_est} < 500"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // ElasticSketch - // ----------------------------------------------------------------------- - println!(); - println!("[ElasticSketch] Step 1/3 — Read elastic.pb"); - let bytes = read_file(in_dir.join("elastic.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode elastic envelope"); - - println!( - "[ElasticSketch] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let elastic_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Elastic(ref s)) => s.clone(), - other => panic!("expected ElasticSketch sketch_state, got {other:?}"), - }; - - println!( - "[ElasticSketch] bucket_count={} light_rows={} light_cols={}", - elastic_state.bucket_count, - elastic_state.light.as_ref().map_or(0, |l| l.rows), - elastic_state.light.as_ref().map_or(0, |l| l.cols) - ); - - // Query "elephant" — inserted 1000 times. - // Go uses CanonicalHashSeed = seedList[5] = 0x6a09e667 - let elephant_hash = xxh3_64_seeded(SEED_5, b"elephant"); - let elephant_est = elastic_query(&elastic_state, "elephant", elephant_hash); - println!("[ElasticSketch] Step 3/3 — 'elephant' est = {elephant_est} (expect ≥ 900)"); - if elephant_est >= 900 { - println!("[ElasticSketch] PASS"); - } else { - eprintln!("[ElasticSketch] FAIL: estimate {elephant_est} < 900"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // UnivMon - // ----------------------------------------------------------------------- - println!(); - println!("[UnivMon] Step 1/3 — Read univmon.pb"); - let bytes = read_file(in_dir.join("univmon.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode univmon envelope"); - - println!( - "[UnivMon] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let um_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Univmon(ref s)) => s.clone(), - other => panic!("expected UnivMon sketch_state, got {other:?}"), - }; - - println!( - "[UnivMon] layer_size={} sketch_rows={} sketch_cols={} heap_size={}", - um_state.layer_size, um_state.sketch_rows, um_state.sketch_cols, um_state.heap_size - ); - - let um_card = univmon_cardinality(&um_state); - // Note: the g-sum heuristic typically underestimates (Go itself reports ~4250 for 10k inserts). - // We verify the Rust result matches Go's algorithm rather than the true cardinality. - println!("[UnivMon] Step 3/3 — cardinality ≈ {um_card:.0} (g-sum heuristic, Go also ~4250)"); - if (1_000.0..=15_000.0).contains(&um_card) { - println!("[UnivMon] PASS"); - } else { - eprintln!("[UnivMon] FAIL: cardinality {um_card:.0} not in [1000, 15000]"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // HydraSketch - // ----------------------------------------------------------------------- - println!(); - println!("[Hydra] Step 1/3 — Read hydra.pb"); - let bytes = read_file(in_dir.join("hydra.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode hydra envelope"); - - println!( - "[Hydra] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let hydra_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Hydra(ref s)) => s.clone(), - other => panic!("expected Hydra sketch_state, got {other:?}"), - }; - - println!( - "[Hydra] row_num={} col_num={} counter_type={} cells={}", - hydra_state.row_num, - hydra_state.col_num, - hydra_state.counter_type, - hydra_state.cells.len() - ); - - // Query "hydra:42" — inserted 51 times (1 base + 50 extra). - // Routing: subkey_hash = xxh3_64_seeded(seedList[6]=0xbb67ae85, b"hydra:42") - // Value hash: value_hash = xxh3_64_seeded(seedList[0]=0xcafe3553, b"hydra:42") - let hydra_subkey_hash = xxh3_64_seeded(SEED_6, b"hydra:42"); - let hydra_value_hash = xxh3_64_seeded(SEED_0, b"hydra:42"); - let hydra_est = hydra_query_cm(&hydra_state, hydra_subkey_hash, hydra_value_hash); - println!("[Hydra] Step 3/3 — 'hydra:42' est = {hydra_est:.0} (expect ≥ 51)"); - if hydra_est >= 51.0 { - println!("[Hydra] PASS"); - } else { - eprintln!("[Hydra] FAIL: estimate {hydra_est:.0} < 51"); - all_ok = false; - } - - // ----------------------------------------------------------------------- - // Sampled CountMinSketch (geometric skip-sampling, p=0.1) — exercises the - // sample_p envelope field + the backend ×1/p rescale at query. - // ----------------------------------------------------------------------- - let sampled_cms_path = in_dir.join("countmin_sampled.pb"); - if sampled_cms_path.exists() { - println!(); - println!("[CountMin/sampled] Step 1/3 — Read countmin_sampled.pb"); - let bytes = read_file(&sampled_cms_path); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode sampled countmin"); - - // Dual-read: 0.0/unset means 1.0; a sampled producer stamps the real p. - let p = effective_sample_p(&env); - println!("[CountMin/sampled] Step 2/3 — envelope sample_p = {p}"); - if (p - 0.1).abs() > 1e-9 { - eprintln!("[CountMin/sampled] FAIL: sample_p {p} != 0.1"); - all_ok = false; - } - - let cm_state = match env.sketch_state { - Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {other:?}"), - }; - let rows = cm_state.rows as usize; - let cols = cm_state.cols as usize; - let counts: Vec = if !cm_state.counts_float.is_empty() { - cm_state.counts_float.clone() - } else { - cm_state.counts_int.iter().map(|&v| v as f64).collect() - }; - - // Raw min-frequency point query for "item:hot". - let hot_hash = xxh3_64_seeded(SEED_0, b"item:hot"); - let bits_per_row = col_bits(cols); - let mask = (cols as u64) - 1; - let mut raw = f64::MAX; - for r in 0..rows { - let shift = (r as u64) * bits_per_row; - let c = ((hot_hash >> shift) & mask) as usize; - raw = raw.min(counts[r * cols + c]); - } - // Backend rescale: ×1/p recovers the full-stream frequency. - let rescaled = rescale_count(raw, p); - let rel_err = (rescaled - 100_000.0).abs() / 100_000.0; - println!( - "[CountMin/sampled] Step 3/3 — raw={raw:.0} rescaled(×1/p)={rescaled:.0} truth=100000 relErr={rel_err:.4}" - ); - if rel_err <= 0.05 { - println!("[CountMin/sampled] PASS"); - } else { - eprintln!("[CountMin/sampled] FAIL: rescaled rel err {rel_err:.4} > 0.05"); - all_ok = false; - } - } - - // ----------------------------------------------------------------------- - // Sampled HLL (hash-threshold sampling, p=0.1) — cardinality ×1/p rescale. - // ----------------------------------------------------------------------- - let sampled_hll_path = in_dir.join("hll_sampled.pb"); - if sampled_hll_path.exists() { - println!(); - println!("[HLL/sampled] Step 1/3 — Read hll_sampled.pb"); - let bytes = read_file(&sampled_hll_path); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode sampled hll"); - - let p = effective_sample_p(&env); - println!("[HLL/sampled] Step 2/3 — envelope sample_p = {p}"); - if (p - 0.1).abs() > 1e-9 { - eprintln!("[HLL/sampled] FAIL: sample_p {p} != 0.1"); - all_ok = false; - } - - let mut hll_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {other:?}"), - }; - // Dual-read the registers (sampling thins them → may be sparse-encoded). - let regs = - asap_sketchlib::message_pack_format::portable::hll::registers_from_state(&hll_state) - .expect("decode sampled hll registers"); - hll_state.registers = regs; - hll_state.registers_sparse = None; - - let raw = hll_ertl_mle_estimate(&hll_state) as f64; - let rescaled = rescale_count(raw, p); - let rel_err = (rescaled - 200_000.0).abs() / 200_000.0; - println!( - "[HLL/sampled] Step 3/3 — raw≈{raw:.0} rescaled(×1/p)≈{rescaled:.0} truth=200000 relErr={rel_err:.4}" - ); - if rel_err <= 0.06 { - println!("[HLL/sampled] PASS"); - } else { - eprintln!("[HLL/sampled] FAIL: rescaled rel err {rel_err:.4} > 0.06"); - all_ok = false; - } - } - - // ----------------------------------------------------------------------- - // Final summary - // ----------------------------------------------------------------------- - println!(); - println!("======================================================="); - if all_ok { - println!(" All cross-language checks PASSED."); - } else { - println!(" One or more cross-language checks FAILED."); - } - println!("======================================================="); - - assert!(all_ok, "one or more cross-language checks failed"); -} - -// --------------------------------------------------------------------------- -// Seed constants matching Go's seedList -// --------------------------------------------------------------------------- - -const SEED_0: u64 = 0xcafe3553; // seedList[0] — Hash64 / default hash -const SEED_5: u64 = 0x6a09e667; // seedList[5] — CanonicalHashSeed -const SEED_6: u64 = 0xbb67ae85; // seedList[6] — defaultHydraSeed - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -fn read_file(path: impl AsRef) -> Vec { - let p = path.as_ref(); - fs::read(p).unwrap_or_else(|e| panic!("cannot read {}: {}", p.display(), e)) -} - -struct XtestDir { - path: PathBuf, - cleanup: bool, -} - -impl XtestDir { - fn prepare() -> Option { - if let Ok(dir) = env::var("XTEST_DIR") { - return Some(Self { - path: PathBuf::from(dir), - cleanup: false, - }); - } - - let go_dir = find_go_dir()?; - let out_dir = new_xtest_temp_dir(); - if !generate_xtest_fixtures(&go_dir, &out_dir) { - let _ = fs::remove_dir_all(&out_dir); - return None; - } - Some(Self { - path: out_dir, - cleanup: true, - }) - } - - fn path(&self) -> &Path { - &self.path - } -} - -impl Drop for XtestDir { - fn drop(&mut self) { - if self.cleanup { - let _ = fs::remove_dir_all(&self.path); - } - } -} - -fn generate_xtest_fixtures(go_dir: &Path, out_dir: &Path) -> bool { - // The producer is a `go test` target (TestXtestProducer) under - // tests/cross_language/; it writes the .pb fixtures into $XTEST_DIR. - let output = match Command::new("go") - .args([ - "test", - "-run", - "TestXtestProducer", - "./tests/cross_language/", - ]) - .env("XTEST_DIR", out_dir) - .current_dir(go_dir) - .output() - { - Ok(output) => output, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return false, - Err(err) => { - panic!( - "failed to run Go xtest producer from {}: {}", - go_dir.display(), - err - ) - } - }; - - if !output.status.success() { - eprintln!( - "Go xtest producer failed with status {}.\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); - return false; - } - - true -} - -fn find_go_dir() -> Option { - if let Ok(dir) = env::var("SKETCHLIB_GO_DIR") { - let dir = PathBuf::from(dir); - if has_xtest_producer(&dir) { - return Some(dir); - } - } - - for ancestor in Path::new(env!("CARGO_MANIFEST_DIR")).ancestors() { - let candidate = ancestor.join("sketchlib-go"); - if has_xtest_producer(&candidate) { - return Some(candidate); - } - } - - None -} - -fn has_xtest_producer(dir: &Path) -> bool { - dir.join("tests/cross_language/xtest_producer_test.go") - .is_file() -} - -fn new_xtest_temp_dir() -> PathBuf { - static COUNTER: AtomicU64 = AtomicU64::new(0); - - let base = env::temp_dir(); - for _ in 0..16 { - let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); - let ts = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before unix epoch") - .as_nanos(); - let candidate = base.join(format!("sketchlib-xtest-{ts}-{nonce}")); - match fs::create_dir(&candidate) { - Ok(()) => return candidate, - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(err) => panic!("failed to create temp dir {}: {}", candidate.display(), err), - } - } - - panic!("failed to allocate temp directory for cross-language fixtures"); -} - -/// Number of bits needed to index into a column vector of given width. -fn col_bits(cols: usize) -> u64 { - let mut width = 1usize; - while width < cols { - width <<= 1; - } - if width <= 1 { - return 0; - } - width.trailing_zeros() as u64 -} - -/// maskBitsForWidth — mirrors Go's common.maskBitsForWidth. -/// Returns the number of bits needed to represent (width-1). -fn mask_bits_for_width(width: usize) -> u64 { - if width <= 1 { - return 1; - } - let mut u = width - 1; - let mut bits = 0u64; - while u > 0 { - bits += 1; - u >>= 1; - } - bits -} - -/// XXH3-64 with explicit seed, matching Go's `hash64_seeded(seed, key)`. -fn xxh3_64_seeded(seed: u64, data: &[u8]) -> u64 { - XxHash3_64::oneshot_with_seed(seed, data) -} - -// --------------------------------------------------------------------------- -// HLL Ertl estimator (Ertl 2017) -// --------------------------------------------------------------------------- -// Mirrors Go's HyperLogLog.Estimate() which uses HLLRegisterBits = 50, -// HLLPrecision = 14, HLLRegisterCount = 16384. - -fn hll_sigma(mut x: f64) -> f64 { - if x == 1.0 { - return f64::INFINITY; - } - let mut y = 1.0f64; - let mut z = x; - loop { - x *= x; - let z_prev = z; - z += x * y; - y += y; - if z_prev == z { - break; - } - } - z -} - -fn hll_tau(mut x: f64) -> f64 { - if x == 0.0 || x == 1.0 { - return 0.0; - } - let mut y = 1.0f64; - let mut z = 1.0 - x; - loop { - x = x.sqrt(); - let z_prev = z; - y *= 0.5; - z -= (1.0 - x).powi(2) * y; - if z_prev == z { - break; - } - } - z / 3.0 -} - -fn hll_ertl_mle_estimate(state: &HyperLogLogState) -> u64 { - let precision = state.precision as usize; - let register_bits = 64 - precision; // Q = 50 - let m = (1usize << precision) as f64; // 16384 - - // Build histogram C[v] for v in 0..=(register_bits+1) - let hist_len = register_bits + 2; - let mut hist = vec![0u32; hist_len]; - for &r in &state.registers { - let v = r as usize; - let capped = v.min(hist_len - 1); - hist[capped] += 1; - } - - // z = m * tau((m - C[register_bits+1]) / m) - let mut z = m * hll_tau((m - hist[register_bits + 1] as f64) / m); - - // for i from register_bits down to 1: z = (z + C[i]) * 0.5 - for i in (1..=register_bits).rev() { - z += hist[i] as f64; - z *= 0.5; - } - - // z += m * sigma(C[0] / m) - z += m * hll_sigma(hist[0] as f64 / m); - - // estimate = round(0.5 / ln(2) * m^2 / z) - (0.5 / std::f64::consts::LN_2 * m * m / z).round() as u64 -} - -// --------------------------------------------------------------------------- -// CountSketch median frequency query (float64 counters, packed hash) -// --------------------------------------------------------------------------- -// Mirrors Go's CountSketch.fastPacked64PosAndSign + ComputeMedianInlineF64. -// hash = xxh3_64_seeded(SEED_0, key) — Go's Hash64 - -fn count_sketch_query_float(state: &CountSketchState, hash: u64) -> f64 { - let rows = state.rows as usize; - let cols = state.cols as usize; - let bits_per_row = col_bits(cols); - let mask = (cols as u64) - 1; - // Read whichever counter field the producer populated (int vs float), - // discriminated by counter_type — same as CountMinSketch. - let counts: Vec = if !state.counts_float.is_empty() { - state.counts_float.clone() - } else { - state.counts_int.iter().map(|&v| v as f64).collect() - }; - let counts = &counts; - - let mut estimates: Vec = Vec::with_capacity(rows); - for r in 0..rows { - let shift = (r as u64) * bits_per_row; - let col = ((hash >> shift) & mask) as usize; - // sign: bit (63-r) of hash → 1 means +1, 0 means −1 - let sign_bit = (hash >> (63 - r)) & 1; - let sign = if sign_bit == 1 { 1.0f64 } else { -1.0f64 }; - estimates.push(counts[r * cols + col] * sign); - } - median_f64(&mut estimates) -} - -fn median_f64(v: &mut [f64]) -> f64 { - if v.is_empty() { - return 0.0; - } - v.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let n = v.len(); - if n % 2 == 1 { - v[n / 2] - } else { - (v[n / 2 - 1] + v[n / 2]) / 2.0 - } -} - -// --------------------------------------------------------------------------- -// CocoSketch estimate -// --------------------------------------------------------------------------- -// Mirrors Go's CocoSketch.EstimateHash: -// for row i: col = DeriveIndex(hash, i, width); if b.HasKey && b.Hash==hash: total += b.Val -// DeriveIndex(hash, row, width) = (hash >> (row * maskBitsForWidth(width))) & (width-1) - -fn coco_estimate(state: &CocoSketchState, hash: u64) -> u64 { - let d = state.d as usize; - let width = state.width as usize; - let mbw = mask_bits_for_width(width); - let mask = (width as u64) - 1; - let mut total = 0u64; - - for i in 0..d { - let shift = (i as u64) * mbw; - let col = ((hash >> shift) & mask) as usize; - let idx = i * width + col; - if state.has_keys[idx] && state.hashes[idx] == hash { - total += state.vals[idx]; - } - } - total -} - -// --------------------------------------------------------------------------- -// ElasticSketch query -// --------------------------------------------------------------------------- -// Mirrors Go's ElasticSketch.queryLocked: -// hash = HashIt(CanonicalHashSeed, []byte(id)) → SEED_5 -// idx = hash % bucket_count -// if flow_ids[idx] == id: if !eviction: return vote_pos[idx] -// else: return vote_pos[idx] + light_estimate -// else: return light_estimate -// Light layer: rows=5, cols=2048, bits=11, mask=2047 -// col_r = (hash >> (r * 11)) & 2047; min across rows - -fn elastic_query(state: &ElasticState, id: &str, hash: u64) -> i64 { - let n = state.bucket_count as usize; - let idx = (hash % n as u64) as usize; - - let heavy_match = idx < state.flow_ids.len() && state.flow_ids[idx] == id; - - if heavy_match { - let vpos = state.vote_pos.get(idx).copied().unwrap_or(0) as i64; - let evicted = state.evictions.get(idx).copied().unwrap_or(false); - if !evicted { - return vpos; - } - return vpos + elastic_light_min(state, hash); - } - elastic_light_min(state, hash) -} - -fn elastic_light_min(state: &ElasticState, hash: u64) -> i64 { - let light = match &state.light { - Some(l) => l, - None => return 0, - }; - let rows = light.rows as usize; - let cols = light.cols as usize; - let bits = col_bits(cols); // trailing zeros of cols = 11 for 2048 - let mask = (cols as u64) - 1; - let counts = &light.counts_float; - - let mut min_val = f64::MAX; - for r in 0..rows { - let shift = (r as u64) * bits; - let col = ((hash >> shift) & mask) as usize; - let v = counts[r * cols + col]; - if v < min_val { - min_val = v; - } - } - if min_val == f64::MAX { - 0 - } else { - min_val as i64 - } -} - -// --------------------------------------------------------------------------- -// UnivMon cardinality (g-sum heuristic) -// --------------------------------------------------------------------------- -// Mirrors Go's UnivSketch.calcGSumHeuristic(g=1, isCard=true): -// Y[L-1] = count of heap items at top layer with count > threshold -// for i from L-2 down to 0: -// tmp = Σ coe*1 for items with count > threshold -// coe = 1 - 2 * ((Hash64(key) >> (i+1)) & 1) -// Y[i] = 2*Y[i+1] + tmp -// return Y[0] -// l2_val = sqrt(median_of_first_3(layer.sketch.l2)) - -fn univmon_cardinality(state: &UnivMonState) -> f64 { - let nlayers = state.layers.len(); - if nlayers == 0 { - return 0.0; - } - - let mut y = vec![0.0f64; nlayers]; - - // Top layer - let top = &state.layers[nlayers - 1]; - let l2_val = cs_l2_from_state(top.sketch.as_ref()); - let threshold = (l2_val * 0.01) as i64; - let mut tmp = 0.0f64; - if let Some(heap) = &top.heap { - for entry in &heap.entries { - if entry.count as i64 > threshold { - tmp += 1.0; - } - } - } - y[nlayers - 1] = tmp; - - // Lower layers from L-2 down to 0 - for i in (0..nlayers - 1).rev() { - tmp = 0.0; - let layer = &state.layers[i]; - let l2_val = cs_l2_from_state(layer.sketch.as_ref()); - let threshold = (l2_val * 0.01) as i64; - - if let Some(heap) = &layer.heap { - for entry in &heap.entries { - if entry.count as i64 > threshold { - let h = xxh3_64_seeded(SEED_0, entry.key.as_bytes()); - let bit = (h >> (i + 1)) & 1; - let coe = 1.0 - 2.0 * bit as f64; - tmp += coe; - } - } - } - y[i] = 2.0 * y[i + 1] + tmp; - } - - y[0] -} - -/// cs_l2_from_state mirrors Go's CountSketchUniv.cs_l2(): -/// f2_value = MedianOfThree(l2[0], l2[1], l2[2]) -/// return sqrt(f2_value) -/// The portable l2 values are raw int64 cast to float64. -fn cs_l2_from_state(cs: Option<&CountSketchState>) -> f64 { - let cs = match cs { - Some(s) => s, - None => return 0.0, - }; - let l2 = &cs.l2; - if l2.len() < 3 { - return 0.0; - } - let med = median_of_three_f64(l2[0], l2[1], l2[2]); - med.abs().sqrt() -} - -fn median_of_three_f64(a: f64, b: f64, c: f64) -> f64 { - if a <= b { - if b <= c { - b - } else if a <= c { - c - } else { - a - } - } else if a <= c { - a - } else if b <= c { - c - } else { - b - } -} - -// --------------------------------------------------------------------------- -// HydraSketch CountMinSketch frequency query -// --------------------------------------------------------------------------- -// Routing (mirrors Go's fillPositionsFromHash with default seeds): -// seedCM1 = 0x1111111111111111, seedCM2 = 0x2222222222222222 -// x = subkey_hash ^ seedCM1; y = subkey_hash ^ seedCM2 -// for r in 0..D: xorshift both; pos[r] = (x ^ (y<<1)) % W -// For each row: query CM cell at cells[r*W + pos[r]] with value_hash. -// CM query: min across rows of count at col=(value_hash>>(r*bits))&mask. -// Final result: median of per-Hydra-row CM estimates. - -const HYDRA_SEED_CM1: u64 = 0x1111111111111111; -const HYDRA_SEED_CM2: u64 = 0x2222222222222222; - -fn xorshift64(x: &mut u64) { - *x ^= *x << 13; - *x ^= *x >> 7; - *x ^= *x << 17; -} - -fn hydra_fill_positions(subkey_hash: u64, d: usize, w: usize) -> Vec { - let mut x = subkey_hash ^ HYDRA_SEED_CM1; - let mut y = subkey_hash ^ HYDRA_SEED_CM2; - if x == 0 { - x = HYDRA_SEED_CM1; - } - if y == 0 { - y = HYDRA_SEED_CM2 | 1; - } - - let mut pos = Vec::with_capacity(d); - for _ in 0..d { - xorshift64(&mut x); - xorshift64(&mut y); - pos.push(((x ^ (y << 1)) % w as u64) as usize); - } - pos -} - -fn hydra_query_cm(state: &HydraState, subkey_hash: u64, value_hash: u64) -> f64 { - let d = state.row_num as usize; - let w = state.col_num as usize; - let cells = &state.cells; - - let pos = hydra_fill_positions(subkey_hash, d, w); - - let mut estimates = Vec::with_capacity(d); - for (r, &p) in pos.iter().enumerate().take(d) { - let cell_idx = r * w + p; - if cell_idx >= cells.len() { - estimates.push(0.0f64); - continue; - } - let cell = &cells[cell_idx]; - let freq = match &cell.sketch { - Some(hydra_cell::Sketch::CountMin(cm)) => cm_query_min(cm, value_hash), - _ => 0.0, - }; - estimates.push(freq); - } - - median_f64(&mut estimates) -} - -/// CountMinSketch min-frequency query with packed hash. -fn cm_query_min(cm: &CountMinState, hash: u64) -> f64 { - let rows = cm.rows as usize; - let cols = cm.cols as usize; - let bits_per_row = col_bits(cols); - let mask = (cols as u64) - 1; - let counts: Vec = if !cm.counts_float.is_empty() { - cm.counts_float.clone() - } else { - cm.counts_int.iter().map(|&v| v as f64).collect() - }; - let counts = &counts; - - let mut min_val = f64::MAX; - for r in 0..rows { - let shift = (r as u64) * bits_per_row; - let col = ((hash >> shift) & mask) as usize; - let v = counts[r * cols + col]; - if v < min_val { - min_val = v; - } - } - if min_val == f64::MAX { 0.0 } else { min_val } -} - -// --------------------------------------------------------------------------- -// Minimal KLL deserialization + CDF query -// --------------------------------------------------------------------------- - -struct KllFromProto { - items: Vec, - levels: Vec, - num_levels: usize, -} - -impl KllFromProto { - fn from_state(s: &KllState) -> Self { - // Dual-read: prefer the value-offset fixed-point representation when - // residuals are present, else fall back to raw f64 items[] (v1). - let items: Vec = if !s.residuals.is_empty() { - asap_sketchlib::message_pack_format::portable::kll::decode_value_offset( - s.offset, - s.value_scale, - &s.residuals, - ) - } else { - s.items.clone() - }; - let levels: Vec = s.levels.iter().map(|&v| v as usize).collect(); - let num_levels = s.num_levels as usize; - Self { - items, - levels, - num_levels, - } - } - - fn weighted_samples(&self) -> Vec<(f64, u64)> { - let mut out = Vec::with_capacity(self.items.len()); - for h in 0..self.num_levels { - let weight: u64 = 1 << h; - let idx = self.num_levels - 1 - h; - if idx + 1 >= self.levels.len() { - continue; - } - let start = self.levels[idx]; - let end = self.levels[idx + 1]; - for &v in &self.items[start..end] { - out.push((v, weight)); - } - } - out - } - - fn quantile(&self, q: f64) -> f64 { - let mut samples = self.weighted_samples(); - if samples.is_empty() { - return 0.0; - } - samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - let total: u64 = samples.iter().map(|(_, w)| w).sum(); - let target = (q * total as f64).ceil() as u64; - let mut acc = 0u64; - for (v, w) in &samples { - acc += w; - if acc >= target { - return *v; - } - } - samples.last().unwrap().0 - } -} - -// --------------------------------------------------------------------------- -// Minimal DDSketch deserialization + quantile query -// --------------------------------------------------------------------------- - -struct DdFromProto { - gamma: f64, - #[allow(dead_code)] - inv_log_gamma: f64, - positive_store_counts: Vec, - positive_store_offset: i32, - // Total count is recovered by summing the bucket array; the - // DataPoint-level count/min/max scalars are not on the wire. - count: u64, -} - -impl DdFromProto { - fn from_state(s: &DdSketchState) -> Self { - let alpha = s.alpha; - let gamma = (1.0 + alpha) / (1.0 - alpha); - let log_gamma = gamma.ln(); - let inv_log_gamma = 1.0 / log_gamma; - Self { - gamma, - inv_log_gamma, - positive_store_counts: s.store_counts.clone(), - positive_store_offset: s.store_offset, - count: s.store_counts.iter().copied().sum(), - } - } - - fn bin_representative(&self, k: i32) -> f64 { - self.gamma.powf(k as f64 + 0.5) - } - - /// Representative value of the lowest non-empty bucket — the - /// α-bounded estimate of the minimum now that the `min` scalar is - /// gone from the wire. - fn min_estimate(&self) -> Option { - self.positive_store_counts - .iter() - .position(|&c| c > 0) - .map(|i| self.bin_representative(self.positive_store_offset + i as i32)) - } - - /// Representative value of the highest non-empty bucket — the - /// α-bounded estimate of the maximum. - fn max_estimate(&self) -> Option { - self.positive_store_counts - .iter() - .rposition(|&c| c > 0) - .map(|i| self.bin_representative(self.positive_store_offset + i as i32)) - } - - fn quantile(&self, q: f64) -> Option { - if self.count == 0 { - return None; - } - if q <= 0.0 { - return self.min_estimate(); - } - if q >= 1.0 { - return self.max_estimate(); - } - let rank = (q * self.count as f64).ceil() as u64; - let mut seen = 0u64; - for (i, &c) in self.positive_store_counts.iter().enumerate() { - if c == 0 { - continue; - } - seen += c; - if seen >= rank { - let bin = self.positive_store_offset + i as i32; - return Some(self.bin_representative(bin)); - } - } - self.max_estimate() - } -} diff --git a/tools/gen-proto/Cargo.lock b/tools/gen-proto/Cargo.lock deleted file mode 100644 index 13170e87..00000000 --- a/tools/gen-proto/Cargo.lock +++ /dev/null @@ -1,646 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" - -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "gen-proto" -version = "0.0.0" -dependencies = [ - "prost-build", - "protoc-bin-vendored", -] - -[[package]] -name = "getrandom" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", - "wasip3", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost", -] - -[[package]] -name = "protoc-bin-vendored" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" -dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-s390_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-aarch_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", -] - -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" - -[[package]] -name = "protoc-bin-vendored-linux-s390_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" - -[[package]] -name = "protoc-bin-vendored-macos-aarch_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" - -[[package]] -name = "quote" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.149" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom", - "once_cell", - "rustix", - "windows-sys", -] - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/tools/gen-proto/Cargo.toml b/tools/gen-proto/Cargo.toml deleted file mode 100644 index 0cc4ec49..00000000 --- a/tools/gen-proto/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "gen-proto" -version = "0.0.0" -edition = "2024" -publish = false -description = "Internal maintainer tool that regenerates src/proto/generated/ from proto/*.proto." - -[dependencies] -prost-build = "0.13" -protoc-bin-vendored = "3" - -# Standalone workspace so this crate is fully isolated from the parent -# `asap_sketchlib` package. It must not appear in the published crate's -# build graph or Cargo.lock. -[workspace] diff --git a/tools/gen-proto/src/main.rs b/tools/gen-proto/src/main.rs deleted file mode 100644 index 1de6ad15..00000000 --- a/tools/gen-proto/src/main.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Maintainer-only tool that regenerates the vendored prost output under -//! `src/proto/generated/` from the `.proto` files in `proto/`. -//! -//! Run from the repository root with: -//! -//! ```bash -//! cargo run --manifest-path tools/gen-proto/Cargo.toml -//! ``` -//! -//! CI enforces that the committed output matches the result of running this -//! tool (see `.github/workflows/ci.yml`). - -use std::fs; -use std::path::{Path, PathBuf}; - -// Inner attributes (`#![...]`) cannot appear inside an `include!` expansion, -// so this header is restricted to comments. Module-level attributes such as -// `#[rustfmt::skip]` and `#[allow(clippy::all)]` are applied to the wrapping -// `pub mod sketchlib { ... }` in `src/proto.rs` instead. -const HEADER: &str = "\ -// @generated by tools/gen-proto -- DO NOT EDIT. -// -// Regenerate with: -// cargo run --manifest-path tools/gen-proto/Cargo.toml -// -// Source of truth: proto/**/*.proto -"; - -const PROTO_FILES: &[&str] = &[ - "common/common.proto", - "countminsketch/countminsketch.proto", - "countsketch/countsketch.proto", - "hll/hll.proto", - "kll/kll.proto", - "ddsketch/ddsketch.proto", - "univmon/univmon.proto", - "hydra/hydra.proto", - "cocosketch/cocosketch.proto", - "elasticsketch/elasticsketch.proto", - "sketchlib.proto", -]; - -fn main() { - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let repo_root = manifest_dir - .parent() - .and_then(Path::parent) - .expect("tools/gen-proto must live two levels below the repo root"); - - let proto_dir = repo_root.join("proto"); - let out_dir = repo_root.join("src/proto/generated"); - - fs::create_dir_all(&out_dir).expect("failed to create generated proto directory"); - - let protoc = - protoc_bin_vendored::protoc_bin_path().expect("failed to locate vendored protoc binary"); - // SAFETY: single-threaded maintainer tool; setting PROTOC before invoking - // prost-build is the documented way to override the protoc binary. - unsafe { - std::env::set_var("PROTOC", protoc); - } - - let protos: Vec = PROTO_FILES.iter().map(|p| proto_dir.join(p)).collect(); - - prost_build::Config::new() - .out_dir(&out_dir) - .compile_protos(&protos, &[proto_dir.clone()]) - .expect("prost_build failed to compile proto files"); - - prepend_header_to_generated(&out_dir); - - eprintln!("Generated proto code under {}", out_dir.display()); -} - -fn prepend_header_to_generated(out_dir: &Path) { - for entry in fs::read_dir(out_dir).expect("failed to read generated directory") { - let entry = entry.expect("failed to iterate generated directory"); - let path = entry.path(); - if path.extension().and_then(|e| e.to_str()) != Some("rs") { - continue; - } - let body = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("failed to read {}: {err}", path.display())); - if body.starts_with(HEADER) { - continue; - } - let mut with_header = String::with_capacity(HEADER.len() + body.len() + 1); - with_header.push_str(HEADER); - with_header.push('\n'); - with_header.push_str(&body); - fs::write(&path, with_header) - .unwrap_or_else(|err| panic!("failed to write {}: {err}", path.display())); - } -} From 18a819b7798a72625995b5e2552414685627d949 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:24:21 +0000 Subject: [PATCH 2/4] refactor!: remove the legacy MessagePack format (#207) Delete `message_pack_format::{portable, native}`, `MessagePackCodec` and its `Error`: the wrapper types (`CountMinSketch`, `CountSketch`, `CountMinSketchWithHeap`, `CountSketchWithHeap`, `KllSketch`, `HllSketch`, `DdSketch`, `HydraKllSketch`), their deltas, heap items, sampling helpers and constants, and the `xxhash-rust` dependency. Tests that exercised only the wrappers go with them. `SetAggregator` and `SetAggregatorDelta` move to `sketches::set_aggregator` and stay re-exported at the crate root. The ASAPv1 framing (`envelope`, `kind`, `delta`, `wire_key`, `admission_rate`) moves to the private module `asapv1`; `peek_kind_id` stays at the crate root. Closes #207 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + Cargo.toml | 1 - docs/api/api_count_sketch.md | 4 +- docs/api/api_ddsketch.md | 13 +- docs/asapv1_wire_format.md | 17 +- docs/ddsketch_signed_values.md | 7 +- docs/e2e_testing_harness.md | 19 +- docs/index.md | 2 +- docs/library_map.md | 10 +- docs/message_pack_format.md | 27 +- docs/tests.md | 302 +--- examples/accuracy_probe.rs | 117 +- .../admission_rate.rs | 2 +- src/{message_pack_format => asapv1}/delta.rs | 0 .../envelope.rs | 9 +- src/{message_pack_format => asapv1}/kind.rs | 0 src/asapv1/mod.rs | 13 + .../wire_key.rs | 2 +- src/common/input.rs | 2 +- src/lib.rs | 12 +- src/message_pack_format/codec.rs | 21 - src/message_pack_format/error.rs | 45 - src/message_pack_format/mod.rs | 31 - .../native/countminsketch.rs | 27 - src/message_pack_format/native/countsketch.rs | 30 - .../native/countsketch_l2_norm.rs | 15 - src/message_pack_format/native/ddsketch.rs | 17 - src/message_pack_format/native/hll.rs | 27 - src/message_pack_format/native/kll.rs | 20 - src/message_pack_format/native/kll_dynamic.rs | 21 - src/message_pack_format/native/kmv.rs | 19 - src/message_pack_format/native/mod.rs | 18 - .../portable/countminsketch.rs | 1000 ------------ .../portable/countminsketch_topk.rs | 447 ------ .../portable/countsketch.rs | 972 ------------ .../portable/countsketch_topk.rs | 471 ------ src/message_pack_format/portable/ddsketch.rs | 1364 ----------------- .../portable/delta_set_aggregator.rs | 66 - src/message_pack_format/portable/hll.rs | 1071 ------------- src/message_pack_format/portable/hydra_kll.rs | 292 ---- src/message_pack_format/portable/kll.rs | 340 ---- src/message_pack_format/portable/mod.rs | 20 - src/message_pack_format/portable/sampling.rs | 133 -- src/sketch_framework/eh/wire.rs | 6 +- src/sketch_framework/eh_bucket_sketch.rs | 2 +- src/sketch_framework/eh_bucket_sketch/wire.rs | 8 +- src/sketch_framework/hydra/wire.rs | 20 +- src/sketch_framework/univmon/wire.rs | 8 +- src/sketch_framework/univmon_q/wire.rs | 6 +- src/sketch_framework/univmon_two_tier/wire.rs | 6 +- src/sketches/bloom/wire.rs | 6 +- src/sketches/coco/wire.rs | 6 +- src/sketches/countminsketch.rs | 2 +- src/sketches/countminsketch/delta.rs | 14 +- src/sketches/countminsketch/wire.rs | 12 +- src/sketches/countminsketch_heap/delta.rs | 10 +- src/sketches/countminsketch_heap/heap_wire.rs | 6 +- src/sketches/countminsketch_heap/wire.rs | 8 +- .../summary_wire.rs | 6 +- .../countminsketch_space_saving/wire.rs | 4 +- src/sketches/countsketch.rs | 2 +- src/sketches/countsketch/delta.rs | 14 +- src/sketches/countsketch/wire.rs | 12 +- src/sketches/countsketch_heap/delta.rs | 10 +- src/sketches/countsketch_heap/wire.rs | 8 +- src/sketches/countsketch_l2_norm/wire.rs | 6 +- src/sketches/countsketch_space_saving/wire.rs | 4 +- src/sketches/ddsketch.rs | 8 +- src/sketches/ddsketch/delta.rs | 10 +- src/sketches/ddsketch/wire.rs | 10 +- src/sketches/elastic/wire.rs | 8 +- src/sketches/hll.rs | 2 +- src/sketches/hll/delta.rs | 6 +- src/sketches/hll/wire.rs | 135 +- src/sketches/kll.rs | 23 +- src/sketches/kll/wire.rs | 6 +- src/sketches/kll_dynamic/wire.rs | 8 +- src/sketches/kmv/wire.rs | 6 +- src/sketches/mod.rs | 4 + .../portable => sketches}/set_aggregator.rs | 74 +- src/sketches/set_aggregator/delta.rs | 11 + .../set_aggregator/wire.rs | 9 +- src/sketches/space_saving/wire.rs | 8 +- src/sketches/uniform/wire.rs | 8 +- tests/PBT_COVERAGE.md | 48 +- tests/README.md | 2 +- tests/asapv1_golden.rs | 69 +- tests/bug_verification.rs | 46 +- tests/common/conformance.rs | 4 +- tests/common/specs.rs | 24 +- tests/conformance_kit.rs | 53 - tests/ddsketch_signed.rs | 60 +- tests/e2e/cardinality.rs | 79 +- tests/e2e/composition.rs | 237 +-- tests/e2e/frequency.rs | 43 - tests/e2e/quantiles.rs | 393 +---- tests/msgpack_compat.rs | 259 ---- tests/pbt/count_min.rs | 375 +---- tests/pbt/count_sketch.rs | 106 +- tests/pbt/ddsketch.rs | 127 +- tests/pbt/hll.rs | 108 +- tests/pbt/hydra.rs | 105 -- tests/pbt/kll.rs | 39 +- tests/pbt/set_aggregator.rs | 11 +- 104 files changed, 436 insertions(+), 9307 deletions(-) rename src/{message_pack_format => asapv1}/admission_rate.rs (99%) rename src/{message_pack_format => asapv1}/delta.rs (100%) rename src/{message_pack_format => asapv1}/envelope.rs (97%) rename src/{message_pack_format => asapv1}/kind.rs (100%) create mode 100644 src/asapv1/mod.rs rename src/{message_pack_format => asapv1}/wire_key.rs (99%) delete mode 100644 src/message_pack_format/codec.rs delete mode 100644 src/message_pack_format/error.rs delete mode 100644 src/message_pack_format/mod.rs delete mode 100644 src/message_pack_format/native/countminsketch.rs delete mode 100644 src/message_pack_format/native/countsketch.rs delete mode 100644 src/message_pack_format/native/countsketch_l2_norm.rs delete mode 100644 src/message_pack_format/native/ddsketch.rs delete mode 100644 src/message_pack_format/native/hll.rs delete mode 100644 src/message_pack_format/native/kll.rs delete mode 100644 src/message_pack_format/native/kll_dynamic.rs delete mode 100644 src/message_pack_format/native/kmv.rs delete mode 100644 src/message_pack_format/native/mod.rs delete mode 100644 src/message_pack_format/portable/countminsketch.rs delete mode 100644 src/message_pack_format/portable/countminsketch_topk.rs delete mode 100644 src/message_pack_format/portable/countsketch.rs delete mode 100644 src/message_pack_format/portable/countsketch_topk.rs delete mode 100644 src/message_pack_format/portable/ddsketch.rs delete mode 100644 src/message_pack_format/portable/delta_set_aggregator.rs delete mode 100644 src/message_pack_format/portable/hll.rs delete mode 100644 src/message_pack_format/portable/hydra_kll.rs delete mode 100644 src/message_pack_format/portable/kll.rs delete mode 100644 src/message_pack_format/portable/mod.rs delete mode 100644 src/message_pack_format/portable/sampling.rs rename src/{message_pack_format/portable => sketches}/set_aggregator.rs (51%) create mode 100644 src/sketches/set_aggregator/delta.rs rename src/{message_pack_format/portable => sketches}/set_aggregator/wire.rs (97%) delete mode 100644 tests/msgpack_compat.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ed568e..ee5bf2dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -334,6 +334,13 @@ signals a backwards-compatible change. - **Protobuf** (breaking). Module `proto`, the `.proto` sources, `tools/gen-proto` and the `prost`/`bytes` dependencies; with them the KLL value-offset sample encoding. +- **Legacy MessagePack** (breaking). Module `message_pack_format` with + `portable`, `native` and `MessagePackCodec`: the wrapper types + `CountMinSketch`, `CountSketch`, `CountMinSketchWithHeap`, + `CountSketchWithHeap`, `KllSketch`, `HllSketch`, `DdSketch`, + `HydraKllSketch`, their deltas and helpers, and the `xxhash-rust` + dependency. `SetAggregator` and `SetAggregatorDelta` move to + `sketches::set_aggregator`; the ASAPv1 framing is private. - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); `PRECOMPUTED_SAMPLE_RATE_1PERCENT` and module `common::precompute_sample2`; module `common::hashspec` (`HashSpec`, `derive_index`, `derive_sign`, diff --git a/Cargo.toml b/Cargo.toml index fa89c4e1..dbafd6ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ serde_bytes = "0.11" rmp-serde = "1.3.0" twox-hash = "2.1.2" smallvec = "1.13.2" -xxhash-rust = { version = "0.8", features = ["xxh32"] } core_affinity = { version = "0.8", optional = true } crossbeam-channel = { version = "0.5", optional = true } diff --git a/docs/api/api_count_sketch.md b/docs/api/api_count_sketch.md index c7e992d6..63a7717d 100644 --- a/docs/api/api_count_sketch.md +++ b/docs/api/api_count_sketch.md @@ -111,9 +111,7 @@ as the state serialization: the changed cells as strictly ascending row-major indices `row * cols + col`, below `rows * cols`, and the non-zero signed value added to each. `new` and the decoder refuse anything else. `merge_delta` adds the delta to the counters; a delta of other dimensions or admission rate, or an -integer add that overflows, fails and leaves the sketch unchanged. The crate -root's `CountSketchDelta` is the legacy portable type; this one is -`asap_sketchlib::sketches::CountSketchDelta`. +integer add that overflows, fails and leaves the sketch unchanged. ## Admission rate diff --git a/docs/api/api_ddsketch.md b/docs/api/api_ddsketch.md index 24634e1d..a5c816b2 100644 --- a/docs/api/api_ddsketch.md +++ b/docs/api/api_ddsketch.md @@ -139,17 +139,8 @@ assert!(p50 >= 1.0); `rank = ceil(q * n)` (1-based), so it answers the order statistic `sorted[ceil(q*n) - 1]`. `q <= 0` and `q >= 1` short-circuit to the exactly retained minimum and maximum rather than to a bucket representative, so the -endpoints carry zero error. - -This differs from the portable wire twin -`message_pack_format::portable::ddsketch::DdSketch::quantile`, which uses the -lower-quantile convention `floor(q * (n - 1))` of the DDSketch paper and of the -Go reference implementation, and which retains no min/max scalars — its -endpoints are ordinary bucket representatives, accurate to `alpha` like any -other rank. The two agree at most `q` and diverge at small `n` or ragged `q` -(at `n = 3, q = 0.4` this type answers `sorted[1]` and the portable one -`sorted[0]`). Both conventions are deliberate and are pinned by -`ddsketch_core_and_portable_answer_different_order_statistics`. +endpoints carry zero error. The convention is pinned by +`ddsketch_answers_the_ceil_nearest_rank_order_statistic`. ## Caveats diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 7a707dd8..5f81a26c 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -26,7 +26,7 @@ If the doc feels long, these sections carry the key points: ## Status -- **Implemented (Rust).** Every kind_id the Section 1 registry marks *implemented* serializes through the shared `message_pack_format::envelope` module per this spec: HLL (three estimators) with the Classic and Ertl-MLE deltas (`HyperLogLogDelta`), Count-Min with its delta (`CountMinSketchDelta`), CountMinSketchWithHeap with its delta (`CountMinSketchWithHeapDelta`), Count Sketch with its delta (`sketches::CountSketchDelta`), CountSketchWithHeap with its delta (`CountSketchWithHeapDelta`), DDSketch with its delta (`DdSketchDelta`), KLL (compact + dynamic), Hydra (five counter variants), Elastic, Coco, UniformSampling, KMV, UnivMon, Two-tier UnivMon, ExponentialHistogram, EhBucketSketch, Bloom, Space-Saving, CountSketchWithL2Norm, UnivMon-Q, CountMinSketchWithSpaceSaving, CountSketchWithSpaceSaving, and SetAggregator with its delta (`SetAggregatorDelta`). +- **Implemented (Rust).** Every kind_id the Section 1 registry marks *implemented* serializes through the shared `asapv1::envelope` module per this spec: HLL (three estimators) with the Classic and Ertl-MLE deltas (`HyperLogLogDelta`), Count-Min with its delta (`CountMinSketchDelta`), CountMinSketchWithHeap with its delta (`CountMinSketchWithHeapDelta`), Count Sketch with its delta (`sketches::CountSketchDelta`), CountSketchWithHeap with its delta (`CountSketchWithHeapDelta`), DDSketch with its delta (`DdSketchDelta`), KLL (compact + dynamic), Hydra (five counter variants), Elastic, Coco, UniformSampling, KMV, UnivMon, Two-tier UnivMon, ExponentialHistogram, EhBucketSketch, Bloom, Space-Saving, CountSketchWithL2Norm, UnivMon-Q, CountMinSketchWithSpaceSaving, CountSketchWithSpaceSaving, and SetAggregator with its delta (`SetAggregatorDelta`). - **Self-describing.** The hash-spec metadata is derived from the hasher's `HashProfile` (read live, never hardcoded), so the bytes truthfully describe how a sketch was hashed; custom hash profiles are supported (Section 2). - **Byte-level encoding** is pinned in Section 4; the resolved decisions are summarized at the end. - **`sketchlib-go`** is aligned separately (see Cross-language contract). @@ -249,7 +249,7 @@ This registry is the master list of algorithms still to design payloads for, and 7. Hand the payload to the per-sketch payload decoder. 8. Fail **closed** on any inconsistency; never merge or query a sketch whose hash spec did not validate. -> Implementation note: the shared envelope module (`src/message_pack_format/envelope.rs`) owns rules 1-3 and the byte framing (`encode` / `split`); it is sketch-agnostic and does not know the registry. +> Implementation note: the shared envelope module (`src/asapv1/envelope.rs`) owns rules 1-3 and the byte framing (`encode` / `split`); it is sketch-agnostic and does not know the registry. > Rule 4 (and metadata/kind_id validation) happens in each sketch's decoder, which checks the `kind_id` against the ones it owns. > `peek_kind_id` checks rules 2-3 and the length checks of rule 1, not that each block is filled exactly, then returns the `kind_id` bytes; otherwise it fails. @@ -730,8 +730,6 @@ Wire counter types are the base Count-Min's: **`"i32"`, `"i64"` and `"f64"`**. ` **A producer-side constraint: a `NaN` key.** `TopKHeap` finds a resident by digest and then compares with `PartialEq`, and `NaN` never equals itself, so offering the same `NaN` key twice seats **two** entries rather than rescoring one. That state is reachable in memory and has **no encoding**: the encode-side duplicate check reads the two keys' bits, sees one key twice, and fails with the same complaint the decoder would make. Nothing repairs it on either door — a producer that inserts float keys must keep `NaN` out of a heap it intends to serialize. -**Not this: `portable/countminsketch_topk.rs`.** Despite the name, that module emits no ASAPv1 bytes. It is a legacy Go-interop struct with its own msgpack shape: an `f64` matrix rounded into the `i64` one on the way in, and heap keys coerced to `String` — a `OwnedDataInput` that is not already a `String` goes through a Rust debug format, which is lossy and reversible by nothing. `0x03 0x00` shares none of that: its matrix travels at the metadata's `counter_type` and its keys travel as the exact `OwnedDataInput` variant `key_type` names. - ### 3.8: DDSketch payload (`0x05 0x00`) `DdSketch` maps the magnitude of a value `v` to the bucket index `floor(ln(|v|) / ln(gamma))`. Every payload carries all eight fields: @@ -774,8 +772,6 @@ A Hydra is a `rows x cols` grid of counters over a fixed set of named key column **Counters are inlined, not nested.** A cell's raw state goes straight into Hydra's positional array in the shape that counter's own section fixes; no cell carries an envelope, a magic or a metadata map of its own. -**Not this: `portable/hydra_kll.rs`.** `HydraKllSketch` is a legacy Go-interop type, not `0x07 0x00`, and it is the counter-example this rule exists to rule out: every cell of its matrix carries a **full ASAPv1 KLL envelope** — magic, version, kind_id, both length prefixes and a complete metadata map — plus `k` repeated once per cell beside it, in a msgpack shape of its own with no ASAPv1 envelope around the whole. That is exactly what Q-NEST forbids, and `0x07 0x00` does none of it: one envelope for the grid, `cell_k` / `cell_m` / `cell_item_type` carried once in its metadata, and each cell just the `[levels, items, coin]` array §3.3 fixes. - **`0x07 0x01` Count-Min counter, `0x07 0x02` Count Sketch counter** — the fixed-size matrix counters tile one array: | Pos | Field | Type | Notes | @@ -1583,23 +1579,20 @@ It does not convert the mode (Regular to/from Fast); that would need re-insertin ## Cross-language contract -Direction: **custom per-sketch payload replaces the `portable` types, and `sketchlib-go` mirrors each payload.** +Direction: **each sketch has a custom payload, and `sketchlib-go` mirrors each payload.** Good direction (more compact, higher fidelity, less Rust-internal duplication), but it moves the contract from shared code to discipline. To keep it safe: 1. **This spec**: byte-level, language-neutral, per sketch. -2. **Golden byte-vector fixtures** in one shared repo, [`sketchlib-golden-bytes`](https://github.com/ProjectASAP/sketchlib-golden-bytes), which each implementation mounts at `asapv1_golden/`; both languages decode and re-encode them byte-identically. These replace the `portable`-as-oracle round-trip test. +2. **Golden byte-vector fixtures** in one shared repo, [`sketchlib-golden-bytes`](https://github.com/ProjectASAP/sketchlib-golden-bytes), which each implementation mounts at `asapv1_golden/`; both languages decode and re-encode them byte-identically. 3. **This registry**, mirrored, never independently allocated. -Every `kind_id` the registry marks *implemented* has at least one fixture; `asapv1_golden/README.md` lists them. `portable` carries its own Go goldens. +Every `kind_id` the registry marks *implemented* has at least one fixture; `asapv1_golden/README.md` lists them. **Hash profile on the Go side.** Rust derives the hash spec from a generic `HashProfile` bound on the hasher type; Go has no generic hasher type, so there is nothing to derive from. On the Go side the profile is simply **written into** the metadata on encode and **read from** it on decode. Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketch is only mergeable/queryable if its `hash_profile_id` + seeds match the profile Go is prepared to reproduce. -Sequencing: the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. -Retire `portable` once the fixtures cover the rest. - --- ## Decisions (resolved) diff --git a/docs/ddsketch_signed_values.md b/docs/ddsketch_signed_values.md index d9d0c5ff..71782c35 100644 --- a/docs/ddsketch_signed_values.md +++ b/docs/ddsketch_signed_values.md @@ -1,17 +1,16 @@ # Signed DDSketch -Both `DdSketch` (core) and `DdSketch` (portable) retain positive samples, negative samples, and zeros. The design follows [DataDog DDSketch](https://github.com/DataDog/sketches-go/blob/master/ddsketch/ddsketch.go): separate positive and negative magnitude stores, plus a zero count. Bucket indices can be negative in either store; an index's sign does not encode an observation's sign. +`DdSketch` retains positive samples, negative samples, and zeros. The design follows [DataDog DDSketch](https://github.com/DataDog/sketches-go/blob/master/ddsketch/ddsketch.go): separate positive and negative magnitude stores, plus a zero count. Bucket indices can be negative in either store; an index's sign does not encode an observation's sign. -`try_add` / `try_update` return errors for NaN, infinities, or magnitudes above the mapping's maximum. Existing `add` / `update` wrappers keep their return types and ignore those errors. Magnitudes below the mapping's minimum are counted as zero, including subnormal inputs. Such nonzero tiny observations are outside the relative-error guarantee. Original sample values are still compressed into buckets, not retained individually. +`try_add` returns errors for NaN, infinities, or magnitudes above the mapping's maximum. `add` ignores those errors. Magnitudes below the mapping's minimum are counted as zero, including subnormal inputs. Such nonzero tiny observations are outside the relative-error guarantee. Original sample values are still compressed into buckets, not retained individually. -Core preserves its existing ceil(q*n) order-statistic convention and exact ingested extrema. Portable preserves floor(q*(n-1)); `quantile_interpolated` preserves the PromQL interpolation convention. For indexable nonzero order statistics the estimate has absolute error at most `alpha * abs(value)`, for either sign. +`DdSketch` answers the ceil(q*n) order statistic and keeps exact ingested extrema. For indexable nonzero order statistics the estimate has absolute error at most `alpha * abs(value)`, for either sign. If interpolation combines endpoints x and y with weight t, the absolute error is bounded by `alpha * ((1-t)*abs(x) + t*abs(y))`, excluding zero-mapped tiny values. Opposite signs can cancel, so the interpolated result need not have relative error alpha. Dividing quantiles additionally requires a finite nonzero true denominator and valid relative-error bounds for both operands. This change does not establish those planner preconditions. ## Wire and downstream upgrade - Native ASAPv1: every state is metadata version 1 with the eight-field payload, negative counts, negative offset and zero count included. `DdSketchDelta` (`0x05 0x80`) carries both stores and a zero-count increment. -- Portable MessagePack: positive-only snapshots keep their three fields; signed snapshots append the same three fields. Positive-only deltas keep their two arrays; signed deltas append negative indices, negative counts, and zero-count increment. New readers accept both layouts. - Octo: `DdSketchBucketDelta` now requires a `sign: DdSign` discriminator. Existing positive delta constructors should use `DdSign::Positive`. Worker residual keys are `(DdSign, i32)`. Both promotion and flush preserve all three categories; partial counts still need flushing before complete queries. Go implementations of the ASAP wire formats and downstream dependency pins are not changed by this Rust library patch. diff --git a/docs/e2e_testing_harness.md b/docs/e2e_testing_harness.md index b42e1e27..818e57fd 100644 --- a/docs/e2e_testing_harness.md +++ b/docs/e2e_testing_harness.md @@ -72,7 +72,7 @@ tests/ ├── e2e_matrix_instances.rs # every (storage, path) instance of the matrix families ├── e2e_numeric_types.rs # every NumericalValue type through Kll / DdSketch ├── e2e_windows.rs # every EhBucketSketch variant + TumblingWindow payloads -├── e2e_composition.rs # HashSketchEnsemble, NitroBatch, UnivMonQ config, portable facade +├── e2e_composition.rs # HashSketchEnsemble, NitroBatch, UnivMonQ config ├── e2e_frameworks.rs # Hydra and UnivMon composition ├── e2e_octo.rs # …the OctoSketch promotion protocol ├── e2e_heavy_hitters.rs # …Space-Saving, CocoSketch and ElasticSketch @@ -141,11 +141,9 @@ Keeping these apart is deliberate. There is no shared `QuantileSpec { rank_tol }` that KLL and DDSketch both use, because they do not promise the same thing: a correct KLL can return a value 100× off on a heavy-tailed stream and still be within its rank guarantee, and a correct -DDSketch has no rank guarantee at all. For the same reason the two DDSketch -implementations do not share a truth helper: `DdSketch::get_value_at_quantile` -answers `sorted[ceil(q·n) − 1]` while the portable `DdSketch::quantile` answers -`sorted[floor(q·(n−1))]`, so `DdRankConvention` carries the choice and each is -scored on the question it actually answers. +DDSketch has no rank guarantee at all. `DdSketch::get_value_at_quantile` +answers `sorted[ceil(q·n) − 1]`, and `DdRankConvention` names that order +statistic for the truth helper. Two distinctions are easy to lose: @@ -190,8 +188,7 @@ suite with the matching spec from Layer 2. `conformance_kit.rs` wires established sketches through the kit as reference adapters. The `e2e_*.rs` suites add depth the kit deliberately does not attempt: serialization round trips, window semantics, framework composition -(Hydra fan-out, tumbling windows, sliding histograms), and cross-implementation -parity between core types and their portable wire twins. +(Hydra fan-out, tumbling windows, sliding histograms). A composition framework can also *be* a battery subject, in the same suite that already covers it. `e2e/frameworks.rs` runs a single-column Hydra — where the @@ -285,8 +282,8 @@ bugs only appeared at deployment-shaped dimensions. ### 4. Add depth where the sketch is unusual Anything not covered by a battery belongs in the matching `e2e_*.rs` suite: -window semantics, merge order-independence, wire-format parity with the -portable type, heavy-hitter recall targets, and so on. +window semantics, merge order-independence, wire round trips, heavy-hitter +recall targets, and so on. ## Tolerance policy @@ -326,7 +323,7 @@ error against its bound. - No assertion depends on unseeded randomness — no `rand::rng()`, no wall-clock seeding, no implicit RNG inside a constructor. An unseeded sketch appears only where the assertion holds for every draw. `Kll::with_k_and_seed`, - `KllDynamic::with_k_and_seed`, `KllSketch::with_seed`, + `KllDynamic::with_k_and_seed`, `NitroBatch::with_target_and_seed`, `UniformSampling::with_seed`, `CocoSketch::with_dimensions_and_seed` and `CocoOctoPlan::with_seed` exist for this. - An estimator whose guarantee is unbiasedness gets no per-key band: its diff --git a/docs/index.md b/docs/index.md index c935f3fc..5dec0b44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,7 +15,7 @@ This page is the docs home for `asap_sketchlib`. - [API Pages Directory](./api/) - [Common API](./api/api_common.md) - [ASAPv1 Wire Format](./asapv1_wire_format.md) - authoritative byte-level spec for the self-describing sketch envelope mirrored by `sketchlib-go` -- [Message Pack Format](./message_pack_format.md) - how the serialization code is organized (shared envelope + per-sketch `wire.rs`; `portable`/`native` retiring) +- [Message Pack Format](./message_pack_format.md) - how the serialization code is organized (shared envelope + per-sketch `wire.rs`) - [Advanced Use Cases](./advanced_use_cases.md) - [TopKHeap Acceleration](./topk_heap_acceleration.md) - what the top-k heap costs per update, how its digest-keyed position index is maintained through the sift, and the throughput it measures at - [Test Coverage Map](./tests.md) diff --git a/docs/library_map.md b/docs/library_map.md index 00e8134e..84f2b8cb 100644 --- a/docs/library_map.md +++ b/docs/library_map.md @@ -12,8 +12,8 @@ - **`src/sketches/`** - Sketch implementations (status source: [apis.md](./apis.md)) - `Ready` in API index: `countminsketch.rs`, `countsketch.rs`, `hll.rs`, `kll.rs`, `ddsketch.rs`, `countminsketch_heap.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `space_saving.rs`, `bloom.rs`, `elastic.rs`, `coco.rs` - `Unstable` in API index: `uniform.rs`, `kmv.rs` - - Not in the API index: `count_min_fold.rs`, `count_sketch_fold.rs`, `kll_dynamic.rs`, `octo_delta.rs`, `mode.rs` (`RegularPath` / `FastPath`) - - A sketch on the ASAPv1 wire format uses a `.rs` (algorithm) + `/wire.rs` (serialization) split: `hll.rs`, `countminsketch.rs`, `countminsketch_heap.rs`, `countsketch.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `kll.rs`, `kll_dynamic.rs`, `ddsketch.rs`, `elastic.rs`, `coco.rs`, `uniform.rs`, `kmv.rs`, `bloom.rs` and `space_saving.rs` (see [asapv1_wire_format.md](./asapv1_wire_format.md)) + - Not in the API index: `count_min_fold.rs`, `count_sketch_fold.rs`, `kll_dynamic.rs`, `octo_delta.rs`, `set_aggregator.rs` (`SetAggregator` / `SetAggregatorDelta`), `mode.rs` (`RegularPath` / `FastPath`) + - A sketch on the ASAPv1 wire format uses a `.rs` (algorithm) + `/wire.rs` (serialization) split: `hll.rs`, `countminsketch.rs`, `countminsketch_heap.rs`, `countsketch.rs`, `countsketch_heap.rs`, `countsketch_l2_norm.rs`, `countminsketch_space_saving.rs`, `countsketch_space_saving.rs`, `kll.rs`, `kll_dynamic.rs`, `ddsketch.rs`, `elastic.rs`, `coco.rs`, `uniform.rs`, `kmv.rs`, `bloom.rs`, `space_saving.rs` and `set_aggregator.rs` (see [asapv1_wire_format.md](./asapv1_wire_format.md)) - Two of those directories carry a second, shared wire module: `countminsketch_heap/heap_wire.rs` holds the top-k heap encoding CountMinSketchWithHeap and CountSketchWithHeap share, and `countminsketch_space_saving/summary_wire.rs` holds the encoding CountMinSketchWithSpaceSaving and CountSketchWithSpaceSaving share. `countsketch_l2_norm/wire.rs` also holds the layer sub-payload the UnivMon family inlines - **`src/sketch_framework/`** - Orchestration and serving layers (status source: [apis.md](./apis.md)) @@ -23,11 +23,9 @@ - Not in the API index: `tumbling.rs`, `sketch_catalog.rs` - The same `.rs` + `/wire.rs` split carries the ASAPv1 wire format for `hydra.rs`, `univmon.rs`, `univmon_two_tier.rs`, `univmon_q.rs`, `eh.rs` and `eh_bucket_sketch.rs` -- **`src/message_pack_format/`** - Serialization plumbing ([message_pack_format.md](./message_pack_format.md)). The current format is **ASAPv1**, specified in [asapv1_wire_format.md](./asapv1_wire_format.md) +- **`src/asapv1/`** (private) - Serialization plumbing ([message_pack_format.md](./message_pack_format.md)). The format is **ASAPv1**, specified in [asapv1_wire_format.md](./asapv1_wire_format.md) - `envelope.rs` — the shared, sketch-agnostic ASAPv1 framing (magic/version/`kind_id` + length prefixes, `encode`/`split`); every `wire.rs` under `src/sketches/` and `src/sketch_framework/` calls into it - - `codec.rs` — the `MessagePackCodec` trait; `error.rs` — the unified `Error` - - `portable/` — **deprecated**, being retired. The older per-sketch wire types (`CountMinSketch`, `HllSketch`, …); ASAPv1 (per-sketch `wire.rs`) is now what `sketchlib-go` mirrors, not these - - `native/` — **deprecated**, being retired. Older `MessagePackCodec` shims over `src/sketches/` byte serialization + - `kind.rs` — the `kind_id` registry constants ## Documentation diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 54ebf0e9..58cefb9c 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -1,7 +1,7 @@ # Message Pack Format -`src/message_pack_format/` holds the Rust-side serialization plumbing. The -current, self-describing wire format is **ASAPv1**; its byte-level layout is the +`src/asapv1/` holds the Rust-side serialization plumbing (a private module). +The self-describing wire format is **ASAPv1**; its byte-level layout is the single source of truth in [**`asapv1_wire_format.md`**](./asapv1_wire_format.md) — read that for the authoritative spec. This page describes how the code is organized. @@ -49,7 +49,7 @@ the metadata field tables, and the byte-level encoding rules. ### Shared framing — `envelope.rs` -`src/message_pack_format/envelope.rs` is the **one shared, sketch-agnostic** +`src/asapv1/envelope.rs` is the **one shared, sketch-agnostic** module every sketch calls into. It owns the magic sentinel, the layout version, and the byte framing (`encode` / `split`). It validates only the magic, version, and framing — it does **not** know the `kind_id` registry or any sketch. Rule @@ -140,30 +140,13 @@ Converted: **EhBucketSketch** (`src/sketch_framework/eh_bucket_sketch/wire.rs`, `0x14 0x00`) — one `[kind_id, descriptor, state]` triple, standalone and per bucket. - **SetAggregator** and **SetAggregatorDelta** - (`src/message_pack_format/portable/set_aggregator/wire.rs`, `0x08 0x00` and + (`src/sketches/set_aggregator/wire.rs`, `0x08 0x00` and `0x08 0x80`) — the members, and the added and removed members, as strictly ascending `str` arrays. NitroBatch, CountMinSketchFold, CountSketchFold, HashSketchEnsemble, ExponentialHistogramUnivMon and OctoSketch have **no `wire.rs`**; their `kind_id`s are reserved with payload TBD. -## `portable/` and `native/` are being retired - -The `portable/` and `native/` sub-modules (and the `MessagePackCodec` trait / -unified `Error` at the module root) are the **older** serialization path and are -being **phased out** in favor of the per-sketch `wire.rs` + the shared -`envelope.rs`. - -- `portable/` holds per-sketch wire types that predate the envelope. They - carry their own byte-parity goldens against `sketchlib-go` — HLL, Count-Min, - Count Sketch and KLL (both directions) — separate from `asapv1_golden/`. -- `native/` is a set of thin `MessagePackCodec` shims over the sketches' - `to_asapv1` / `from_asapv1`. Every type it wraps emits the - ASAPv1 envelope, so each shim is a pass-through. - -Sequencing note (from the spec): `portable` is not deleted until the golden -byte-vector fixtures are the drift guard on both sides. - ## Cross-language parity ASAPv1 parity with `sketchlib-go` is proven by **golden byte-vectors** in @@ -178,8 +161,6 @@ That proof covers the eight `kind_id`s the test checks — HLL's three estimators, Count-Min, Count Sketch, compact KLL, SetAggregator and its delta. Every other implemented kind has **no ASAPv1 golden and therefore no ASAPv1 drift guard**; [its README](https://github.com/ProjectASAP/sketchlib-golden-bytes#coverage) lists what is covered. -The older `portable` path is guarded separately, by the `sketchlib-go` goldens -of its own listed above. ## Cross-Reference diff --git a/docs/tests.md b/docs/tests.md index 9686d50e..453dd702 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -125,7 +125,7 @@ Delta tests: [`src/sketches/hll/delta.rs`](../src/sketches/hll/delta.rs) | `hll_correctness_test` | Register update logic matches expected bucket/index behavior for all HLL variants. | Runs fixed hashed inserts against Classic, Ertl, and HIP variants; asserts exact expected register values at specific bucket indices and confirms an untouched bucket remains zero. | | `hll_envelope_structure_and_kind_id_guard` | The envelope frames an Ertl-MLE sketch under kind_id `0x01 0x02`, and a Classic decoder refuses it. | For 1,000 inserts into `HyperLogLog`, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x01 0x02`, that the decoded registers match the source, and that a `HyperLogLog` decode fails. | | `hll_hip_round_trip_preserves_state` | The HIP running scalars travel beside the registers. | For 1,000 inserts into `HyperLogLogHip`, verifies the bytes carry kind_id `0x01 0x03` and that the decoded sketch matches the source on the registers and on `kxq0`, `kxq1`, and `est`. | -| `native_and_portable_hll_bytes_match` | The native and portable encoders emit the same envelope. | For 1,000 inserts each, verifies an `Ertl` sketch's bytes equal the portable `HllSketch` `Datafusion` encoding of its registers, and a `HyperLogLogHip` sketch's bytes equal the portable `Hip` encoding of its registers plus `kxq0`, `kxq1`, and `est`. | +| `known_state_matches_golden` | Known register and HIP state serializes to the `asapv1_golden/` bytes and decodes back to it. | Builds P12 `Classic` and `Ertl` sketches with registers `0 -> 1`, `1 -> 7`, `100 -> 42`, `4095 -> 3`, and `HyperLogLogHipImpl` sketches at P12 (`kxq0` `1.5`, `kxq1` `2.5`, `est` `3.0`) and P14 (registers `0 -> 1`, `1 -> 7`, `8192 -> 42`, `16383 -> 51`; `kxq0` `16380.5`, `kxq1` `0.25`, `est` `4.125`), verifies each `to_asapv1` equals `hll_classic_p12`, `hll_ertl_mle_p12`, `hll_hip_p12` and `hll_hip_p14`, and that decoding each golden gives back those registers and HIP scalars. | | `hll_custom_hasher_profile_round_trips_and_is_self_describing` | The metadata describes the hasher that built the sketch rather than a hardcoded profile. | For a P14 `Ertl` sketch over a hasher declaring its own `HashProfile`, verifies the registers round-trip, the bytes differ from the standard-profile sketch's over the same 1,000 inserts, and a standard-profile decode rejects them. | | `hll_metadata_rejects_unknown_keys` | An unexpected metadata key fails closed. | Encodes the eight `HllMetadata` fields plus a `bogus_field` as a named map and verifies it does not decode as `HllMetadata`. | | `hll_precision_cross_rejection` | The metadata `precision` pins the register storage the bytes belong to. | Verifies a populated `HyperLogLogP12` envelope fails to decode as the P14 `HyperLogLog`. | @@ -857,6 +857,24 @@ Wire tests: [`src/sketches/uniform/wire.rs`](../src/sketches/uniform/wire.rs) | `uniform_sampling_rejects_crafted_bytes_without_panicking` | Truncated, foreign and garbage bytes are errors, never panics. | Verifies six truncations of a valid envelope, 64 bytes of `0xff`, and a valid envelope carrying a garbage payload all fail to decode. | | `uniform_sampling_rejects_serializing_an_over_full_sampler` | The encode side refuses a sampler its own decoder would reject. | Verifies a sampler holding two samples whose `total_seen` is set to `1` fails to serialize. | +### SetAggregator + +Test file: [`src/sketches/set_aggregator.rs`](../src/sketches/set_aggregator.rs) + +Wire tests: [`src/sketches/set_aggregator/wire.rs`](../src/sketches/set_aggregator/wire.rs) + +| test_name | test_description | what_is_tested | +| --- | --- | --- | +| `test_creation` | A fresh aggregator holds nothing. | Verifies `SetAggregator::new()` has an empty `values` set. | +| `test_insert` | A repeated key is inserted once. | Feeds `"web"`, `"api"`, and `"web"` again and verifies `values` holds 2 keys, both `"web"` and `"api"`. | +| `test_merge` | Merge is set union. | Merges an aggregator holding `{"api", "db"}` into one holding `{"web", "api"}` and verifies the result holds exactly `"web"`, `"api"`, and `"db"`. | +| `set_aggregator_rejects_unordered_and_duplicate_members` | `values` must be strictly ascending by UTF-8 bytes. | Verifies crafted payloads `["b", "a"]` and `["a", "a"]` each fail to decode. | +| `a_bin_member_is_rejected` | A member must be msgpack `str`. | Verifies a set payload whose one member is the `bin` `b"a"`, and a delta whose `removed` holds that `bin`, each fail to decode. | +| `metadata_must_be_exactly_metadata_version_1` | The metadata map is exactly `{metadata_version: 1}`. | Verifies a map with an extra `bogus_field` key, and one with `metadata_version` `2`, each fail to decode. | +| `each_decoder_rejects_a_foreign_kind_id` | Each decoder accepts only its own kind_id. | Verifies a well-formed set payload framed under `0x08 0x80` fails as a `SetAggregator`, and a well-formed delta payload under `0x08 0x00` or the retired `0x09 0x00` fails as a `SetAggregatorDelta`. | +| `delta_result_rejects_a_member_both_added_and_removed` | `added` and `removed` are disjoint on both sides of the wire. | Verifies a crafted delta with `added` `["a", "b"]` and `removed` `["b"]` fails to decode, and that a `SetAggregatorDelta` holding `"a"` in both sets fails to serialize. | +| `delta_result_rejects_unordered_sides` | Each side is strictly ascending. | Verifies a delta with `added` `["b", "a"]`, and one with `removed` `["c", "c"]`, each fail to decode. | + ## Sketch Frameworks ### Hydra @@ -1330,213 +1348,6 @@ Test file: [`src/sketch_framework/tumbling.rs`](../src/sketch_framework/tumbling | `tumbling_very_small_windows` | Five hundred rotations under a fifty-window cap still answer the retained tail. | For `TumblingWindow::new(10, 50, cfg, 52)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 1024, fold_level: 3, heap_size: 10 }` fed a seeded (`0x77_1100`) Zipf(`domain = 500`, `exponent = 1.2`) stream of `5000` samples at timestamps `i` - `500` windows, of which only the newest `50` closed plus the active survive - verifies over `85%` of the truth built from samples `4500..5000` is within `(e / 1024) * L1` under `query_all`. | | `tumbling_skewed_load` | A window carrying most of the stream does not break the Count-Min bound. | For `TumblingWindow::new(10_000, 5, cfg, 7)` over `CountMinSketchFoldConfig { rows: 3, full_cols: 4096, fold_level: 4, heap_size: 20 }` fed five seeded (`0xBE_EF00 + w`) Zipf(`domain = 5000`, `exponent = 1.1`) phases - `1_000` samples each except `36_000` for phase `2`, each phase followed by a jump to the next window boundary - verifies over `90%` of keys are within `(e / 4096) * L1`, the truth covering all `40_000` samples including the two light windows the `max_windows` cap has already evicted. | -## MessagePack Portable Wire Types - -### Portable CountMinSketch - -Test file: [`src/message_pack_format/portable/countminsketch.rs`](../src/message_pack_format/portable/countminsketch.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `msgpack_delta_against_empty_round_trips` | A msgpack delta against an empty snapshot carries the window. | For a `5x1024` sketch fed 200 updates of `"hot"` at weight `1.0`, verifies the msgpack delta computed against a fresh `5x1024` sketch at threshold `1.0`, applied to another fresh sketch, reproduces the `"hot"` estimate to within 10%. | -| `msgpack_delta_wire_layout_is_parallel_arrays` | The delta is a five-element array of scalars plus parallel index arrays. | For a `3x8` sketch fed `"x"` at weight `5.0`, verifies the delta bytes equal `rmp_serde::to_vec(&(rows as u64, cols as u64, row_idx: Vec, col_idx: Vec, d_count: Vec))` recomputed from the non-zero cells of `sketch()`, and that decoding reports `rows` `3` and `cols` `8`. | -| `test_count_min_sketch_creation` | A fresh sketch has the requested shape and zeroed cells. | For `CountMinSketch::new(4, 1000)`, verifies `rows` is `4`, `cols` is `1000`, `sketch()` is 4 rows of 1,000, and every cell is `0.0`. | -| `test_count_min_sketch_update` | An insert never reads below its weight. | For a `2x10` sketch fed `"key1"` at weight `1.0`, verifies `estimate("key1")` is at or above `1.0`. | -| `test_count_min_sketch_query_empty` | An empty sketch answers zero. | Verifies a `2x10` sketch reports `estimate("anything")` of exactly `0.0`. | -| `test_count_min_sketch_merge` | Merge sums cells element-wise. | Merges two `2x3` legacy matrices, the left holding `5.0` at `(0,0)` and `10.0` at `(1,2)` and the right `3.0` at `(0,0)` and `7.0` at `(0,1)`, and verifies the merged cells read `8.0`, `7.0`, and `10.0`. | -| `test_count_min_sketch_merge_dimension_mismatch` | Mismatched row counts are refused. | Verifies merging a `3x3` sketch into a `2x3` one fails. | -| `test_count_min_sketch_msgpack_round_trip` | The msgpack round trip preserves shape and estimates. | Round-trips a `4x256` sketch fed `"apple"` at `5.0` then `2.0` and `"banana"` at `3.0`, and verifies the decode reports `rows` `4`, `cols` `256`, and estimates at or above `7.0` for `"apple"` and `3.0` for `"banana"`. | -| `test_aggregate_count` | `aggregate_count` builds a decodable sketch from parallel key and weight arrays. | For `aggregate_count(4, 100, ["a", "b", "a"], [1.0, 2.0, 3.0])`, verifies the returned bytes decode and report estimates at or above `4.0` for `"a"` and `2.0` for `"b"`. | -| `test_aggregate_count_empty` | An empty stream produces nothing. | Verifies `aggregate_count(4, 100, &[], &[])` returns `None`. | -| `test_apply_delta_additive` | Delta cells add into the matrix. | For a `2x3` sketch holding `[[1, 2, 3], [4, 5, 6]]`, applies cells `(0,0,10)` and `(1,2,100)` and verifies the matrix becomes `[[11, 2, 3], [4, 5, 106]]`. | -| `test_apply_delta_matches_full_merge` | Applying a delta lands where merging the same addition lands. | For a `2x2` base `[[1, 2], [3, 4]]`, verifies merging the addition `[[10, 0], [0, 20]]` and applying the delta cells `(0,0,10)` and `(1,1,20)` give the same matrix. | -| `test_apply_delta_out_of_range` | A cell outside the declared geometry is refused. | Verifies a delta carrying cell `(5, 0, 1)` against a `2x3` sketch fails to apply. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The proto envelope for a populated matrix is byte-identical to the checked-in golden. | For a `4x2048` sketch fed 50 updates of `flow-{i % 10}` at weight `1.0`, builds a `CountMinState` with `counter_type` `INT64`, row-major `counts_int`, and per-row `l1`/`l2`, wraps it in a `SketchEnvelope` with `format_version` `1` and `sample_p` `0.0`, and verifies the prost encoding matches the 8,275-byte [`cms_envelope_golden.hex`](../src/sketches/testdata/cms_envelope_golden.hex) on both length and content. | -| `test_hh_keys_matches_go_golden_bytes` | Heavy-hitter keys at tag 6 reach the wire in the golden's shape. | For the same `4x2048` and 50-update `flow-{i % 10}` stream, decodes the packed delta as a proto `CountMinSketchDelta`, sets `hh_keys` to `["flow-0", "flow-3", "flow-7"]`, and verifies the re-encoding equals a 263-byte inline golden, then that those bytes apply to a fresh `4x2048` sketch. | -| `test_compute_delta_against_empty_round_trips` | A proto delta against an empty snapshot carries the whole matrix. | For a `4x2048` sketch fed 200 updates of `flow-{i % 37}` at weight `1.0`, verifies the delta computed against an empty sketch at threshold `1.0`, applied to a fresh `4x2048` sketch, reproduces `sketch()` cell for cell. | -| `test_compute_delta_then_apply_matches_current` | A delta between two non-empty snapshots reconstructs the later one. | For a `2x64` base fed 40 updates of `k{i % 8}` and a current fed 30 more, verifies the delta applied to the base reproduces the current matrix. | -| `test_compute_delta_matches_go_golden_bytes` | The packed delta bytes are pinned by a golden. | For a `4x2048` sketch fed 50 updates of `flow-{i % 10}` at weight `1.0`, verifies the delta against an empty sketch at threshold `1.0` equals a 239-byte inline golden. | - -### Portable CountMinSketchWithHeap - -Test file: [`src/message_pack_format/portable/countminsketch_topk.rs`](../src/message_pack_format/portable/countminsketch_topk.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `test_creation` | A fresh composite has the requested matrix and an empty heap. | For `CountMinSketchWithHeap::new(4, 1000, 20)`, verifies `rows` `4`, `cols` `1000`, `heap_size` `20`, a `sketch_matrix()` of 4 rows of 1,000, and no heap items. | -| `test_query_empty` | An empty composite answers zero. | Verifies a `2x10` composite with a 5-slot heap reports `estimate("anything")` of `0.0`. | -| `test_merge` | Merge sums the matrices and adopts the peer's heap bound. | Merges a `2x10` composite holding `5.0` at `(0,0)`, `15.0` at `(1,1)`, heap `{key3: 75.0, key1: 80.0}` and `heap_size` `3` into one holding `10.0` at `(0,0)`, `20.0` at `(1,1)`, heap `{key1: 100.0, key2: 50.0}` and `heap_size` `5`, and verifies the merged cells read `15.0` and `35.0`, `heap_size` becomes `3`, and the heap holds at most 3 items. | -| `test_merge_dimension_mismatch` | Mismatched row counts are refused. | Verifies merging a `3x10` composite into a `2x10` one fails. | -| `test_msgpack_round_trip` | The round trip carries the matrix, the heap bound, and the heap itself. | Round-trips a `4x128` composite with a 3-slot heap fed `"hot"` at `100.0` and `"cold"` at `1.0`, and verifies the decode reports `rows` `4`, `cols` `128`, `heap_size` `3`, a non-empty heap holding `"hot"` at or above `100.0`, and estimates at or above `100.0` and `1.0`. | -| `test_aggregate_topk` | `aggregate_topk` builds a decodable composite honouring its heap bound. | For `aggregate_topk(4, 100, 2, ["a", "b", "a", "c"], [1.0, 2.0, 3.0, 0.5])`, verifies the decode reports `heap_size` `2` and at most 2 heap items; the heap contents themselves are not checked. | -| `test_aggregate_topk_empty` | An empty stream produces nothing. | Verifies `aggregate_topk(4, 100, 10, &[], &[])` returns `None`. | - -### Portable CountSketch - -Test file: [`src/message_pack_format/portable/countsketch.rs`](../src/message_pack_format/portable/countsketch.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `msgpack_delta_against_empty_round_trips` | A msgpack delta against an empty snapshot carries the signed cells exactly. | For a `5x1024` sketch fed 200 updates of `"hot"` at weight `1.0`, verifies the delta against a fresh sketch at threshold `1.0`, applied to another fresh sketch, reproduces `sketch()` cell for cell. | -| `msgpack_delta_wire_layout_is_parallel_arrays` | The delta is a five-element array of scalars plus parallel index arrays. | For a `3x8` sketch fed `"x"` at weight `5.0`, verifies the delta bytes equal `rmp_serde::to_vec(&(rows as u64, cols as u64, row_idx: Vec, col_idx: Vec, d_count: Vec))` recomputed from the non-zero cells, and that decoding reports `rows` `3` and `cols` `8`. | -| `test_new_empty` | A fresh sketch has the requested shape and zeroed cells. | For `CountSketch::new(2, 3)`, verifies `rows` is `2`, `cols` is `3`, and `sketch()` is `[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]`. | -| `test_from_legacy_matrix` | A legacy matrix is adopted unchanged, signs included. | Verifies `from_legacy_matrix([[1.0, -2.0, 3.0], [-4.0, 5.0, -6.0]], 2, 3)` reports exactly that matrix from `sketch()`. | -| `test_merge_element_wise` | Merge sums signed cells element-wise. | Merges the `2x2` matrix `[[-1.0, -2.0], [-3.0, -4.0]]` into `[[1.0, 2.0], [3.0, 4.0]]` and verifies every merged cell is `0.0`. | -| `test_merge_dimension_mismatch` | Mismatched row counts are refused. | Verifies merging a `3x3` sketch into a `2x3` one fails. | -| `test_merge_refs` | `merge_refs` sums every input. | Merges three `1x2` sketches holding `[1.0, 2.0]`, `[3.0, 4.0]`, and `[5.0, 6.0]` and verifies the result is `[[9.0, 12.0]]`. | -| `test_apply_delta_additive` | Delta cells add into the matrix, negative weights included. | For a `2x3` sketch holding `[[1.0, -2.0, 3.0], [-4.0, 5.0, -6.0]]`, applies cells `(0,0,10)`, `(0,2,-3)`, and `(1,1,-15)` and verifies the matrix becomes `[[11.0, -2.0, 0.0], [-4.0, -10.0, -6.0]]`. | -| `test_apply_delta_matches_full_merge` | Applying a delta lands where merging the same addition lands. | For a `2x2` base `[[1.0, 2.0], [3.0, 4.0]]`, verifies merging the addition `[[10.0, 0.0], [0.0, 20.0]]` and applying the delta cells `(0,0,10)` and `(1,1,20)` give the same matrix. | -| `test_apply_delta_out_of_range` | A cell outside the declared geometry is refused. | Verifies a delta carrying cell `(2, 0, 1)` against a two-row `2x3` sketch fails to apply. | -| `test_apply_delta_rebuilds_topk_from_hh_keys` | The `hh_keys` channel rebuilds Top-K by re-querying the merged matrix. | For a `3x16` sketch fed `"alpha"` at `5.0` and `"beta"` at `3.0`, applies a delta with no cells and `hh_keys` `["alpha", "beta"]` and verifies `topk` holds both keys and that `"alpha"` reads above `"beta"`. | -| `test_apply_delta_hh_keys_topk_capacity` | The rebuilt Top-K is bounded by `COUNT_SKETCH_TOPK_CAPACITY`. | For a `3x1024` sketch fed 105 keys `k0000`..`k0104` at weights `1.0`..`105.0`, applies a delta whose `hh_keys` carries all 105 and verifies `topk` holds exactly `COUNT_SKETCH_TOPK_CAPACITY` (`100`) entries; which entries survive is not checked. | -| `test_msgpack_round_trip` | The msgpack round trip preserves the signed matrix and its shape. | Round-trips a `2x2` sketch holding `[[1.5, -2.5], [3.5, -4.5]]` and verifies the matrix, `rows`, and `cols` are unchanged. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The proto envelope for a populated matrix is byte-identical to the golden. | For a `3x512` sketch fed 25 updates over the five keys `k-a`..`k-e` (each five times) at weight `1.0`, builds a `CountSketchState` with `counter_type` `INT64`, row-major `counts_int`, per-row `l2` of `125.0`, and no `topk`, wraps it in a `SketchEnvelope` with `format_version` `1`, and verifies the prost encoding equals a 1,577-byte inline golden. | -| `test_hh_keys_matches_go_golden_bytes` | Heavy-hitter keys at tag 6 reach the wire in the golden's shape. | For the same `3x512` sketch over `k-a`..`k-e`, decodes the packed delta against an empty sketch as a proto `CountSketchDelta`, sets `hh_keys` to `["flow-0", "flow-3", "flow-7"]`, and verifies the re-encoding equals a 117-byte inline golden. | -| `test_compute_delta_against_empty_round_trips` | A proto delta against an empty snapshot carries the whole signed matrix. | For a `3x512` sketch fed 200 updates of `k-{i % 23}` at weight `1.0`, verifies the delta against an empty sketch at threshold `1.0`, applied to a fresh `3x512` sketch, reproduces `sketch()` cell for cell. | -| `test_compute_delta_then_apply_matches_current` | A delta between two non-empty snapshots reconstructs the later one. | For a `3x64` base fed 40 updates of `x{i % 8}` and a current fed 30 more, verifies the delta applied to the base reproduces the current matrix. | -| `test_compute_delta_matches_go_golden_bytes` | The packed delta bytes and per-row L2 are pinned by a golden. | For a `3x512` sketch fed the 25 `k-a`..`k-e` updates, verifies the delta against an empty sketch at threshold `1.0` equals a 93-byte inline golden carrying the packed cell arrays and a per-row L2 of `125.0`. | - -### Portable CountSketchWithHeap - -Test file: [`src/message_pack_format/portable/countsketch_topk.rs`](../src/message_pack_format/portable/countsketch_topk.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `test_creation` | A fresh composite has the requested matrix and an empty heap. | For `CountSketchWithHeap::new(4, 1000, 20)`, verifies `rows` `4`, `cols` `1000`, `heap_size` `20`, a `sketch_matrix()` of 4 rows of 1,000, and no heap items. | -| `test_query_empty` | An empty composite answers zero. | Verifies a `2x10` composite with a 5-slot heap reports `estimate("anything")` of `0.0`. | -| `test_merge` | Merge sums the matrices and adopts the peer's heap bound. | Merges a `2x10` composite holding `5.0` at `(0,0)`, `15.0` at `(1,1)`, heap `{key3: 75.0, key1: 80.0}` and `heap_size` `3` into one holding `10.0` at `(0,0)`, `20.0` at `(1,1)`, heap `{key1: 100.0, key2: 50.0}` and `heap_size` `5`, and verifies the merged cells read `15.0` and `35.0`, `heap_size` becomes `3`, and the heap holds at most 3 items. | -| `test_merge_dimension_mismatch` | Mismatched row counts are refused. | Verifies merging a `3x10` composite into a `2x10` one fails. | -| `test_msgpack_round_trip` | The round trip carries the matrix, the heap bound, and the heap itself. | Round-trips a `4x128` composite with a 3-slot heap fed `"hot"` at `100.0` and `"cold"` at `1.0`, and verifies the decode reports `rows` `4`, `cols` `128`, `heap_size` `3`, a non-empty heap holding `"hot"` at or above `100.0`, and estimates at or above `100.0` and `1.0`. | -| `test_aggregate_topk` | `aggregate_topk` builds a decodable composite honouring its heap bound. | For `aggregate_topk(4, 100, 2, ["a", "b", "a", "c"], [1.0, 2.0, 3.0, 0.5])`, verifies the decode reports `heap_size` `2` and at most 2 heap items; the heap contents themselves are not checked. | -| `test_aggregate_topk_empty` | An empty stream produces nothing. | Verifies `aggregate_topk(4, 100, 10, &[], &[])` returns `None`. | -| `test_median_estimator_differs_from_cms_style_min` | The median-over-rows estimator tracks the true count rather than inflating. | For a `5x64` composite with a 10-slot heap fed `"heavy"` 50 times at weight `1.0` and 20 keys `light-0`..`light-19` once each, verifies `estimate("heavy")` falls in `40.0..=60.0`. | - -### Portable DdSketch - -Test file: [`src/message_pack_format/portable/ddsketch.rs`](../src/message_pack_format/portable/ddsketch.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `msgpack_delta_against_empty_round_trips` | A msgpack delta against an empty snapshot carries the bucket store. | For an `alpha` `0.01` sketch fed `1.0..=200.0`, verifies the delta against an empty sketch at threshold `1` applied to a fresh sketch gives the same `total_count` and quantiles at `0.5`, `0.9`, and `0.99` within 1%. | -| `msgpack_delta_wire_layout_is_parallel_arrays` | The delta is a two-element array of parallel index and count arrays. | For an `alpha` `0.01` sketch fed `1.0`, `1.0`, and `1000.0`, verifies the delta bytes equal `rmp_serde::to_vec(&(idx: Vec, d_count: Vec))` recomputed from the non-zero store cells at absolute index, and that decoding reports two buckets. | -| `test_new_empty` | A fresh sketch holds nothing. | For `DdSketch::new(0.01)`, verifies `total_count` is `0` and `positive_store_counts` is empty. | -| `test_merge_aligned_same_offset` | Aligned stores add cell by cell. | Merges `[10, 20, 30]` at offset `-1` into `[1, 2, 3]` at offset `-1` and verifies the store becomes `[11, 22, 33]`, the offset stays `-1`, and `total_count` is `66`. | -| `test_merge_overlapping_offsets` | Merge widens the window across overlapping stores. | Merges `[10, 10, 10]` at offset `0` into `[1, 1, 1]` at offset `-1` and verifies the store becomes `[1, 11, 11, 10]` at offset `-1` with `total_count` `33`. | -| `test_merge_disjoint_offsets` | Merge zero-fills the gap between disjoint stores. | Merges `[3, 4]` at offset `5` into `[1, 2]` at offset `0` and verifies the store becomes `[1, 2, 0, 0, 0, 3, 4]` at offset `0`. | -| `test_apply_delta_additive_inside_store` | Delta buckets inside the store add to their cells. | For a base `[1, 2, 3]` at offset `-1`, applies a delta with buckets `(-1,4)`, `(0,8)`, `(1,12)`, `d_count` `24`, `d_sum` `120.0`, and `new_max` `9.0`, and verifies the store becomes `[5, 10, 15]` with `total_count` `30`. | -| `test_apply_delta_expands_store_on_new_bucket` | A delta bucket past the store grows it. | For a base `[1, 2]` at offset `0`, applies a delta with the single bucket `(4, 7)`, `d_count` `7`, and `d_sum` `35.0`, and verifies the store becomes `[1, 2, 0, 0, 7]` at offset `0` with `total_count` `10`. | -| `test_apply_delta_matches_full_merge` | Applying a delta lands where merging the same addition lands. | For a base `[1, 2, 3]` at offset `0`, verifies merging `[10, 0, 20]` at offset `0` and applying the delta buckets `(0,10)` and `(2,20)` give the same store and `total_count`. | -| `test_merge_alpha_mismatch` | Sketches of different relative accuracy are not mergeable. | Verifies merging an `alpha` `0.02` sketch into an `alpha` `0.01` one fails. | -| `test_msgpack_round_trip` | The msgpack round trip preserves alpha, store, and offset. | Round-trips `from_raw(0.01, [1, 2, 3], -2)` and verifies `alpha`, `positive_store_counts`, `positive_store_offset`, and `total_count` are unchanged. | -| `test_msgpack_is_three_element_array` | The wire form is a three-element array, not a map. | For `from_raw(0.01, [1, 2, 3], -2)`, verifies the first byte of `to_msgpack` is `0x93`, the msgpack fixarray marker for length `3`. | -| `test_insert_and_quantile_lognormal` | Quantiles track a known log-normal distribution. | Feeds 100,000 xorshift64\*/Box-Muller log-normal samples with `mu` `3` and `sigma` `0.7` into an `alpha` `0.01` sketch and verifies `\|ln(P50 / 20.09)\|` and `\|ln(P99 / 102.4)\|` are each below `0.05`; P90 is not asserted. | -| `test_delta_chain_preserves_quantile_accuracy` | A chain of applied deltas stays inside the alpha bound of the direct path. | Over five batches of 10,000 log-normal samples at `alpha` `0.01`, seeds one sketch from the first batch and applies four deltas built by the test's own `compute_dd_delta` helper, then verifies its `total_count` equals the directly fed sketch's exactly and that quantiles at `0.5`, `0.9`, and `0.99` agree within `alpha`; the shipped `compute_delta` is not the path exercised here. | -| `test_compute_delta_against_empty_round_trips` | A delta against an empty snapshot carries the whole store. | For an `alpha` `0.01` sketch fed `1.0..=200.0`, verifies the delta against empty at threshold `1`, applied to a fresh sketch, gives the exact `total_count`, the same count for every non-zero bucket compared at absolute index, and quantiles at `0.5`, `0.9`, and `0.99` within 1%. | -| `test_consecutive_windows_emit_own_state` | Each window emits its own state rather than a cross-window difference. | Builds `win1` from `1.0..=50.0` inserted twice (count `2` per bucket) and `win2` from `1.0..=50.0` once, and verifies `win2`'s delta against empty reconstructs `win2`'s `total_count` and that those bytes differ from `win2`'s delta against `win1`. | -| `test_update_then_envelope_matches_go_golden_bytes` | The proto envelope for a populated store is byte-identical to the golden. | For an `alpha` `0.01` sketch fed `1.0..=50.0`, builds a `DdSketchState` from `wire_alpha()`, `positive_store_counts`, and `positive_store_offset`, wraps it in a `SketchEnvelope` with `format_version` `1`, and verifies the prost encoding equals a 403-byte inline golden carrying a 384-entry store and a `positive_store_offset` of `-64` (sint32 zigzag `0x7f`); the golden constant records that it is the Rust-produced value and still to be reconciled against the Go reference. | - -### Portable SetAggregatorDelta - -Test file: [`src/message_pack_format/portable/delta_set_aggregator.rs`](../src/message_pack_format/portable/delta_set_aggregator.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `test_msgpack_round_trip` | Both key sets survive the round trip. | Round-trips a `SetAggregatorDelta` with `added` `{"web", "api"}` and `removed` `{"db"}` and verifies the decode holds 2 added keys including both names and 1 removed key. | -| `test_empty_sets` | A `SetAggregatorDelta` holding nothing round-trips. | Verifies a `SetAggregatorDelta` with both sets empty encodes and decodes back with both sets still empty. | - -### Portable HllSketch - -Test file: [`src/message_pack_format/portable/hll.rs`](../src/message_pack_format/portable/hll.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `estimate_holds_past_the_32_bit_range` | The estimator follows the HyperLogLog formula rather than saturating. | For each rank in `12`, `16`, `18`, and `20` filled across all 16,384 registers of a precision-14 sketch, verifies `estimate()` is within a relative `1e-9` of `alpha_m * m * 2^rank` with `alpha_m = 0.7213 / (1 + 1.079 / m)` and `m = 16384`. | -| `msgpack_delta_against_empty_round_trips` | A msgpack register delta against an empty snapshot carries every register. | For a precision-12 sketch fed the little-endian bytes of `0..2000u64`, verifies the delta against an empty sketch at threshold `1`, applied to a fresh sketch, reproduces the register array exactly and the estimate to within 0.1%. | -| `msgpack_delta_wire_layout_is_parallel_arrays` | The register delta is two parallel arrays of index and value. | For a precision-4 sketch fed `b"a"` and `b"b"`, verifies the delta bytes equal `rmp_serde::to_vec(&(reg_idx: Vec, reg_val: Vec))` recomputed from the non-zero registers, and that decoding yields one update per non-zero register. | -| `test_new_empty` | A fresh sketch is `2^precision` zeroed registers. | For `HllSketch::new(Regular, 4)`, verifies the register array is 16 long and every register is `0`. | -| `test_merge_register_wise_max` | Merge takes the register-wise maximum. | Merges the precision-2 registers `[4, 2, 6, 0]` into `[1, 5, 3, 7]` and verifies the result is `[4, 5, 6, 7]`. | -| `test_apply_delta_max_semantics` | A delta raises registers to the maximum, never lowers them. | Applies the updates `(0,4)`, `(1,2)`, `(2,6)`, `(3,0)` to the precision-2 registers `[1, 5, 3, 7]` and verifies the result is `[4, 5, 6, 7]`. | -| `test_apply_delta_out_of_range` | A register index past the array is refused. | Verifies a delta carrying update `(7, 3)` against a precision-2 (four-register) sketch fails to apply. | -| `test_apply_delta_matches_full_merge` | Applying a delta lands where merging the same addition lands. | For the precision-2 base `[1, 5, 3, 7]`, verifies merging the addition `[4, 0, 6, 0]` and applying the delta updates `(0,4)` and `(2,6)` give the same register array. | -| `test_merge_variant_mismatch` | Two estimator variants are not mergeable. | Verifies merging a `Datafusion` precision-4 sketch into a `Regular` precision-4 one fails. | -| `test_merge_precision_mismatch` | Two precisions are not mergeable. | Verifies merging a precision-5 `Regular` sketch into a precision-4 one fails. | -| `test_merge_refs` | `merge_refs` takes the maximum across every input. | Merges three precision-1 sketches holding `[1, 0]`, `[0, 3]`, and `[2, 2]` and verifies the result is `[2, 3]`. | -| `test_update_then_estimate_within_2pct` | The estimate lands within 2% on a 50,000-key stream. | Feeds `key-0`..`key-49999` into a precision-12 `Regular` sketch and verifies the relative error of `estimate()` against `50_000` is below `0.02`. | -| `test_estimate_empty_is_zero` | An empty sketch estimates zero. | Verifies a precision-4 `Regular` sketch reports an `estimate()` of exactly `0.0`. | -| `test_msgpack_round_trip` | The encoding is an ASAPv1 envelope and the HIP state survives it. | Round-trips a `Hip` precision-3 sketch holding registers `[0, 1, 2, 3, 4, 5, 6, 7]` with `hip_kxq0` `1.0`, `hip_kxq1` `2.0`, and `hip_est` `3.0`, and verifies the bytes open with the `ASAPv1` magic and that the decode matches on registers, precision, and a `hip_kxq0` of `1.0`. | -| `from_msgpack_rejects_a_sampled_hll` | The portable type refuses a sampled HLL. | Verifies `HllSketch::from_msgpack` over a `p = 0.5` Ertl-MLE envelope fails with a message naming `admission_rate`. | -| `test_update_then_envelope_matches_sketchlib_go_bytes` | The proto envelope for a populated register array is byte-identical to the checked-in golden. | For a `Datafusion` precision-14 sketch fed the little-endian `f64` bytes of `1..=50`, builds a `HyperLogLogState` from `wire_proto_variant()`, the precision, and the registers with `registers_sparse` unset, wraps it in a `SketchEnvelope` with `format_version` `1` and `sample_p` `0.0`, and verifies the prost encoding matches the 16,398-byte [`hll_envelope_golden.hex`](../src/sketches/testdata/hll_envelope_golden.hex) on both length and content. | -| `sparse_round_trip_exact` | The sparse register encoding is exact. | For a 16,384-register array holding `1` at index `0`, `7` at `1`, `51` at `300`, `12` at `8191`, and `3` at `16383`, verifies `encode_sparse_registers` reports `num_registers` `16384` and that the decode returns the array unchanged. | -| `registers_from_state_reads_both` | `registers_from_state` reads the sparse and the dense form alike. | For the same 16,384-register array at `variant` `2` and `precision` `14`, verifies the registers come back identical whether the state carries `registers_sparse` with `registers` empty or `registers` with `registers_sparse` unset. | -| `sparse_empty_is_all_zero` | An all-zero register array encodes to nothing. | For 16,384 zeroed registers, verifies the sparse `packed` field is empty and the decode returns the zeros. | -| `sparse_rejects_out_of_range_index` | A sparse index past `num_registers` is refused. | Verifies `HllSparseRegisters { num_registers: 4, packed: [5, 3] }`, whose first uvarint delta of `5` lands outside a four-register array, fails to decode. | -| `test_compute_delta_against_empty_round_trips` | A proto register delta against an empty snapshot carries every register. | For a `Datafusion` precision-14 sketch fed the little-endian bytes of `0..5000u64`, verifies the delta against an empty sketch at threshold `0`, applied to a fresh sketch, reproduces the register array. | -| `test_compute_delta_then_apply_matches_current` | A delta between two non-empty snapshots reconstructs the later one. | For a `Datafusion` precision-12 base fed `0..2000u64` and a current fed `2000..5000u64` on top, verifies the delta applied to the base reproduces the current register array. | -| `test_compute_delta_matches_go_golden_bytes` | The packed register delta bytes are pinned by a golden. | For a `Datafusion` precision-14 sketch fed the little-endian `f64` bytes of `1..=50`, verifies the delta against an empty sketch at threshold `0` equals a 132-byte inline golden in the packed `(index_delta, value)` form. | -| `test_compute_subwindow_delta_matches_go_golden_bytes` | A sub-window delta's bytes are pinned by a golden and still reconstruct the window. | For a `Datafusion` precision-14 base fed `0..2000` as `f64` little-endian bytes and a current fed `2000..5000` on top, verifies the delta carries only the registers that grew and equals a 5,097-byte inline golden, then that applying it to the base reproduces the current register array. | - -### Portable HydraKllSketch - -Test file: [`src/message_pack_format/portable/hydra_kll.rs`](../src/message_pack_format/portable/hydra_kll.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `test_creation` | A fresh matrix-of-KLL has the requested shape. | For `HydraKllSketch::new(2, 3, 200)`, verifies `rows` is `2`, `cols` is `3`, and the cell matrix is 2 rows of 3. | -| `test_update_and_query` | A per-key quantile query answers after inserts. | For a `2x10` matrix at `k` `200` fed `"key1"` at `5.0` and `10.0`, verifies `quantile("key1", 0.5)` is at or above `0.0`; the returned value itself is not pinned. | -| `test_merge` | Merge keeps the matrix geometry. | Merges a `2x5` matrix at `k` `200` fed `"key1"` with `6.0..=10.0` into one fed `1.0..=5.0` and verifies only that `rows` stays `2` and `cols` stays `5`; no quantile is checked. | -| `test_merge_dimension_mismatch` | Mismatched row counts are refused. | Verifies merging a `3x5` matrix into a `2x5` one fails. | -| `test_msgpack_round_trip` | The round trip preserves the matrix geometry. | Round-trips a `2x3` matrix at `k` `200` fed `"key1"` at `5.0` and `"key2"` at `10.0` and verifies the decode reports `rows` `2` and `cols` `3`; the per-cell KLL state is not checked. | -| `test_aggregate_hydrakll` | `aggregate_hydrakll` builds a decodable matrix from parallel key and value arrays. | For `aggregate_hydrakll(2, 5, 200, ["a", "b", "a"], [1.0, 2.0, 3.0])`, verifies the returned bytes decode and report `rows` `2` and `cols` `5`; no quantile is checked. | -| `test_aggregate_hydrakll_empty` | An empty stream produces nothing. | Verifies `aggregate_hydrakll(2, 5, 200, &[], &[])` returns `None`. | - -### Portable KllSketch - -Test file: [`src/message_pack_format/portable/kll.rs`](../src/message_pack_format/portable/kll.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `kll_data_msgpack_matches_kll_sketch` | The DTO codec is byte-identical to the facade's. | For a `k` `200` sketch fed the 500 values `0.0..=499.0`, verifies `KllSketch::to_msgpack` equals `KllSketchData { k, sketch_bytes }::to_msgpack` and that the DTO round-trips with `k` and `sketch_bytes` unchanged. | -| `test_kll_creation` | A fresh sketch is empty and keeps its `k`. | For `KllSketch::new(200)`, verifies `count()` is `0` and `k` is `200`. | -| `test_kll_update` | Every insert is counted. | Feeds `10.0`, `20.0`, and `15.0` into a `k` `200` sketch and verifies `count()` is `3`. | -| `test_kll_quantile` | The extremes are exact and the median brackets the true one. | For a `k` `200` sketch fed `1.0..=10.0`, verifies `quantile(0.0)` is `1.0`, `quantile(1.0)` is `10.0`, and the median falls in `5.0..=6.0`. | -| `test_kll_merge` | Merge conserves the count and the extremes. | Merges a `k` `200` sketch fed `6.0..=10.0` into one fed `1.0..=5.0` and verifies `count()` is `10`, `quantile(0.0)` is `1.0`, and `quantile(1.0)` is `10.0`. | -| `test_msgpack_round_trip` | The round trip preserves `k`, the count, and the extremes. | Round-trips a `k` `200` sketch fed `1.0..=5.0` and verifies the decode reports `k` `200`, `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | -| `test_aggregate_kll` | `aggregate_kll` builds a decodable sketch from a value array. | For `aggregate_kll(200, [1.0, 2.0, 3.0, 4.0, 5.0])`, verifies the returned bytes decode to a sketch with `count()` `5`, `quantile(0.0)` `1.0`, and `quantile(1.0)` `5.0`. | -| `test_aggregate_kll_empty` | An empty value array produces nothing. | Verifies `aggregate_kll(200, &[])` returns `None`. | - -### Portable Sampling Rescale - -Test file: [`src/message_pack_format/portable/sampling.rs`](../src/message_pack_format/portable/sampling.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `dual_read_zero_means_one` | An unset `sample_p` reads as no sampling. | Verifies both `effective_sample_p` over an envelope carrying `sample_p` `0.0` and `sample_p_or_default(0.0)` return `1.0`. | -| `out_of_range_falls_back_to_one` | A malformed probability falls back to no sampling. | Verifies `sample_p_or_default` returns `1.0` for `-0.5`, for `1.5`, and for `f64::NAN`. | -| `valid_p_passes_through` | A probability inside `(0, 1]` is used as given. | Verifies `effective_sample_p` over an envelope carrying `sample_p` `0.1` returns `0.1`, and that `sample_p_or_default(1.0)` returns `1.0`. | -| `rescale_count_inverts_probability` | A count-like estimate is rescaled by `1/p`. | Verifies `rescale_count(10_000.0, 0.1)` is `100_000.0` within `1e-6`, that `rescale_count(123.0, 1.0)` is `123.0`, and that a `p` of `0.0` returns `123.0` rather than dividing by zero. | -| `rescale_with_env` | The envelope's own `sample_p` drives the rescale. | Verifies `rescale_count_with_env(5_000.0, env)` at `sample_p` `0.05` is `100_000.0` within `1e-6`, and that at `sample_p` `0.0` an input of `42.0` comes back unchanged. | -| `quantiles_are_scale_invariant` | The quantile family is declared to need no rescale. | Asserts `is_quantile_scale_invariant()`, a `const fn` returning `true`; nothing about a quantile estimate is computed. | - -### Portable SetAggregator - -Test file: [`src/message_pack_format/portable/set_aggregator.rs`](../src/message_pack_format/portable/set_aggregator.rs) - -Wire tests: [`src/message_pack_format/portable/set_aggregator/wire.rs`](../src/message_pack_format/portable/set_aggregator/wire.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `test_creation` | A fresh aggregator holds nothing. | Verifies `SetAggregator::new()` has an empty `values` set. | -| `test_insert` | A repeated key is inserted once. | Feeds `"web"`, `"api"`, and `"web"` again and verifies `values` holds 2 keys, both `"web"` and `"api"`. | -| `test_merge` | Merge is set union. | Merges an aggregator holding `{"api", "db"}` into one holding `{"web", "api"}` and verifies the result holds exactly `"web"`, `"api"`, and `"db"`. | -| `test_msgpack_round_trip` | The set survives the round trip. | Round-trips an aggregator holding `{"web", "api"}` and verifies the decode holds 2 keys, both names present. | -| `test_msgpack_matches_wire_format` | The bytes are a named map under a `values` key. | For an aggregator holding `{"a"}`, verifies the bytes decode into a locally declared `StringSet { values: HashSet }` that still contains `"a"`. | -| `set_aggregator_rejects_unordered_and_duplicate_members` | `values` must be strictly ascending by UTF-8 bytes. | Verifies crafted payloads `["b", "a"]` and `["a", "a"]` each fail to decode. | -| `a_bin_member_is_rejected` | A member must be msgpack `str`. | Verifies a set payload whose one member is the `bin` `b"a"`, and a delta whose `removed` holds that `bin`, each fail to decode. | -| `metadata_must_be_exactly_metadata_version_1` | The metadata map is exactly `{metadata_version: 1}`. | Verifies a map with an extra `bogus_field` key, and one with `metadata_version` `2`, each fail to decode. | -| `each_decoder_rejects_a_foreign_kind_id` | Each decoder accepts only its own kind_id. | Verifies a well-formed set payload framed under `0x08 0x80` fails as a `SetAggregator`, and a well-formed delta payload under `0x08 0x00` or the retired `0x09 0x00` fails as a `SetAggregatorDelta`. | -| `delta_result_rejects_a_member_both_added_and_removed` | `added` and `removed` are disjoint on both sides of the wire. | Verifies a crafted delta with `added` `["a", "b"]` and `removed` `["b"]` fails to decode, and that a `SetAggregatorDelta` holding `"a"` in both sets fails to serialize. | -| `delta_result_rejects_unordered_sides` | Each side is strictly ascending. | Verifies a delta with `added` `["b", "a"]`, and one with `removed` `["c", "c"]`, each fail to decode. | - ## Common ### Common Hash Utilities @@ -1680,7 +1491,7 @@ Test file: [`src/common/structures/matrix_storage.rs`](../src/common/structures/ ### ASAPv1 Envelope (MessagePack) -Test file: [`src/message_pack_format/envelope.rs`](../src/message_pack_format/envelope.rs) +Test file: [`src/asapv1/envelope.rs`](../src/asapv1/envelope.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | @@ -1690,7 +1501,7 @@ Test file: [`src/message_pack_format/envelope.rs`](../src/message_pack_format/en ### ASAPv1 Admission Rate -Test file: [`src/message_pack_format/admission_rate.rs`](../src/message_pack_format/admission_rate.rs) +Test file: [`src/asapv1/admission_rate.rs`](../src/asapv1/admission_rate.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | @@ -1738,14 +1549,6 @@ Test file: [`tests/e2e/frequency.rs`](../tests/e2e/frequency.rs) | `count_min_sketch_fold_and_count_sketch_fold_counts_and_hierarchical_merge` | Weighted counts are exact on sparse dimensions and survive a same-level merge, a signed update, and a hierarchical merge. | On `CountMinSketchFold::::new(3, 2048, 0, 32)` sketches: 100 inserts of `Str("alpha")` at weight 2 into each of two sketches plus 100 of `Str("beta")` at weight 1 into the second, verifying `query` reads `200`, `200` and `100`; after `merge_same_level`, `400` for alpha and `100` for beta. On `CountSketchFold::::new(3, 2048, 0, 32)`, `Str("gamma")` at `+60` then `-20` is verified to query `40`. Finally `CountMinSketchFold::hierarchical_merge` of two sketches built by `build_fold_cms` (30 and 45 inserts at weight 3) is verified to query `90` and `135`. | | `folded_sketches_keep_their_own_bounds_through_a_sixteen_way_merge` | Folded sketches keep their own theorems, evaluated at the folded width, through a sixteen-way hierarchical merge. | Splits `zipf_u64(240_000, 10_000, 1.1, 1009)` into 16 windows of 15,000, giving each window a `CountMinSketchFold` and a `CountSketchFold` of `3x4096` at `fold_level = 4` and `top_k = 20`, then `hierarchical_merge`s each family. Asserts `CountMinSpec::new(3, 256)` over `CountMinSketchFold::estimate` and `CountSketchSpec::new(3, 256)` over `CountSketchFold::estimate` - the bounds are evaluated at the folded width `4096 >> 4 = 256`, not the unfolded 4096 - with the specs' usual rules: zero violations for one-sidedness and both simultaneous bounds at `delta = 1e-3`, and the marginal bounds as rate pins at `e^-3` and `P[Bin(3, 1/3) >= 2]`. | -### Frequency: Portable Count-Min and Count Sketch - -Test file: [`tests/e2e/frequency.rs`](../tests/e2e/frequency.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `portable_cms_and_cs_string_keys_satisfy_their_own_bounds` | The portable wire twins satisfy their own theorems over string keys with no extra slack. | Feeds `zipf_u64(50_000, 2048, 1.1, 1006)` as string keys `"k{k}"` through `update(&key, 1.0)` into `portable::countminsketch::CountMinSketch::new(3, 4096)` and `portable::countsketch::CountSketch::new(5, 4096)`, then asserts `CountMinSpec::new(3, 4096).assert_contract` over the portable Count-Min's `estimate` and `CountSketchSpec::new(5, 4096).assert_contract` over the portable Count Sketch's `estimate`, against a `FreqTruth` keyed on the underlying `i64` identities. | - ### Frequency: Documented Input Matrix Test file: [`tests/e2e/frequency.rs`](../tests/e2e/frequency.rs) @@ -1764,13 +1567,12 @@ Test file: [`tests/e2e/cardinality.rs`](../tests/e2e/cardinality.rs) | --- | --- | --- | | `hll_{classic,ertl_mle,hip}_p{12,14,16}_satisfies_..._error_model` | Each core HLL alias tracks the true cardinality inside its own estimator's `z = 4` band, ignores replayed identities, and reproduces a single pass from disjoint shards. | One `hll_battery!` body per invocation, over `CHECKPOINTS = [10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]`. A fresh sketch per checkpoint is fed `DataInput::U64(base + k)` from its own namespace `base = (1 << 40) * (i + 1)`, and each `estimate()` goes through `CardinalityConfidenceSpec::check` at `z = 4.0`: relative band `4 * 1.04/sqrt(m)` for the `hll` model, `4 * sqrt(ln 2 / m)` for `hll_hip`. Then re-inserting the first `min(target, 200_000)` already-seen identities must leave `estimate()` bit-identical (`assert_eq!`). Then over a fresh `200_000`-identity namespace, an even/odd shard merge must give `registers_as_slice()` and `estimate()` equal to the single pass. Invocations: `hll_classic_p12_satisfies_its_register_error_model` (`HyperLogLogP12`, `hll`, mergeable), `hll_classic_p14_satisfies_its_register_error_model` (`HyperLogLogP14`), `hll_classic_p16_satisfies_its_register_error_model` (`HyperLogLogP16`), `hll_ertl_mle_p12_satisfies_the_cramer_rao_error_model` (`HyperLogLogP12`), `hll_ertl_mle_p14_satisfies_the_cramer_rao_error_model` (`HyperLogLogP14`), `hll_ertl_mle_p16_satisfies_the_cramer_rao_error_model` (`HyperLogLogP16`), `hll_hip_p12_satisfies_the_hip_error_model` (`HyperLogLogHipP12`, `hll_hip`, not_mergeable), `hll_hip_p14_satisfies_the_hip_error_model` (`HyperLogLogHipP14`), `hll_hip_p16_satisfies_the_hip_error_model` (`HyperLogLogHipP16`). The three `not_mergeable` HIP invocations still build and fill the three `200_000`-identity shard sketches but assert nothing about them. | | `hll_{classic,ertl_mle,hip}_custom_p{10..18}_satisfies_..._error_model` | Every custom register geometry from p10 to p18 holds its estimator's `z = 4` band over its own checkpoint grid, is unmoved by a replay, and merges shard-for-shard. | The same `hll_battery!` body as the core aliases (fresh sketch per checkpoint on namespace `(1 << 40) * (i + 1)`, `CardinalityConfidenceSpec::check` at `z = 4.0`, replay of `min(target, 200_000)` identities asserted bit-identical, `200_000`-identity even/odd merge asserted equal to the single pass register for register), but over `HyperLogLogImpl` (`hll`, mergeable), `HyperLogLogImpl` (`hll`, mergeable) and `HyperLogLogHipImpl` (`hll_hip`, not_mergeable, so its merge block asserts nothing), each with the precision's own checkpoints: p10 `[100, 1_000, 20_000, 200_000]` for `hll_classic_custom_p10_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p10_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p10_satisfies_the_hip_error_model`; p11 `[200, 2_000, 40_000, 200_000]` for `hll_classic_custom_p11_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p11_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p11_satisfies_the_hip_error_model`; p12 `[400, 4_000, 50_000, 300_000]` for `hll_classic_custom_p12_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p12_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p12_satisfies_the_hip_error_model`; p13 `[1_000, 10_000, 100_000, 500_000]` for `hll_classic_custom_p13_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p13_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p13_satisfies_the_hip_error_model`; p14 `[2_000, 16_000, 200_000, 800_000]` for `hll_classic_custom_p14_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p14_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p14_satisfies_the_hip_error_model`; p15 `[4_000, 32_000, 300_000, 1_000_000]` for `hll_classic_custom_p15_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p15_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p15_satisfies_the_hip_error_model`; p16 `[5_000, 60_000, 500_000, 1_200_000]` for `hll_classic_custom_p16_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p16_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p16_satisfies_the_hip_error_model`; p17 `[10_000, 100_000, 800_000, 1_500_000]` for `hll_classic_custom_p17_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p17_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p17_satisfies_the_hip_error_model`; p18 `[10_000, 100_000, 1_500_000]` for `hll_classic_custom_p18_satisfies_its_register_error_model`, `hll_ertl_mle_custom_p18_satisfies_the_cramer_rao_error_model`, `hll_hip_custom_p18_satisfies_the_hip_error_model`. | -| `portable_hll_variants_and_precisions_satisfy_the_register_error_model` | Every portable wire variant and precision holds the register error model and merges by register-wise maximum. | For each of `HllVariant::Regular`, `HllVariant::Datafusion`, `HllVariant::Hip` crossed with precisions `12`, `14`, `16`, draws its own stream `uniform_u64(200_000, u64::MAX / 4, seed)` with `seed = 2001 + (v * 3 + p)`, feeds `HllSketch::update` with `to_be_bytes()`, and checks `HllSketch::estimate()` against the `HashSet` distinct count via `CardinalityConfidenceSpec::hll(precision, 4.0)` — all three tags are held to `4 * 1.04/sqrt(m)`, since `estimate()` runs the same register formula for each. Then `merge` of a second sketch built over the same distinct identities must leave `estimate()` unchanged, and an even/odd shard merge of the same stream must give exactly that same `estimate()`. | | `hll_accuracy_improves_with_precision_as_the_error_model_predicts` | A larger precision actually buys accuracy, and the measured RSE stays near the predicted `1.04/sqrt(m)`. | Measures the RMS relative error of `HyperLogLogP12`, `HyperLogLogP14` and `HyperLogLogP16` over `BLOCKS = 6` disjoint blocks of `N = 1_000_000` identities (`DataInput::U64(b * N + i)`), asserts each measured RSE is `<= 2.0 *` `CardinalityConfidenceSpec::hll(p, 4.0).sigma_rel()`, and asserts the p12 RSE is at least `2.0x` the p16 RSE; the model's predicted 4x improvement is not asserted, only 2x. | | `hll_classic_switchover_band_stays_within_the_documented_empirical_band` | The Classic estimator's error at the linear-counting switchover stays inside a 1.5x-to-10x empirical band around the asymptotic RSE. | For `HyperLogLogP12/P14/P16` at `n = 2.5m` (`10_240`, `40_960`, `163_840`), computes the RMS relative error over `BLOCKS = 6` disjoint blocks (identities `b * 20_000_000 + i`) and forms `ratio = rse / (1.04/sqrt(m))` from `CardinalityConfidenceSpec::hll(p, 4.0).sigma_rel()`; asserts `ratio <= 10.0` and `ratio >= 1.5`, so both a widened cliff and a vanished one fail. | | `set_aggregator_union_is_exact` | A `SetAggregator` union keeps every member exactly. | Feeds `uniform_u64(20_000, 500, 2002)` as `member-{k}` strings into `SetAggregator::update`, merges a second aggregator holding `extra-a` and `extra-b`, then asserts `agg.values.len()` equals the length of a parallel `HashSet` of the same members and that `agg.values` contains every one of them; the distinct count is compared against the in-test truth set rather than a pinned number. | | `custom_precision_accuracy_improves_with_precision_as_the_error_model_predicts` | A higher custom precision does not lose to a lower one on the same stream. | Feeds the same `N = 200_000` identities from `BASE = (1 << 40) * 41` into `HyperLogLogImpl`, `` and `` and asserts only the two orderings `e18 <= e10` and `e18 <= e13` on absolute relative error; no error band is asserted, p10 against p13 is not compared, and with one trial per precision no error ratio is pinned. | | `a_custom_precision_merge_reproduces_the_single_pass_registers_for_every_estimator` | A shard merge at a custom precision reproduces the single pass exactly, for both register estimators. | Over `N = 120_000` identities from `BASE = (1 << 40) * 42` split even/odd by `k % 2`, builds single/even/odd `HyperLogLogImpl` and `HyperLogLogImpl`, calls `merge`, and asserts for each estimator that the merged `registers_as_slice()` equals the single pass's and that `estimate()` is equal too. | -| `a_set_aggregator_delta_describes_the_change_and_survives_the_wire` | A `SetAggregatorDelta` over two set snapshots survives the MessagePack wire and replays into the later snapshot. | Builds `SetAggregator`s over `["web", "api", "db", "cache"]` and `["web", "api", "queue"]`, forms a `SetAggregatorDelta` from the two `HashSet::difference` results, and asserts `added == {"queue"}` and `removed == {"db", "cache"}` (these two pin the test's own set arithmetic, not a library call); then `to_msgpack` / `SetAggregatorDelta::from_msgpack` must return both sets unchanged, applying the decoded delta to the earlier aggregator must reproduce `after.values`, and an all-empty `SetAggregatorDelta` must round-trip still empty. | +| `a_set_aggregator_delta_describes_the_change_and_survives_the_wire` | A `SetAggregatorDelta` over two set snapshots survives ASAPv1 and replays into the later snapshot. | Builds `SetAggregator`s over `["web", "api", "db", "cache"]` and `["web", "api", "queue"]`, forms a `SetAggregatorDelta` from the two `HashSet::difference` results, and asserts `added == {"queue"}` and `removed == {"db", "cache"}` (these two pin the test's own set arithmetic, not a library call); then `to_asapv1` / `SetAggregatorDelta::from_asapv1` must return both sets unchanged, applying the decoded delta to the earlier aggregator must reproduce `after.values`, and an all-empty `SetAggregatorDelta` must round-trip still empty. | ### Cardinality: Duplicate-Heavy Zipf Streams @@ -1830,12 +1632,12 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `ddsketch_core_and_portable_satisfy_the_relative_value_error_contract` | Both DDSketch implementations meet the relative-value-error guarantee against the order statistic of their own rank convention. | Crosses `alpha in {0.001, 0.01, 0.05, 0.1}` x `n in {1_000, 20_000, 100_000}` x the six `dds_streams` shapes (`adversarial-bucket-edges` log-uniform on `gamma` edges, positive-filtered `normal(1000, 250)`, `exponential(1e-3)`, `uniform` over `[1e6, 1e7)`, `zipf(1.1)` over 8192 keys in `[1e6, 1e7]`, and `wide-dynamic-range` spanning nine decades). Asserts `DdSketch::get_count()` and `PortableDds::total_count()` both equal `n` (no sample dropped), then per alpha tallies `RelativeQuantileSpec::core(alpha)` on `get_value_at_quantile` and `RelativeQuantileSpec::portable(alpha)` on `quantile` over the q grid `[0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0]` and closes both with `assert_none`: every estimate must be within `alpha + 8*f64::EPSILON*(1 + \|ln v\|)` of `sorted[ceil(q*n) - 1]` for the core and `sorted[floor(q*(n-1))]` for the portable twin. Zero violations tolerated, since the guarantee is deterministic. | -| `ddsketch_core_and_portable_answer_different_order_statistics` | The core and portable quantile conventions differ, and the divergence is pinned rather than absorbed into `alpha`. | Runs 8 `(n, q, core_idx, port_idx)` probes — `(3, 0.4, 1, 0)`, `(4, 0.34, 1, 1)`, `(4, 0.3, 1, 0)`, `(7, 0.2, 1, 1)`, `(7, 0.6, 4, 3)`, `(5, 0.5, 2, 2)`, `(10, 0.25, 2, 2)`, `(10, 0.15, 1, 1)` — on values `100 * gamma^(3i)` at `ALPHA = 0.01`, so one rank of movement is a whole bucket. Asserts `DdRankConvention::CeilNearestRank.index(n, q)` equals `core_idx` and `DdRankConvention::LowerFloor.index(n, q)` equals `port_idx`, that each implementation satisfies its own `RelativeQuantileSpec::check` against `sorted[its own index]`, that the two answers are unequal on every probe whose indices disagree, and that at least `3` probes disagree. | -| `ddsketch_core_endpoints_are_exact_and_portable_endpoints_are_alpha_relative` | The core sketch's endpoints are exact while the portable twin's are only `alpha`-relative. | For each of `alpha in {0.001, 0.01, 0.05, 0.1}` over 5_000 `log_uniform_f64` values (seed `4_242`, exponent range `3..30`), asserts `core.get_value_at_quantile(0.0) == Some(truth.min())`, `(1.0) == Some(truth.max())`, and `core.min()`/`core.max()` equal the exact extremes; the portable `quantile(0.0)` and `quantile(1.0)` are only checked through `RelativeQuantileSpec::portable(alpha).check` against the true min/max, i.e. within `alpha + a few ULP`, with no exactness claimed. | -| `ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries` | Values on, just below and just above a bucket edge all stay inside `alpha` on both sides. | For each of the four alphas and `k in {-40, -7, 0, 1, 13, 60, 200}` computes `edge = gamma^k` and probes `edge`, `edge*(1 - f64::EPSILON)`, `edge*(1 + f64::EPSILON)`, `edge*sqrt(gamma)` and `edge*gamma*(1 - 1e-12)`. Each probe is the single sample of a fresh core and portable sketch, so both rank conventions land on index 0 and the mapping is isolated; `quantile(0.5)` must satisfy each spec's `alpha + numerical_slack` tolerance against the probe itself. Probes the mapping refuses (`get_count() == 0`) are skipped rather than asserted on. | -| `ddsketch_merge_and_delta_replay_preserve_the_relative_error_contract` | Shard merge and bucket-delta replay both leave the relative-error contract intact on both implementations. | Per `alpha in {0.001, 0.01, 0.05, 0.1}` over `zipf_f64(40_000, 8_192, 1.1, 1e3, 1e7)`: a 4-way round-robin core shard merge must report `get_count() == 40_000` exactly and satisfy the contract over the full q grid; a core sketch rebuilt by replaying every non-zero `positive_store_counts()` entry as an `octo_delta::DdSketchBucketDelta` at `positive_store_offset() + i` must reproduce `get_count()` and satisfy the contract over the interior grid `DDS_QS[1..6] = [0.01, 0.1, 0.5, 0.9, 0.99]` (its endpoints are bucket representatives, so `q=0`/`q=1` are excluded). On the portable side, an even/odd split merged with `merge` is scored over the full grid, and a `DdSketchDelta` carrying the merged sketch's buckets plus `d_count` replayed into an empty sketch is scored over the interior grid. All four tallies use `assert_none`. | -| `ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_parameters` | The relative-value contract still holds at accuracy parameters three orders of magnitude either side of the usual range. | For `alpha in {1e-5, 1e-4, 0.3, 0.5, 0.9}` (seed `0x0DDA_0000 + i*7919`), takes the six `dds_streams(alpha, 20_000, seed)` shapes filtered to `[min, max]` from `ddsketch::ddsketch_indexable_bounds(alpha)`, skipping any shape left with fewer than 100 values. Asserts core `get_count()` and portable `total_count()` both equal the surviving count, then `assert_none` on `RelativeQuantileSpec::core(alpha)` and `::portable(alpha)` over the 7-point `DDS_QS` grid. | +| `ddsketch_satisfies_the_relative_value_error_contract` | DDSketch meets the relative-value-error guarantee against the order statistic of its rank convention. | Crosses `alpha in {0.001, 0.01, 0.05, 0.1}` x `n in {1_000, 20_000, 100_000}` x the six `dds_streams` shapes (`adversarial-bucket-edges` log-uniform on `gamma` edges, positive-filtered `normal(1000, 250)`, `exponential(1e-3)`, `uniform` over `[1e6, 1e7)`, `zipf(1.1)` over 8192 keys in `[1e6, 1e7]`, and `wide-dynamic-range` spanning nine decades). Asserts `DdSketch::get_count()` equals `n` (no sample dropped), then per alpha tallies `RelativeQuantileSpec::core(alpha)` on `get_value_at_quantile` over the q grid `[0.0, 0.01, 0.1, 0.5, 0.9, 0.99, 1.0]` and closes with `assert_none`: every estimate must be within `alpha + 8*f64::EPSILON*(1 + \|ln v\|)` of `sorted[ceil(q*n) - 1]`. Zero violations tolerated, since the guarantee is deterministic. | +| `ddsketch_answers_the_ceil_nearest_rank_order_statistic` | `get_value_at_quantile` answers the `ceil(q*n)` order statistic. | Runs 8 `(n, q, idx)` probes — `(3, 0.4, 1)`, `(4, 0.34, 1)`, `(4, 0.3, 1)`, `(7, 0.2, 1)`, `(7, 0.6, 4)`, `(5, 0.5, 2)`, `(10, 0.25, 2)`, `(10, 0.15, 1)` — on values `100 * gamma^(3i)` at `ALPHA = 0.01`, so one rank of movement is several buckets. Asserts `DdRankConvention::CeilNearestRank.index(n, q)` equals `idx` and that the answer satisfies `RelativeQuantileSpec::core(ALPHA).check` against `sorted[idx]`. | +| `ddsketch_endpoints_are_exact` | The endpoints are the exact stored minimum and maximum. | For each of `alpha in {0.001, 0.01, 0.05, 0.1}` over 5_000 `log_uniform_f64` values (seed `4_242`, exponent range `3..30`), asserts `get_value_at_quantile(0.0) == Some(truth.min())`, `(1.0) == Some(truth.max())`, and `min()`/`max()` equal the exact extremes. | +| `ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries` | Values on, just below and just above a bucket edge all stay inside `alpha`. | For each of the four alphas and `k in {-40, -7, 0, 1, 13, 60, 200}` computes `edge = gamma^k` and probes `edge`, `edge*(1 - f64::EPSILON)`, `edge*(1 + f64::EPSILON)`, `edge*sqrt(gamma)` and `edge*gamma*(1 - 1e-12)`. Each probe is the single sample of a fresh sketch, so the mapping is isolated from any rank effect; `get_value_at_quantile(0.5)` must satisfy `alpha + numerical_slack` against the probe itself. Probes the mapping refuses (`get_count() == 0`) are skipped rather than asserted on. | +| `ddsketch_merge_and_delta_replay_preserve_the_relative_error_contract` | Shard merge and bucket-delta replay both leave the relative-error contract intact. | Per `alpha in {0.001, 0.01, 0.05, 0.1}` over `zipf_f64(40_000, 8_192, 1.1, 1e3, 1e7)`: a 4-way round-robin shard merge must report `get_count() == 40_000` exactly and satisfy the contract over the full q grid; a sketch rebuilt by replaying every non-zero `positive_store_counts()` entry as an `octo_delta::DdSketchBucketDelta` at `positive_store_offset() + i` must reproduce `get_count()` and satisfy the contract over the interior grid `DDS_QS[1..6] = [0.01, 0.1, 0.5, 0.9, 0.99]` (its endpoints are bucket representatives, so `q=0`/`q=1` are excluded). Both tallies use `assert_none`. | +| `ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_parameters` | The relative-value contract still holds at accuracy parameters three orders of magnitude either side of the usual range. | For `alpha in {1e-5, 1e-4, 0.3, 0.5, 0.9}` (seed `0x0DDA_0000 + i*7919`), takes the six `dds_streams(alpha, 20_000, seed)` shapes filtered to `[min, max]` from `ddsketch::ddsketch_indexable_bounds(alpha)`, skipping any shape left with fewer than 100 values. Asserts `get_count()` equals the surviving count, then `assert_none` on `RelativeQuantileSpec::core(alpha)` over the 7-point `DDS_QS` grid. | | `ddsketch_bucket_width_widens_monotonically_with_the_accuracy_parameter` | A looser accuracy parameter never needs more buckets. | Feeds 20_000 values `1.0 + uniform_u64(.., 1_000_000, 0x0DDA_9001)` into a `DdSketch` at each `alpha in {1e-5, 1e-4, 0.3, 0.5, 0.9}` in that order, asserts nothing was dropped (`get_count() == 20_000`), and asserts the occupied-bucket count (`positive_store_counts()` entries above zero) is no larger than the previous alpha's. The check is non-strict, so equal bucket counts pass and no bucket width is measured directly. | ### Quantiles: DDSketch Input Rejection @@ -1844,8 +1646,7 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `ddsketch_rejects_untrackable_values_and_mapping_mismatches` | Untrackable values are dropped without allocating or corrupting state, and a mismatched mapping is a runtime error on both types. | At `alpha = 0.01` feeds `NAN`, `-INFINITY`, `INFINITY`, `-5.0`, `0.0`, `f64::MIN_POSITIVE`, `5e-324`, `f64::MAX` and `1e308` into both implementations and asserts `get_count() == 0`, `total_count() == 0`, `port.positive_store_counts.len() < 10_000`, and `port.quantile(0.5) == None`. Then `[1.0, 2.0, 4.0, 8.0, 16.0]` must give `get_count() == 5` / `total_count() == 5` with the store between `1` and `512` buckets. Using the shared `ddsketch_indexable_bounds(0.01)`, `min*(1 + 1e-9)` and `max*(1 - 1e-9)` are both accepted (count `2` in both, and the two counts equal), while `min*0.5` and `max*(1 + 1e-6)` leave the counts at `2`. `DdSketch::merge` and `PortableDds::merge` across `alpha = 0.01` vs `0.05` must both return `Err`. Finally at `alpha = 1e-9`, `1e-300` is dropped by both (`total_count() == 0`, `get_count() == 0`) with `positive_store_counts.len() == 0`, so no allocation occurs for a rejected value. | -| `portable_ddsketch_rejects_hostile_delta_spans` | A delta spanning an implausible bucket range is rejected before any allocation, and a benign one still applies. | Seeds a portable sketch at `alpha = 0.01` with `1.0`, then asserts `apply_delta` on `buckets: [(i32::MAX - 1, 1), (7, 3)]` returns `Err` and leaves both `positive_store_counts.len()` and `positive_store_offset` at their prior values. A benign delta `[(6, 5)]` then applies successfully and `quantile(0.9)` must equal exactly `Some(gamma.powf(6.0) * (1.0 + alpha))`, so the applied count is visible through queries. | +| `ddsketch_rejects_untrackable_values_and_mapping_mismatches` | Untrackable values are dropped, and a mismatched mapping is a runtime error. | At `alpha = 0.01` feeds `NAN`, `-INFINITY`, `INFINITY`, `f64::MAX` and `1e308` and asserts `get_count() == 0`; then `[1.0, 2.0, 4.0, 8.0, 16.0]` must give `get_count() == 5`. Using the shared `ddsketch_indexable_bounds(0.01)`, `min*(1 + 1e-9)` and `max*(1 - 1e-9)` are both accepted (count `2`), `min*0.5` is counted as zero and `max*(1 + 1e-6)` is rejected (count `3`, `zero_count() == 1`). `DdSketch::merge` across `alpha = 0.01` vs `0.05` must return `Err`. Finally at `alpha = 1e-9`, `1e-300` is counted (`get_count() == 1`). | ### Quantiles: UnivMon-Q Estimators @@ -1870,7 +1671,6 @@ Test file: [`tests/e2e/quantiles.rs`](../tests/e2e/quantiles.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | | `tumbling_kll_windows_are_exact_and_answers_satisfy_the_rank_contract` | Window membership is exact arithmetic while the answers inside a window carry KLL's rank error. | Drives 4_000 uniform values over `[0, 1M)` (seed `3009`) into a `TumblingWindow` with window `400`, `16` retained, `k=200`, `m=8`, pool size `4`, and asserts `closed_count() == 9` by equality. Then 16 trials, each with its own compaction seed `kll_trial_seed(0x7717_0000 + t)`, reduce the three window views — `query_all` against the whole stream, `query_recent(1)` against the last 800 values, and `active_sketch()` against the last 400 — to one worst rank error over the q grid `[0.1, 0.25, 0.5, 0.75, 0.9]`, which must not exceed `eps(200) ~ 0.0165`; accepted by `assert_independent_binomial` at `p = 0.01`. Rotation is then checked by a band, not an equality: `query_all().count()` must drift from 4_000 by at most `eps(200)`, so a dropped window (12.5% of the stream) would fail but the retained mass is not pinned exactly. | -| `portable_hydra_kll_per_key_medians_satisfy_the_rank_characterization` | Each Hydra KLL cell answers its own key under the same rank characterization as a standalone KLL. | Builds `HydraKllSketch::with_seed(3, 256, 200, seed)` for 12 trials with prototype seeds `kll_trial_seed(0x5EED_0500 + t)`, feeding two keys of 4_000 values each in sorted order: `svc-a` from `\|normal(100, 5)\|` (seed `3010`) and `svc-b` from `\|normal(900, 45)\|` (seed `3011`). Because every cell is cloned from one seeded prototype the two keys are not independent, so each grid is reduced to its worst rank error across both keys over the q grid `[0.1, 0.25, 0.5, 0.75, 0.9]` — wider than the median the name mentions — checked against `eps(200) ~ 0.0165` and accepted by `assert_independent_binomial` at `p = 0.01` over the twelve prototype seeds. | ### Quantiles: Documented Input Matrix @@ -2082,16 +1882,6 @@ Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) | `univmonq_with_window_bound_chooses_a_hierarchy_that_satisfies_its_own_inequality` | `with_window_bound` picks the smallest level count whose deepest stratum still fits the candidate table. | For `max_updates` in `10_000`, `1_000_000` and `100_000_000` at `delta = 1e-3`, calls `UnivMonQConfig::default().with_window_bound` and re-derives the Bernstein upper tail `mean + sqrt(2*mean*ln(1/delta)) + (2/3)*ln(1/delta)` with `mean = max_updates / 2^(levels-1)`, asserting it is strictly below `cfg.candidates`; asserts minimality by requiring the same bound at `levels - 1` to be at or above `candidates` whenever `levels > 2`; asserts `levels` is non-decreasing across the three windows; builds each chosen config and asserts `count() == 1_000` after `1_000` updates; and asserts `with_window_bound(1_000, 1.5)` is an error. | | `univmonq_multi_shard_merge_with_distinct_source_ids_covers_the_union` | Shards with distinct source ids merge into a sketch whose exact aggregates cover the union. | Builds `4` shards with `UnivMonQ::with_hasher_and_source_id(UnivMonQConfig::default(), i + 1)`, deals `40_000` uniform values over `1_000_000` (seed `0xC090_5101`) round robin by `i % 4`, asserts each shard's `source_id()` is `i + 1`, merges the other three into the first with `merge` returning `Ok`, and asserts the merged `count()` is `40_000` and `min()`/`max()` equal the whole stream's exact extremes. Only the distinct-id path is exercised: no assertion covers a merge of shards that share a source id, and no estimator is checked. | -### Composition: Portable Sketch-With-Heap Facades - -Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `portable_count_min_with_heap_satisfies_the_count_min_bound_through_merge_and_wire` | The portable Count-Min-plus-heap holds Count-Min's bound and an agreeing heap through a merge and a MessagePack round trip. | Feeds `40_000` Zipf(1.1) keys over a `2_048` domain (seed `0xC090_5101`) as `"k{key}"` strings into `CountMinSketchWithHeap::new(3, 4_096, 32)`, plus even/odd halves into two more. Runs `CountMinSpec::new(3, 4_096).assert_contract` on the single sketch, asserts every `topk_heap_items()` entry's `value` equals `estimate(&item.key)` with zero violations, re-runs the contract on `CountMinSketchWithHeap::merge_refs(&[&left, &right])` against the full truth, and after `to_msgpack`/`from_msgpack` asserts the decoded sketch's `estimate` is exactly equal to the original's for every observed key and still satisfies the contract. | -| `portable_count_sketch_with_heap_satisfies_the_l2_bound_through_merge_and_wire` | The portable Count-Sketch-plus-heap holds the L2 bound and an agreeing heap through a merge and a MessagePack round trip. | Feeds the same `40_000`-key Zipf(1.1) stream (domain `2_048`, seed `0xC090_5101`) as `"k{key}"` strings into `CountSketchWithHeap::new(5, 4_096, 32)`, plus even/odd halves into two more, and runs `CountSketchSpec::new(5, 4_096).assert_contract` (simultaneous L2 band at `delta = 1e-3` with zero violations, marginal `sqrt(3/4096)*\|\|f_-i\|\|_2` rate pinned at `P[Bin(5, 1/3) >= 3] = 51/243`) on the single sketch, on `merge_refs(&[&left, &right])` and on the `from_msgpack` decode; every `topk_heap_items()` entry's `value` must equal `estimate(&item.key)`. Unlike the Count-Min twin, the wire round trip is only re-checked against the band - no per-key equality across the encode/decode is asserted. | -| `portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_and_wire` | The portable KLL facade holds the rank-error characterization single-pass and after a two-shard merge, and its wire round trip is bit-for-bit. | Over `12` compaction seeds `0x5EED_0400 + t * 0x9E37_79B9_7F4A_7C15`, builds `KllSketch::with_seed(200, seed)` plus even/odd shards at `seed ^ 0xAAAA` and `seed ^ 0x5555`, feeds all of them `40_000` uniform values over `1_000_000` (seed `0xC090_5101`), and merges the shards with `merge`. Each seed records two trials - the single-pass sketch and the merged pair - of the worst normalized rank error over the grid `[0.1, 0.25, 0.5, 0.75, 0.9]` against `KllRankSpec::datasketches(200)`'s `eps = 2.446/200^0.9433 = 0.0165`, and the `24` outcomes are accepted by `assert_independent_binomial` at `p = 0.01`. Per seed it also asserts `to_msgpack`/`from_msgpack` preserves `k() == 200`, `count()`, and every grid quantile exactly, and that merging a `k = 400` sketch into the single-pass one errs. | - ### Composition: Documented Input Matrix Test file: [`tests/e2e/composition.rs`](../tests/e2e/composition.rs) @@ -2286,11 +2076,11 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `ehsketchlist_cm_i32_fast_2x4_matches_golden` | An EhBucketSketch Count-Min serializes to its golden bytes. | Verifies `EhBucketSketch::CountMinSketch` over `[[127, -1, 0, 65536], [-32768, 128, i32::MAX, i32::MIN]]` serializes to `asapv1_golden/ehsketchlist_cm_i32_fast_2x4.hex`, and that the decode holds those cells and re-serializes identically. | | `ehsketchlist_ddsketch_a001_matches_golden` | An EhBucketSketch DDSketch carries the DDSketch's own blocks. | Verifies `EhBucketSketch::DdSketch` over the decoded `ddsketch_signed_a001` fixture serializes to `asapv1_golden/ehsketchlist_ddsketch_a001.hex`, and that the decode's inner DDSketch serializes to `ddsketch_signed_a001` and the whole re-serializes identically. | | `hll_classic_p12_admission_matches_golden` | A sampled HLL state closes its metadata with `admission_rate`. | Verifies the `hll_classic_p12` state with `with_admission_rate(0.5)` serializes to `asapv1_golden/hll_classic_p12_admission.hex`, and that decoding the fixture recovers the P12 register pattern, `admission_rate()` `0.5`, and a byte-identical re-serialization. | -| `hll_classic_p12_matches_golden` | A known P12 Classic register pattern serializes to its golden bytes on both the portable and native paths. | Builds `HllSketch::from_raw(HllVariant::Regular, 12, regs, 0.0, 0.0, 0.0)` where `regs` is 4,096 zeros with `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, verifies `to_msgpack()` equals `asapv1_golden/hll_classic_p12.hex`, that `HyperLogLogP12::::from_asapv1` on the fixture recovers those registers and re-serializes identically, and that `HllSketch::from_msgpack` reports the same registers, `precision` `12` and `HllVariant::Regular`. | +| `hll_classic_p12_matches_golden` | The P12 Classic golden decodes to its known register pattern and re-serializes identically. | Verifies `HyperLogLogP12::::from_asapv1` on `asapv1_golden/hll_classic_p12.hex` gives 4,096 registers that are zero except `[0] = 1`, `[1] = 7`, `[100] = 42` and `[4095] = 3`, and that `to_asapv1` reproduces the fixture. | | `hll_classic_p14_delta_matches_golden` | A known P14 Classic delta serializes to its golden bytes and merges by `max`. | Verifies `HyperLogLogDelta::::new` over indices `[0, 2, 200, 8192, 16383]` and registers `[51, 1, 9, 7, 1]` serializes to `asapv1_golden/hll_classic_p14_delta.hex`, that the decoded fixture re-serializes identically, and that merged into `hll_classic_p14` it gives registers `[0] = 51`, `[1] = 7`, `[2] = 1`, `[200] = 9`, `[8192] = 42`, `[16383] = 51` and zero elsewhere. | -| `hll_ertl_mle_p12_matches_golden` | The Datafusion variant tag reaches the bytes for the same P12 register pattern. | Builds `HllSketch::from_raw(HllVariant::Datafusion, 12, regs, 0.0, 0.0, 0.0)` over the shared P12 pattern (`[0] = 1`, `[1] = 7`, `[100] = 42`, `[4095] = 3` in 4,096 registers), verifies `to_msgpack()` equals `asapv1_golden/hll_ertl_mle_p12.hex`, that `HyperLogLogP12::::from_asapv1` recovers the registers and re-serializes identically, and that `HllSketch::from_msgpack` reports `HllVariant::Datafusion`. | +| `hll_ertl_mle_p12_matches_golden` | The P12 Ertl-MLE golden decodes to the same register pattern and re-serializes identically. | Verifies `HyperLogLogP12::::from_asapv1` on `asapv1_golden/hll_ertl_mle_p12.hex` gives the shared P12 pattern and that `to_asapv1` reproduces the fixture. | | `hll_ertl_mle_p14_delta_matches_golden` | The same delta under the Ertl-MLE kind_id. | Runs the `hll_classic_p14_delta_matches_golden` checks for `Ertl` against `hll_ertl_mle_p14_delta` and `hll_ertl_mle_p14`. | -| `hll_hip_p12_matches_golden` | The HIP variant's three running scalars travel on the wire beside its registers. | Builds `HllSketch::from_raw(HllVariant::Hip, 12, regs, 1.5, 2.5, 3.0)` over the shared P12 pattern, verifies `to_msgpack()` equals `asapv1_golden/hll_hip_p12.hex`, that `HyperLogLogHipP12::from_asapv1` re-serializes the fixture identically, and that `HllSketch::from_msgpack` reports the registers, `HllVariant::Hip`, `hip_kxq0` `1.5`, `hip_kxq1` `2.5` and `hip_est` `3.0`. | +| `hll_hip_p12_matches_golden` | The P12 HIP golden round-trips. | Verifies `HyperLogLogHipP12::from_asapv1` decodes `asapv1_golden/hll_hip_p12.hex` and `to_asapv1` reproduces it. | | `kll_f64_k200_matches_golden` | A deterministic f64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `Kll::::with_k_and_seed(200, 42)` fed `1..=50` (below the level-0 capacity, so no compaction fires) serializes to `asapv1_golden/kll_f64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kll_i64_k200_matches_golden` | A deterministic i64 KLL serializes to its golden bytes, which round-trip unchanged. | Verifies `Kll::::with_k_and_seed(200, 42)` fed `1..=50` serializes to `asapv1_golden/kll_i64_k200.hex`, and that decoding the fixture re-serializes to the same bytes and reports `quantile(0.0)` `1.0` and `quantile(1.0)` `50.0`. | | `kmv_k8_matches_golden` | A KMV of known digests serializes to its golden bytes (`experimental`). | Verifies `Kmv::new(8)` fed the digests `[1, 127, 128, 65535, 4294967296, 0xfedcba9876543210]` out of order through `update_with_hash` serializes to `asapv1_golden/kmv_k8.hex`, and that the decode holds `k` `8` and those digests and re-serializes identically. | @@ -2302,30 +2092,6 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `univmon_optimized_l5_e2_3x8_1x4_h6_matches_golden` | A two-tier UnivMon pyramid serializes to its golden bytes. | Verifies `UnivMonTwoTier::new(6, 2, 3, 8, 1, 4, 5)` with counters set by `apply_delta`, the heaps `{"alpha": 40, "beta": 7, "gamma": 7}`, `{"delta": 3}`, `{}`, `{"eps": 1}`, `{}` and `bucket_size` `70` serializes to `asapv1_golden/univmon_optimized_l5_e2_3x8_1x4_h6.hex`, and that the decode reports that layout, every layer's cells and heap, and re-serializes identically. | | `univmon_q_i32_l3_c8_r3_matches_golden` | A UnivMon-Q golden decodes to its state and re-serializes byte-identically. | Verifies `asapv1_golden/univmon_q_i32_l3_c8_r3.hex` decodes to the config `levels` `3`, `width` `8`, `cols_halving_period` `1`, `depth` `3`, `counter_bits` `32`, `candidates` `2`, `ordered_samples` `3`, `seed_index` `5`, `source_id()` `7`, `count()` `12`, `min()` `-2.5` and `max()` `5.0`, and re-serializes to the same bytes. | -### MessagePack Envelope Compatibility - -Test file: [`tests/msgpack_compat.rs`](../tests/msgpack_compat.rs) - -| test_name | test_description | what_is_tested | -| --- | --- | --- | -| `count_min_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty apart from three commented-out lines sketching an `include_bytes!` of `fixtures/msgpack/count_min.msgpack` and a `CountMinSketch::from_msgpack` check, and it is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `count_min_sketch_round_trip` | A portable `CountMinSketch` survives a msgpack round trip with its geometry intact. | Updates `CountMinSketch::new(3, 64)` with `alpha` `+1.0`, `beta` `+2.0` and `alpha` `+4.0`, then verifies `from_msgpack(to_msgpack())` reports `rows` `3` and `cols` `64`; the counter matrix and the per-key estimates are not checked. | -| `count_min_sketch_with_heap_round_trip` | A portable `CountMinSketchWithHeap` survives a msgpack round trip with its geometry and heap size intact. | Updates `CountMinSketchWithHeap::new(3, 64, 8)` with `hot` `+100.0` and `warm` `+10.0`, then verifies the decode reports `rows` `3`, `cols` `64` and `heap_size` `8`; the heap's contents are not checked. | -| `count_min_wire_shape` | The `CountMinSketchWire` DTO round-trips through `rmp_serde` with its field shape intact. | Serializes `CountMinSketchWire { sketch: vec![vec![1.0, 2.0]; 3], rows: 3, cols: 2 }` with `rmp_serde::to_vec` and reads it back with `rmp_serde::from_slice`, verifying `rows` `3`, `cols` `2` and a `sketch` of length `3`; the cell values are not checked. | -| `count_min_with_heap_wire_shape` | The nested `CountMinSketchWithHeapWire` DTO round-trips through `rmp_serde` with its nesting intact. | Serializes a wire struct holding a `CountMinSketchInnerWire` of `2x4` zeros, one `CmsHeapItem { key: "hot", value: 42.0 }` and `heap_size: 8`, then verifies the `rmp_serde` decode reports `heap_size` `8`, one heap entry and the key `hot`; the item's value and the inner matrix are not re-checked. | -| `count_sketch_round_trip` | A portable `CountSketch` survives a msgpack round trip with its geometry intact. | Updates `CountSketch::new(3, 64)` with `k1` `+1.0` and `k2` `+2.0`, then verifies the decode reports `rows` `3` and `cols` `64`; the signed counters and the estimates are not checked. | -| `dd_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `dd_sketch_round_trip` | A portable `DdSketch` survives a msgpack round trip carrying its full count in the bucket array. | Updates `DdSketch::new(0.01)` with `1.0`, `10.0` and `100.0`, then verifies the decode's `total_count()` — recovered by summing the buckets, since the count is not on the wire — is `3`; `alpha`, the bucket offset and the quantiles are not checked. | -| `set_aggregator_delta_round_trip` | A `SetAggregatorDelta` survives a msgpack round trip with both of its sets. | Serializes a `SetAggregatorDelta` whose `added` holds `a` and whose `removed` holds `b`, then verifies the decode's `added` contains `a` and its `removed` contains `b`. | -| `hll_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `hll_sketch_round_trip` | A portable `HllSketch` survives a msgpack round trip with its register count intact. | Updates `HllSketch::new(HllVariant::Regular, 8)` with `b"a"`, `b"b"` and `b"c"`, then verifies only that the decode's `registers` has the same length as the source's; the register values, the precision and the variant are not checked. | -| `hydra_kll_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `hydra_kll_sketch_round_trip` | A portable `HydraKllSketch` survives a msgpack round trip with its grid shape intact. | Updates `HydraKllSketch::with_seed(2, 4, 200, 0x5EED_0900)` with `("a", 1.0)`, `("a", 2.0)` and `("b", 3.0)`, then verifies the decode reports `rows` `2` and `cols` `4`; the nested KLL payloads are not checked. | -| `hydra_kll_wire_shape` | The `HydraKllSketchWire` DTO round-trips through `rmp_serde` with its two levels of nesting intact. | Serializes a wire struct of `rows: 2`, `cols: 3` and a `2x3` grid of `KllSketchData { k: 200, sketch_bytes: vec![] }`, then verifies the `rmp_serde` decode reports `rows` `2`, `cols` `3`, an outer length of `2` and an inner length of `3`. | -| `kll_sketch_decodes_go_bytes` | The test asserts nothing. | The test asserts nothing: the body is empty and the test is marked `#[ignore]` with the reason that no cross-language msgpack fixture exists in the repository. | -| `kll_sketch_round_trip` | A portable `KllSketch` survives a msgpack round trip with its `k` and retained mass intact. | Feeds `KllSketch::with_seed(200, 0x5EED_0800)` the values `0..100` as `f64`, then verifies the decode reports `k` `200` and `count()` `100`; the retained items and the quantile answers are not checked. | -| `set_aggregator_round_trip` | A `SetAggregator` survives a msgpack round trip with its members. | Updates a `SetAggregator::new()` with `web` and `api`, then verifies the decode holds two values, one of which is `web`. | - ### Conformance Kit Test file: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) @@ -2336,9 +2102,8 @@ Test file: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) | `count_sketch_with_l2_norm_passes_signed_frequency_and_merge_conformance` | `CountSketchWithL2Norm` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `CountSketchWithL2Norm::with_dimensions(5, 4_096)` over a 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`) through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0` on every key of true count at least `25`, through `turnstile_battery` on key `42` (`+500` then `-200` must estimate `300` within `1.0`, then `-300` must cancel to within `1e-6`), and through `merge_equivalence_battery`, which shards the stream by parity, merges, and requires agreement with a single-pass sketch within `25.0 + 0.06 * count`. | | `countmin_passes_frequency_and_merge_conformance` | `CountMinSketch` passes the one-sided frequency and merge-equivalence batteries. | Drives `CountMinSketch::, FastPath>::with_dimensions(4, 4096)` over the 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`) through `frequency_battery` one-sided at `rel_tol` `0.01` and `abs_tol` `4.0`, which requires the absent sentinel key `i64::MIN` to estimate within `±4.0` and each key of true count at least `25` to land in `[count, count * 1.01 + 4.0]`, then through `merge_equivalence_battery` at the same spec. | | `countsketch_passes_signed_frequency_conformance` | `CountSketch` passes the two-sided frequency, turnstile and merge-equivalence batteries. | Drives `CountSketch::, RegularPath>::with_dimensions(5, 4096)` over the 60,000-draw Zipf stream through `frequency_battery` two-sided at `rel_tol` `0.06` and `abs_tol` `25.0`, `turnstile_battery` on key `42` via `update_weighted` (`+500 - 200` must read `300` within `1.0`, then cancel to within `1e-6`), and `merge_equivalence_battery` at the same spec. | -| `ddsketch_passes_relative_quantile_conformance` | The portable `DdSketch` holds its relative-value-error promise under its own rank convention. | Runs `relative_quantile_battery` on `PortableDds::new(0.01)` over the positive samples of `normal_f64(40_000, 500.0, 80.0, 7001)`, requiring at q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` a relative error of at most `0.01` plus a few-ULP numerical slack against the exact `floor(q*(n-1))` order statistic. | | `heap_backed_frequency_sketches_pass_frequency_and_merge_conformance` | Both heap-backed matrix sketches pass the frequency and merge-equivalence batteries. | Over the 60,000-draw Zipf stream (domain 2,048, exponent 1.1, seed `9001`), drives `CountMinSketchWithHeap::, FastPath>::new(4, 4_096, 64)` through `frequency_battery` and `merge_equivalence_battery` one-sided at `rel_tol` `0.01` and `abs_tol` `8.0`, and `CountSketchWithHeap::, RegularPath>::new(5, 4_096, 64)` through both two-sided at `rel_tol` `0.06` and `abs_tol` `25.0`. | -| `hll_variants_pass_cardinality_conformance` | All three native HyperLogLog estimators and the portable one estimate within 3%. | Runs `cardinality_battery` at `rel_tol` `0.03` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)` on `HyperLogLog`, `HyperLogLog`, `HyperLogLogHip` and the portable `HllSketch::new(HllVariant::Regular, 14)`; each must estimate `100_000` within 3% and stay in band after the whole stream is replayed as duplicates. | +| `hll_variants_pass_cardinality_conformance` | All three HyperLogLog estimators estimate within 3%. | Runs `cardinality_battery` at `rel_tol` `0.03` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)` on `HyperLogLog`, `HyperLogLog` and `HyperLogLogHip`; each must estimate `100_000` within 3% and stay in band after the whole stream is replayed as duplicates. | | `kll_family_passes_quantile_conformance` | The three KLL adapters hold the shared rank-error band. | Runs `quantile_battery` at the default `rank_tol` of `0.03` over q `0.1`, `0.25`, `0.5`, `0.75` and `0.9` on the positive samples of `normal_f64(40_000, 500.0, 80.0, 7001)` for `Kll::with_k_and_seed(200, 0x4017_0001)`, `KllDynamic::::with_k_and_seed(200, 0x4017_0001)`, and a `Kll` at the same `k` and seed queried through `quantile_cached`. | | `kmv_passes_cardinality_conformance` | `Kmv` estimates a 100,000-key distinct count within 8%. | Compiled only under the `experimental` feature: runs `cardinality_battery` at `rel_tol` `0.08` on `Kmv::new(4_096)` over 100,000 distinct keys `i.wrapping_mul(0x9E37_79B9_7F4A_7C15)`, requiring the estimate in band both after the first pass and after the duplicate replay. | | `uniform_sampling_at_full_rate_passes_quantile_conformance` | A sampler retaining everything answers quantiles exactly. | Compiled only under the `experimental` feature: runs `quantile_battery` at `rank_tol` `0.0` on `UniformSampling::with_seed(1.0, 7_704)` over 20,000 Zipf draws (domain 4,096, exponent 1.1, seed `7_703`), with each quantile read from the sorted retained samples at index `ceil(q * len)`. | @@ -2374,5 +2139,4 @@ Test file: [`tests/bug_verification.rs`](../tests/bug_verification.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `portable_ddsketch_respects_alpha_at_bucket_edges` | Both DDSketch implementations hold the advertised relative accuracy on a value sitting at a bucket's lower edge. | Sets `alpha = 0.05`, `gamma = (1+alpha)/(1-alpha)`, and feeds the bucket-edge value `gamma^20 * (1 + 1e-6)` 10,000 times into both `message_pack_format::portable::ddsketch::DdSketch::new(0.05)` and `DdSketch::new(0.05)`, then asserts the portable `quantile(0.5)` and the core `get_value_at_quantile(0.5)` are each within `alpha * (1 + 1e-6)` relative error of that value — a representative reported as the bucket log-midpoint `gamma^(k+0.5)` gives about `5.13%` here and fails. Exact agreement between the two is not asserted, because the core sketch clamps representatives to its observed min/max and the portable wire format does not carry them. | | `count_sketch_with_l2_norm_f2_survives_beyond_i64_max` | `CountSketchWithL2Norm`'s hot-path L2 accumulation stays exact inside `i64` and saturates beyond it. | Builds `CountSketchWithL2Norm::::with_dimensions_and_seed_index(4, 2048, 7)` and calls `update_weighted(&DataInput::U32(1), 3_000_000_000)`, asserting `l2_squared()` matches the exact `9e18` to a relative `1e-12`; a second key `U32(2)` at the same count puts the true F2 at `1.8e19`, past `i64::MAX`, and `l2_squared()` must then equal `i64::MAX` (`9.223e18`) to a relative `1e-12` rather than wrapping to `0` or panicking in debug. | diff --git a/examples/accuracy_probe.rs b/examples/accuracy_probe.rs index 78a99150..2e33b37d 100644 --- a/examples/accuracy_probe.rs +++ b/examples/accuracy_probe.rs @@ -7,11 +7,6 @@ //! printing EXPECTED vs ACTUAL and a theory-based verdict. use asap_sketchlib::common::input::{HydraCounter, HydraQuery}; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketch as PortableCountMinSketch; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; -use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; -use asap_sketchlib::message_pack_format::portable::kll::KllSketch as PortableKll; use asap_sketchlib::{ CountMinSketch, CountMinSketchWithHeap, CountSketch as CoreCount, CountSketchWithHeap, CountSketchWithL2Norm, DataInput, DdSketch, Hydra, HyperLogLog, Kll, UnivMon, @@ -119,7 +114,6 @@ fn main() { #[cfg(feature = "experimental")] probe_eh_univ(); probe_tumbling(); - probe_portable_wire_types(); println!("===================================================================="); println!("Probe complete. FAIL lines above are candidate wrong-query-results."); @@ -525,66 +519,48 @@ fn quantile_report(qf: impl Fn(f64) -> f64, data: &[f64], qs: &[f64]) -> (f64, S // ---------------------------------------------------------------- DDSketch fn probe_ddsketch() { let mut p = Probe::new(); - println!("--- DDSketch core vs portable (alpha=0.05) ---"); + println!("--- DDSketch (alpha=0.05) ---"); let alpha = 0.05; let gamma = (1.0f64 + alpha) / (1.0 - alpha); // Isolated adversarial probe: one distinct value sitting just above a // bucket's lower edge gamma^k, repeated N times. The only correct answer // for every quantile is v itself. let mut worst_core = 0.0f64; - let mut worst_port = 0.0f64; for k in [10i32, 20, 30] { let v = gamma.powi(k) * (1.0 + 1e-6); // just inside bucket k let n = 10_000u64; let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); for _ in 0..n { core.add(&v); - port.update(v); } let rc = rel_err(core.get_value_at_quantile(0.5).unwrap(), v); - let rp = rel_err(port.quantile(0.5).unwrap(), v); worst_core = worst_core.max(rc); - worst_port = worst_port.max(rp); } p.check( "core DDSketch honors alpha=0.05 at bucket edges", format!("max rel err {worst_core:.5}"), worst_core <= alpha * (1.0 + 1e-6), ); - p.check( - "portable DdSketch honors alpha=0.05 at bucket edges", - format!("max rel err {worst_port:.5} (shared gamma^k*(1+alpha) representative)"), - worst_port <= alpha * (1.0 + 1e-6), - ); - // Mixed-stream sanity on both. + // Mixed-stream sanity. let mut rng = StdRng::seed_from_u64(47); let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); let mut truth: Vec = Vec::new(); for _ in 0..20_000 { let k = rng.random_range(5..40); let frac = rng.random::(); let v = gamma.powi(k) * (1.0 + frac * (gamma - 1.0)); core.add(&v); - port.update(v); truth.push(v); } truth.sort_by(|a, b| a.partial_cmp(b).unwrap()); let mc = qs_max_rel(&truth, |q| core.get_value_at_quantile(q).unwrap(), &qs_dd()); - let mp = qs_max_rel(&truth, |q| port.quantile(q).unwrap(), &qs_dd()); p.check( "core mixed stream <= alpha", format!("max rel {mc:.5}"), mc <= alpha, ); - p.check( - "portable mixed stream <= alpha", - format!("max rel {mp:.5}"), - mp <= alpha, - ); p.finish("DDSketch"); } @@ -1030,92 +1006,3 @@ fn probe_tumbling() { ); p.finish("Tumbling"); } - -// ---------------------------------------------------- Portable wire types -fn probe_portable_wire_types() { - let mut p = Probe::new(); - println!("--- Portable wire types (Go-parity DTOs) ---"); - - // Portable CountMinSketch. - let mut pcs = PortableCountMinSketch::new(3, 4096); - let mut truth: HashMap = HashMap::new(); - let mut z = Zipf::new(2048, 1.1, 54); - for _ in 0..50_000 { - let k = format!("k{}", z.sample_usize()); - *truth.entry(k.clone()).or_insert(0.0) += 1.0; - pcs.update(&k, 1.0); - } - let mut by_freq: Vec<(String, f64)> = truth.iter().map(|(k, v)| (k.clone(), *v)).collect(); - by_freq.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); - let eps = std::f64::consts::E / 4096.0; - let mut bad = 0usize; - for (k, v) in by_freq.iter().take(50) { - let est = pcs.estimate(k); - if est < *v || est > v + eps * 50_000.0 { - bad += 1; - } - } - p.check( - "portable CMS one-sided bound (top-50)", - format!("{bad} violations"), - bad == 0, - ); - - // Portable HLL precision 12. - let mut ph = HllSketch::new(HllVariant::Regular, 12); - for i in 0..100_000u64 { - ph.update(i.to_le_bytes().as_slice()); - } - let est = ph.estimate(); - p.check( - "portable HLL p12 within 3%", - format!( - "expected 100000, got {est:.0} (rel {:.4})", - rel_err(est, 100_000.0) - ), - rel_err(est, 100_000.0) < 0.03, - ); - - // Portable KLL. - let mut pk = PortableKll::new(200); - let mut rng = StdRng::seed_from_u64(55); - let mut vals: Vec = (0..50_000).map(|_| rng.random::() * 1e6).collect(); - vals.sort_by(|a, b| a.partial_cmp(b).unwrap()); - for v in &vals { - pk.update(*v); - } - let med = pk.quantile(0.5); - let t = vals[25_000]; - p.check( - "portable KLL median", - format!("expected ~{t:.0}, got {med:.0}"), - (med - t).abs() / t < 0.05, - ); - - // HydraKllSketch: per-key quantiles, median across rows. - let mut hk = HydraKllSketch::new(3, 256, 200); - let mut per_key: HashMap<&str, Vec> = HashMap::new(); - let mut rng2 = StdRng::seed_from_u64(56); - for key in ["svc-a", "svc-b"] { - let base = if key == "svc-a" { 100.0 } else { 900.0 }; - let vs: Vec = (0..2_000) - .map(|_| base + rng2.random::() * 50.0) - .collect(); - for v in &vs { - hk.update(key, *v); - } - per_key.insert(key, vs); - } - for (key, vs) in per_key { - let mut s = vs.clone(); - s.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let t = s[s.len() / 2]; - let got = hk.quantile(key, 0.5); - p.check( - &format!("HydraKll median({key})"), - format!("expected ~{t:.1}, got {got:.1}"), - (got - t).abs() / t < 0.05, - ); - } - p.finish("Portable wire types"); -} diff --git a/src/message_pack_format/admission_rate.rs b/src/asapv1/admission_rate.rs similarity index 99% rename from src/message_pack_format/admission_rate.rs rename to src/asapv1/admission_rate.rs index 565403da..9be948db 100644 --- a/src/message_pack_format/admission_rate.rs +++ b/src/asapv1/admission_rate.rs @@ -71,7 +71,7 @@ mod tests { use std::panic::{AssertUnwindSafe, catch_unwind}; use crate::input::HydraCounter; - use crate::message_pack_format::envelope; + use crate::asapv1::envelope; use crate::sketch_framework::eh_bucket_sketch::wire::{SketchState, rebuild_sketch}; use crate::sketches::CountSketchDelta; use crate::*; diff --git a/src/message_pack_format/delta.rs b/src/asapv1/delta.rs similarity index 100% rename from src/message_pack_format/delta.rs rename to src/asapv1/delta.rs diff --git a/src/message_pack_format/envelope.rs b/src/asapv1/envelope.rs similarity index 97% rename from src/message_pack_format/envelope.rs rename to src/asapv1/envelope.rs index fe5c9c1e..8c44413c 100644 --- a/src/message_pack_format/envelope.rs +++ b/src/asapv1/envelope.rs @@ -190,8 +190,6 @@ mod tests { fn decode_paths() -> Vec { use crate::input::HydraCounter; - use crate::message_pack_format::MessagePackCodec; - use crate::message_pack_format::portable::hll::HllSketch; use crate::*; macro_rules! case { @@ -300,14 +298,11 @@ mod tests { let hydra_kll = hydra(HydraCounter::Kll(Kll::default()), DataInput::F64(2.5)); let um = UnivMon::new(4, 2, 16, 3); let hydra_um = hydra(HydraCounter::UnivMon(um), DataInput::Str("alpha")); - let portable = |b: &[u8]| HllSketch::from_msgpack(b).is_ok(); let cases: Vec = vec![ - case!("HLL classic", classic.clone(), HllClassic), - case!("HLL HIP", hip.clone(), HyperLogLogHipP12), + case!("HLL classic", classic, HllClassic), + case!("HLL HIP", hip, HyperLogLogHipP12), case!("HLL classic delta", classic_delta, HyperLogLogDelta), - ("HLL portable classic", classic, portable), - ("HLL portable HIP", hip, portable), case!("CMS", cms, Cms), case!("CMS delta", cms_delta, CountMinSketchDelta), case!("CS", cs, Cs), diff --git a/src/message_pack_format/kind.rs b/src/asapv1/kind.rs similarity index 100% rename from src/message_pack_format/kind.rs rename to src/asapv1/kind.rs diff --git a/src/asapv1/mod.rs b/src/asapv1/mod.rs new file mode 100644 index 00000000..c798d284 --- /dev/null +++ b/src/asapv1/mod.rs @@ -0,0 +1,13 @@ +//! ASAPv1 framing shared by every sketch's `to_asapv1` / `from_asapv1`. +//! +//! Each sketch serializes into one self-delimiting envelope: a sketch-agnostic +//! frame (magic, version, `kind_id`, two length prefixes) around a metadata +//! map and a payload array. The framing is in `envelope`; the `kind_id`, +//! metadata and payload are per-sketch, in the `wire.rs` beside each sketch. +//! `docs/asapv1_wire_format.md` is the spec. + +pub(crate) mod admission_rate; +pub(crate) mod delta; +pub(crate) mod envelope; +pub(crate) mod kind; +pub(crate) mod wire_key; diff --git a/src/message_pack_format/wire_key.rs b/src/asapv1/wire_key.rs similarity index 99% rename from src/message_pack_format/wire_key.rs rename to src/asapv1/wire_key.rs index ddd49374..8ea3dc54 100644 --- a/src/message_pack_format/wire_key.rs +++ b/src/asapv1/wire_key.rs @@ -253,7 +253,7 @@ impl<'de, V: Visitor<'de>> Visitor<'de> for Float32Only { mod tests { use super::*; use crate::input::HydraCounter; - use crate::message_pack_format::envelope::{self, hex}; + use crate::asapv1::envelope::{self, hex}; use crate::*; fn float64(value: f64) -> Vec { diff --git a/src/common/input.rs b/src/common/input.rs index b84954f6..6cf74c62 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -1,7 +1,7 @@ //! Shared input, query, and counter types used across sketches. //! This module defines value wrappers, heap-friendly items, and Hydra/UnivMon helpers. -use crate::message_pack_format::admission_rate; +use crate::asapv1::admission_rate; use serde::{Deserialize, Serialize}; use std::{ fmt, diff --git a/src/lib.rs b/src/lib.rs index d2952725..b09b6537 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,22 +7,16 @@ //! reusable utilities used across sketches. //! - [`sketch_framework`]: higher-level composition layers such as Hydra, //! UnivMon, tumbling windows, and batch/parallel execution helpers. -//! - [`message_pack_format`]: MessagePack/proto wire format shared with -//! `sketchlib-go`. `message_pack_format::portable` owns the legacy wire-DTO -//! sketch types consumed by the ASAP query engine (`portable::{CountMinSketch, -//! CountSketch, KllSketch, HllSketch, DdSketch, HydraKllSketch, -//! CountMinSketchWithHeap}`); `SetAggregator` and `SetAggregatorDelta` are -//! re-exported at the crate root. //! //! Most users can start with the crate-root re-exports such as [`DataInput`], //! [`CountMinSketch`], [`HyperLogLog`], [`Kll`], and [`DdSketch`]. Reach for the //! submodules directly when you need lower-level storage, hashing, or //! framework-specific APIs. +pub(crate) mod asapv1; /// Shared primitives used across sketches, including input wrappers, hashers, /// storage backends, and reusable utilities. pub mod common; -pub mod message_pack_format; /// Higher-level composition layers such as Hydra, UnivMon, tumbling windows, /// and batch/parallel execution helpers. pub mod sketch_framework; @@ -38,8 +32,6 @@ pub mod __private { } pub use common::*; -pub use message_pack_format::envelope::peek_kind_id; -pub use message_pack_format::portable::delta_set_aggregator::SetAggregatorDelta; -pub use message_pack_format::portable::set_aggregator::SetAggregator; +pub use asapv1::envelope::peek_kind_id; pub use sketch_framework::*; pub use sketches::*; diff --git a/src/message_pack_format/codec.rs b/src/message_pack_format/codec.rs deleted file mode 100644 index 8ab7de32..00000000 --- a/src/message_pack_format/codec.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! `MessagePackCodec` trait — unified encode/decode contract for the -//! wire-format-aligned types in [`crate::message_pack_format::portable`]. - -use super::Error; - -/// Round-trippable MessagePack codec. -/// -/// All [`crate::message_pack_format::portable`] sketch / aggregator types implement this trait; -/// it is the single entry point for encode/decode against the wire format. -/// -/// There is no `dyn`-safe usage — callers always know the concrete -/// type — so impls are dispatched statically (zero-cost) via -/// monomorphization. -pub trait MessagePackCodec: Sized { - /// Serialize `self` to MessagePack bytes matching the wire format - /// shared with `sketchlib-go`. - fn to_msgpack(&self) -> Result, Error>; - - /// Deserialize a MessagePack byte slice into `Self`. - fn from_msgpack(bytes: &[u8]) -> Result; -} diff --git a/src/message_pack_format/error.rs b/src/message_pack_format/error.rs deleted file mode 100644 index a69452bb..00000000 --- a/src/message_pack_format/error.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Unified error type for MessagePack encode/decode operations. - -use std::error::Error as StdError; -use std::fmt; - -/// Error returned by [`crate::message_pack_format::MessagePackCodec`] -/// implementations and by the `dto`-level free functions in -/// [`crate::message_pack_format`]. -#[derive(Debug)] -pub enum Error { - /// MessagePack encoding failed. - Encode(rmp_serde::encode::Error), - /// MessagePack decoding failed. - Decode(rmp_serde::decode::Error), -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Error::Encode(e) => write!(f, "MessagePack encode failed: {e}"), - Error::Decode(e) => write!(f, "MessagePack decode failed: {e}"), - } - } -} - -impl StdError for Error { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - match self { - Error::Encode(e) => Some(e), - Error::Decode(e) => Some(e), - } - } -} - -impl From for Error { - fn from(value: rmp_serde::encode::Error) -> Self { - Error::Encode(value) - } -} - -impl From for Error { - fn from(value: rmp_serde::decode::Error) -> Self { - Error::Decode(value) - } -} diff --git a/src/message_pack_format/mod.rs b/src/message_pack_format/mod.rs deleted file mode 100644 index 75dfdfc0..00000000 --- a/src/message_pack_format/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Rust-side MessagePack serialization plumbing. -//! -//! The current format is **ASAPv1**: each sketch serializes into one -//! self-delimiting envelope — a sketch-agnostic frame (magic, version, -//! `kind_id`, two length prefixes) around a metadata map and a payload -//! array. The framing lives in `envelope.rs`; the `kind_id`, metadata and -//! payload are per-sketch, in the `wire.rs` beside each sketch under -//! [`crate::sketches`] and [`crate::sketch_framework`]. `sketchlib-go` -//! mirrors ASAPv1, `asapv1_golden/` guards against drift, and -//! `docs/asapv1_wire_format.md` is the spec. -//! -//! Two sub-modules are the older path, being retired: -//! -//! - [`portable`] — pre-envelope types with their own Go goldens. -//! - [`native`] — thin shims over the sketches' `to_asapv1` / -//! `from_asapv1`, each a pass-through to ASAPv1. -//! -//! [`MessagePackCodec`] and [`Error`] are the contract both share. - -pub(crate) mod admission_rate; -pub mod codec; -pub(crate) mod delta; -pub(crate) mod envelope; -pub mod error; -pub(crate) mod kind; -pub mod native; -pub mod portable; -pub(crate) mod wire_key; - -pub use codec::MessagePackCodec; -pub use error::Error; diff --git a/src/message_pack_format/native/countminsketch.rs b/src/message_pack_format/native/countminsketch.rs deleted file mode 100644 index bef49c21..00000000 --- a/src/message_pack_format/native/countminsketch.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::countminsketch::CountMinSketch`]. -//! -//! Only the canonical wire configs — i32/i64/f64 counters (`CmsWireCounter`) -//! with a fast/regular mode (`CmsWireMode`) — are serializable. Exotic in-memory -//! counters (i128/…) must be converted to a wire type first. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::countminsketch::{CmsWireCounter, CmsWireMode, CountMinSketch}; -use crate::{HashProfile, SketchHasher, Vector2D}; - -impl MessagePackCodec for CountMinSketch, Mode, H> -where - // `AddAssign` is required for `Vector2D: MatrixStorage`. - T: CmsWireCounter + std::ops::AddAssign + Serialize + for<'de> Deserialize<'de>, - Mode: CmsWireMode, - H: SketchHasher + HashProfile, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/countsketch.rs b/src/message_pack_format/native/countsketch.rs deleted file mode 100644 index a93c0ff0..00000000 --- a/src/message_pack_format/native/countsketch.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::countsketch::CountSketch`]. -//! -//! Only the canonical wire configs — `i32`, `i64` or `f64` counters -//! (`CsWireCounter`) with a fast/regular mode (`CsWireMode`) — are -//! serializable; `i128` and non-`Vector2D` storage must be converted first. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::countsketch::{CountSketch, CountSketchCounter, CsWireCounter, CsWireMode}; -use crate::{HashProfile, SketchHasher, Vector2D}; - -impl MessagePackCodec for CountSketch, Mode, H> -where - T: CsWireCounter - + CountSketchCounter - + std::ops::AddAssign - + Serialize - + for<'de> Deserialize<'de>, - Mode: CsWireMode, - H: SketchHasher + HashProfile, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/countsketch_l2_norm.rs b/src/message_pack_format/native/countsketch_l2_norm.rs deleted file mode 100644 index 523ac0b2..00000000 --- a/src/message_pack_format/native/countsketch_l2_norm.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm`]. - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; -use crate::{HashProfile, SketchHasher}; - -impl MessagePackCodec for CountSketchWithL2Norm { - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/ddsketch.rs b/src/message_pack_format/native/ddsketch.rs deleted file mode 100644 index 7dfff203..00000000 --- a/src/message_pack_format/native/ddsketch.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::ddsketch::DdSketch`]. -//! -//! Produces and consumes the ASAPv1 envelope (kind_id `0x05 0x00`): `alpha` in -//! the metadata, the eight-field payload of the wire spec §3.8. - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::ddsketch::DdSketch; - -impl MessagePackCodec for DdSketch { - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/hll.rs b/src/message_pack_format/native/hll.rs deleted file mode 100644 index f22df7b3..00000000 --- a/src/message_pack_format/native/hll.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Native MessagePack codec impls for [`crate::sketches::hll`]. - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::hll::{HllWireVariant, HyperLogLogHipImpl, HyperLogLogImpl}; -use crate::{HashProfile, HllRegisterStorage, SketchHasher}; - -impl - MessagePackCodec for HyperLogLogImpl -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} - -impl MessagePackCodec for HyperLogLogHipImpl { - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/kll.rs b/src/message_pack_format/native/kll.rs deleted file mode 100644 index 7662f328..00000000 --- a/src/message_pack_format/native/kll.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::kll::Kll`]. - -use serde::{Deserialize, Serialize}; - -use crate::common::numerical::NumericalValue; -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::kll::{Kll, KllWireItem}; - -impl MessagePackCodec for Kll -where - T: NumericalValue + KllWireItem + Serialize + for<'de> Deserialize<'de>, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/kll_dynamic.rs b/src/message_pack_format/native/kll_dynamic.rs deleted file mode 100644 index 1cc3e487..00000000 --- a/src/message_pack_format/native/kll_dynamic.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::kll_dynamic::KllDynamic`]. - -use serde::{Deserialize, Serialize}; - -use crate::common::numerical::NumericalValue; -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::kll::KllWireItem; -use crate::sketches::kll_dynamic::KllDynamic; - -impl MessagePackCodec for KllDynamic -where - T: NumericalValue + KllWireItem + Serialize + for<'de> Deserialize<'de>, -{ - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/kmv.rs b/src/message_pack_format/native/kmv.rs deleted file mode 100644 index b8323b0f..00000000 --- a/src/message_pack_format/native/kmv.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Native MessagePack codec impl for [`crate::sketches::kmv::Kmv`]. -//! -//! The bytes are the ASAPv1 wire envelope (kind `0x0e 0x00`); the impl is -//! bounded on `H: HashProfile`, so a sketch built with an unprofiled hasher has -//! no codec. - -use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::kmv::Kmv; -use crate::{HashProfile, SketchHasher}; - -impl MessagePackCodec for Kmv { - fn to_msgpack(&self) -> Result, Error> { - Ok(self.to_asapv1()?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(Self::from_asapv1(bytes)?) - } -} diff --git a/src/message_pack_format/native/mod.rs b/src/message_pack_format/native/mod.rs deleted file mode 100644 index 5fea5948..00000000 --- a/src/message_pack_format/native/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Older `MessagePackCodec` shims over the pure-Rust sketch types in -//! [`crate::sketches`], being retired. -//! -//! Each impl forwards to that sketch's own `to_asapv1` / -//! `from_asapv1`, so the bytes are ASAPv1 — the framing in -//! `envelope.rs` plus the per-sketch `wire.rs` — and the shim adds nothing -//! to them. Call those methods directly; `docs/asapv1_wire_format.md` -//! specifies the format `sketchlib-go` mirrors. - -pub mod countminsketch; -pub mod countsketch; -pub mod countsketch_l2_norm; -pub mod ddsketch; -pub mod hll; -pub mod kll; -pub mod kll_dynamic; -#[cfg(feature = "experimental")] -pub mod kmv; diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs deleted file mode 100644 index 3b99f664..00000000 --- a/src/message_pack_format/portable/countminsketch.rs +++ /dev/null @@ -1,1000 +0,0 @@ -//! Wire-format-aligned Count-Min sketch types. The wire DTO and -//! runtime ops live together here. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::sketches::countminsketch::CountMinSketch as NativeCountMinSketch; -use crate::{DataInput, FastPath, Vector2D}; - -// ===================================================================== -// asap_sketchlib wire-format-aligned variant. -// -// `CountMinSketch` and `CountMinDelta` below are the public-field, -// proto-decode-friendly types consumed by the ASAP query engine -// accumulators, backed by `asap_sketchlib`'s in-tree CountMin. The -// high-throughput in-process variant is -// `crate::sketches::countminsketch::CountMin`, which keeps its own design. -// ===================================================================== - -// ----- asap_sketchlib-backed Count-Min helpers ----- -// Used below by `CountMinSketch`. Lives in this file so the wire-format -// type and its backend share a single home. - -/// Concrete Count-Min type backing the wire-format `CountMinSketch`. -/// Uses f64 counters (`Vector2D`) for weighted updates without -/// integer rounding, and the `FastPath` packed-hash strategy so the -/// matrix-cell layout is byte-parity with `sketchlib-go`. Locked in by -/// `tests/sketches_go_parity_probe.rs`. -pub type SketchlibCms = NativeCountMinSketch, FastPath>; - -/// Creates a fresh sketchlib Count-Min sketch with the given dimensions. -pub fn new_sketchlib_cms(row_num: usize, col_num: usize) -> SketchlibCms { - SketchlibCms::with_dimensions(row_num, col_num) -} - -/// Builds a sketchlib Count-Min sketch from an existing `sketch` matrix. -pub fn sketchlib_cms_from_matrix( - row_num: usize, - col_num: usize, - sketch: &[Vec], -) -> SketchlibCms { - let matrix = Vector2D::from_fn(row_num, col_num, |r, c| { - sketch - .get(r) - .and_then(|row| row.get(c)) - .copied() - .unwrap_or(0.0) - }); - SketchlibCms::from_storage(matrix) -} - -/// Converts a sketchlib Count-Min sketch into a `Vec>` matrix. -pub fn matrix_from_sketchlib_cms(inner: &SketchlibCms) -> Vec> { - let storage: &Vector2D = inner.as_storage(); - let rows = storage.rows(); - let cols = storage.cols(); - let mut sketch = vec![vec![0.0; cols]; rows]; - - for (r, row) in sketch.iter_mut().enumerate().take(rows) { - for (c, cell) in row.iter_mut().enumerate().take(cols) { - if let Some(v) = storage.get(r, c) { - *cell = *v; - } - } - } - - sketch -} - -/// Helper to update a sketchlib Count-Min with a weighted key. -pub fn sketchlib_cms_update(inner: &mut SketchlibCms, key: &str, value: f64) { - if value <= 0.0 { - return; - } - inner.update_weighted(&DataInput::String(key.to_owned()), value); -} - -/// Helper to query a sketchlib Count-Min for a key, returning f64. -pub fn sketchlib_cms_query(inner: &SketchlibCms, key: &str) -> f64 { - inner.estimate(&DataInput::String(key.to_owned())) -} - -/// Sparse delta between two consecutive CountMinSketch snapshots — -/// the input shape for [`CountMinSketch::apply_delta`]. Mirrors the -/// `CountMinSketchDelta` proto in -/// `sketchlib-go/proto/countminsketch/countminsketch.proto` (packed -/// encoding only). -/// -/// Cells apply additively: `matrix[row][col] += d_count`. Per-row -/// L1 and L2 norm deltas are carried for downstream error-accounting -/// but are not consumed by `apply_delta` itself. -/// -/// Structurally identical to [`super::countsketch::CountSketchDelta`]: the -/// optional `hh_keys` channel carries heavy-hitter candidate keys forwarded by -/// an upstream tracker. Count-Min can track heavy hitters, but whether -/// `hh_keys` is populated is a control-plane decision — it is empty when -/// heavy-hitter tracking is not enabled. The receiver re-queries the merged -/// matrix for each key to (re)build its Top-K. Mirrors the Go reference -/// implementation's `Delta.HHKeys`. -#[derive(Debug, Clone, Default)] -pub struct CountMinSketchDelta { - pub rows: u32, - pub cols: u32, - pub cells: Vec<(u32, u32, i64)>, - pub l1: Vec, - pub l2: Vec, - pub hh_keys: Vec, -} - -/// Provides approximate frequency counts with error bounds. -/// The msgpack wire format is the contract between sketch producers and -/// the query engine consumer. -#[derive(Debug, Clone)] -pub struct CountMinSketch { - pub rows: usize, - pub cols: usize, - pub(crate) backend: SketchlibCms, -} - -impl CountMinSketch { - pub fn new(rows: usize, cols: usize) -> Self { - Self { - rows, - cols, - backend: new_sketchlib_cms(rows, cols), - } - } - - /// Number of hash rows in the sketch matrix. - pub fn rows(&self) -> usize { - self.rows - } - - /// Number of columns (width) in the sketch matrix. - pub fn cols(&self) -> usize { - self.cols - } - - /// Returns the sketch matrix (for wire format, serialization, tests). - pub fn sketch(&self) -> Vec> { - matrix_from_sketchlib_cms(&self.backend) - } - - /// Construct from a `Vec>` matrix (used by deserialization and query engine). - pub fn from_legacy_matrix(sketch: Vec>, rows: usize, cols: usize) -> Self { - Self { - rows, - cols, - backend: sketchlib_cms_from_matrix(rows, cols, &sketch), - } - } - - /// Insert a single weighted observation. Delegates to the - /// `FastPath` backend so the matrix-cell layout matches - /// `sketchlib-go::CountMinSketch.InsertWithHash` bit-for-bit; the - /// parity is locked in by `tests/sketches_go_parity_probe.rs`. - /// - /// Negative or zero values are skipped, mirroring Go's - /// `UpdateWeight` behavior on `many == 0`. - pub fn update(&mut self, key: &str, value: f64) { - if value <= 0.0 || self.rows == 0 || self.cols == 0 { - return; - } - self.backend - .update_weighted(&DataInput::String(key.to_owned()), value); - } - - /// Estimate the frequency of `key` (CountMin point query). - /// Delegates to the `FastPath` backend, which derives the same - /// per-row column index that [`Self::update`] used. - pub fn estimate(&self, key: &str) -> f64 { - if self.rows == 0 || self.cols == 0 { - return 0.0; - } - self.backend.estimate(&DataInput::String(key.to_owned())) - } - - /// Merge another CountMinSketch into self in place. Both operands - /// must have identical dimensions. - pub fn merge( - &mut self, - other: &CountMinSketch, - ) -> Result<(), Box> { - if self.rows != other.rows || self.cols != other.cols { - return Err(format!( - "CountMinSketch dimension mismatch: self={}x{}, other={}x{}", - self.rows, self.cols, other.rows, other.cols - ) - .into()); - } - self.backend.merge(&other.backend); - Ok(()) - } - - /// Merge from references, allocating only the output — no input clones. - pub fn merge_refs( - accumulators: &[&Self], - ) -> Result> { - if accumulators.is_empty() { - return Err("No accumulators to merge".into()); - } - - let rows = accumulators[0].rows; - let cols = accumulators[0].cols; - - for acc in accumulators { - if acc.rows != rows || acc.cols != cols { - return Err( - "Cannot merge CountMinSketch accumulators with different dimensions".into(), - ); - } - } - - let mut merged = CountMinSketch::new(rows, cols); - for acc in accumulators { - merged.backend.merge(&acc.backend); - } - Ok(merged) - } - - /// Apply a sparse delta in place. Matches the `ApplyDelta` - /// semantics in `sketchlib-go/sketches/CountMinSketch/delta.go`: - /// `matrix[row][col] += d_count` for each cell in the delta. - /// - /// The FFI handle is opaque, so we snapshot the matrix, apply - /// cell updates, and rebuild the backend. The rebuild is - /// O(rows × cols) per delta and is acceptable for ingest-side - /// reconstitution — no delta should fire more than once per - /// window (10s–300s in the paper's B3 / B4 configs). - pub fn apply_delta( - &mut self, - delta: &CountMinSketchDelta, - ) -> Result<(), Box> { - for (row, col, _) in &delta.cells { - let r = *row as usize; - let c = *col as usize; - if r >= self.rows || c >= self.cols { - return Err(format!( - "CountMinSketchDelta cell ({r},{c}) out of range (matrix={}x{})", - self.rows, self.cols - ) - .into()); - } - } - let mut matrix = self.sketch(); - for (row, col, d_count) in &delta.cells { - matrix[*row as usize][*col as usize] += *d_count as f64; - } - self.backend = sketchlib_cms_from_matrix(self.rows, self.cols, &matrix); - Ok(()) - } - - /// Compute a sparse, proto-marshalled `CountMinDelta` of `self` - /// against a `snapshot`. A cell is included when its absolute count - /// delta `|Δcount|` (self − snapshot) is `>= threshold` and non-zero. - /// The full per-row L1/L2 norm deltas are always carried (one entry - /// per row, negligible size). - /// - /// This is the Rust twin of the Go reference implementation's - /// `ComputeDelta` + `SerializeDelta`: it iterates the matrix in - /// row-major order, subtracts the snapshot's value for each cell, and - /// emits the surviving cell deltas using the packed-array proto - /// encoding (`cell_rows`/`cell_cols`/`d_counts`). L1/L2 row deltas are - /// derived from the matrices — `l1[r] = Σ_c count[r][c]` and - /// `l2[r] = Σ_c count[r][c]^2` — which telescopes to the same value the - /// Go producer maintains incrementally. Heavy-hitter candidate keys - /// (`hh_keys`) are sourced from an upstream tracker that this minimal - /// wrapper does not maintain, so the field is left empty (an empty repeated - /// field encodes to nothing on the wire), exactly as CountSketch's - /// `compute_delta` does. The returned bytes are a `prost`-encoded - /// [`crate::proto::sketchlib::CountMinDelta`], byte-identical to the Go - /// `proto.Marshal(CountMinDelta)` output for the same inputs when no - /// heavy-hitter keys are forwarded (cross-language byte parity). - /// - /// Delta-against-empty: when `snapshot` is the all-zero sketch, every - /// surviving cell delta equals this window's own cell count, so the - /// result is this window's full state encoded as a delta (no - /// cross-window subtraction). CMS deltas carry only sketch-internal - /// cells/norms — there are no DataPoint-level metric scalars to drop. - pub fn compute_delta( - &self, - snapshot: &CountMinSketch, - threshold: f64, - ) -> Result, Box> { - use crate::proto::sketchlib::CountMinDelta as ProtoDelta; - use prost::Message; - - if self.rows != snapshot.rows || self.cols != snapshot.cols { - return Err(format!( - "CountMinSketch dimension mismatch: self={}x{}, snapshot={}x{}", - self.rows, self.cols, snapshot.rows, snapshot.cols - ) - .into()); - } - - let cur = self.sketch(); - let snap = snapshot.sketch(); - - let mut cell_rows: Vec = Vec::new(); - let mut cell_cols: Vec = Vec::new(); - let mut d_counts: Vec = Vec::new(); - let mut l1: Vec = Vec::with_capacity(self.rows); - let mut l2: Vec = Vec::with_capacity(self.rows); - - for r in 0..self.rows { - let mut cur_l1 = 0.0f64; - let mut cur_l2 = 0.0f64; - let mut snap_l1 = 0.0f64; - let mut snap_l2 = 0.0f64; - for c in 0..self.cols { - let cv = cur[r][c]; - let sv = snap[r][c]; - cur_l1 += cv; - cur_l2 += cv * cv; - snap_l1 += sv; - snap_l2 += sv * sv; - // CMS counts are non-negative integers in the wire form; - // mirror the Go reference's signed Δ + |Δ| threshold test. - let dc = (cv - sv) as i64; - if dc != 0 && (dc.unsigned_abs() as f64) >= threshold { - cell_rows.push(r as u32); - cell_cols.push(c as u32); - d_counts.push(dc); - } - } - l1.push(cur_l1 - snap_l1); - l2.push(cur_l2 - snap_l2); - } - - let delta = ProtoDelta { - rows: self.rows as u32, - cols: self.cols as u32, - cells_legacy: Vec::new(), - l1, - l2, - // Heavy-hitter keys are control-plane-gated; this wrapper has no - // tracker, so the field is empty (mirrors CountSketch::compute_delta). - hh_keys: Vec::new(), - cell_rows, - cell_cols, - d_counts, - }; - Ok(delta.encode_to_vec()) - } - - /// Apply a `prost`-encoded [`crate::proto::sketchlib::CountMinDelta`] - /// to this sketch in place (additive cell merge). The Rust twin of the - /// Go reference implementation's `DeserializeDelta` + `ApplyDelta`. - /// Reads the packed cell arrays (`cell_rows`/`cell_cols`/`d_counts`) - /// and falls back to the legacy per-cell records for payloads from - /// older producers. Heavy-hitter keys (`hh_keys`) are read symmetrically - /// (mirroring CountSketch) and carried onto the decoded delta; a plain CMS - /// keeps no Top-K, so they are not otherwise consumed here. - /// - /// Returns `Err` if `bytes` is not a valid `CountMinDelta` proto or a - /// cell is out of range for this sketch's dimensions. - pub fn apply_delta_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - use crate::proto::sketchlib::CountMinDelta as ProtoDelta; - use prost::Message; - - let proto = ProtoDelta::decode(bytes)?; - let cells: Vec<(u32, u32, i64)> = if !proto.cell_rows.is_empty() { - proto - .cell_rows - .iter() - .zip(proto.cell_cols.iter()) - .zip(proto.d_counts.iter()) - .map(|((&r, &c), &d)| (r, c, d)) - .collect() - } else { - proto - .cells_legacy - .iter() - .map(|c| (c.row, c.col, c.d_count as i64)) - .collect() - }; - let delta = CountMinSketchDelta { - rows: proto.rows, - cols: proto.cols, - cells, - l1: proto.l1, - l2: proto.l2, - // Heavy-hitter keys are carried through symmetrically (mirrors - // CountSketch). This wrapper keeps no Top-K, so they are not - // otherwise consumed; a control-plane sink would re-query the - // merged matrix for each key. Empty for a plain CMS. - hh_keys: proto.hh_keys, - }; - self.apply_delta(&delta) - } - - /// MessagePack twin of [`Self::compute_delta`]. Computes the same - /// sparse cell delta of `self` against `snapshot` (a cell is carried - /// when `|Δcount|` is non-zero and `>= threshold`), but serializes it - /// as the cells-only parallel-array MessagePack layout via - /// [`CountMinSketchDelta`]'s [`MessagePackCodec`] instead of proto. The - /// per-row `l1`/`l2` norms and `hh_keys` the proto delta carries are - /// dropped — the plain-sketch estimate path does not consume them. - /// Delta-against-empty carries this window's own cell counts. - pub fn compute_delta_msgpack( - &self, - snapshot: &CountMinSketch, - threshold: f64, - ) -> Result, Box> { - if self.rows != snapshot.rows || self.cols != snapshot.cols { - return Err(format!( - "CountMinSketch dimension mismatch: self={}x{}, snapshot={}x{}", - self.rows, self.cols, snapshot.rows, snapshot.cols - ) - .into()); - } - let cur = self.sketch(); - let snap = snapshot.sketch(); - let mut delta = CountMinSketchDelta { - rows: self.rows as u32, - cols: self.cols as u32, - ..CountMinSketchDelta::default() - }; - for r in 0..self.rows { - for c in 0..self.cols { - let dc = (cur[r][c] - snap[r][c]) as i64; - if dc != 0 && (dc.unsigned_abs() as f64) >= threshold { - delta.cells.push((r as u32, c as u32, dc)); - } - } - } - Ok(delta.to_msgpack()?) - } - - /// MessagePack twin of [`Self::apply_delta_bytes`]. Decodes the - /// cells-only parallel-array MessagePack [`CountMinSketchDelta`] and - /// applies it in place (additive cell merge). - /// - /// Returns `Err` if `bytes` is not a valid MessagePack - /// `CountMinSketchDelta` or a cell is out of range for this sketch's - /// dimensions. - pub fn apply_delta_msgpack_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - let delta = CountMinSketchDelta::from_msgpack(bytes)?; - self.apply_delta(&delta) - } - - /// One-shot aggregation: build a sketch from parallel key/value slices - /// and return the msgpack bytes. - pub fn aggregate_count( - depth: usize, - width: usize, - keys: &[&str], - values: &[f64], - ) -> Option> { - if keys.is_empty() { - return None; - } - let mut sketch = Self::new(depth, width); - for (key, &value) in keys.iter().zip(values.iter()) { - sketch.update(key, value); - } - sketch.to_msgpack().ok() - } - - /// Same as aggregate_count — CMS accumulates sums by construction. - pub fn aggregate_sum( - depth: usize, - width: usize, - keys: &[&str], - values: &[f64], - ) -> Option> { - Self::aggregate_count(depth, width, keys, values) - } -} - -// ----- Wire format ----- - -/// Wire DTO for [`CountMinSketch`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountMinSketchWire { - pub sketch: Vec>, - #[serde(rename = "row_num")] - pub rows: usize, - #[serde(rename = "col_num")] - pub cols: usize, -} - -impl MessagePackCodec for CountMinSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wire = CountMinSketchWire { - sketch: self.sketch(), - rows: self.rows, - cols: self.cols, - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wire: CountMinSketchWire = rmp_serde::from_slice(bytes)?; - let backend = sketchlib_cms_from_matrix(wire.rows, wire.cols, &wire.sketch); - Ok(Self { - rows: wire.rows, - cols: wire.cols, - backend, - }) - } -} - -impl MessagePackCodec for CountMinSketchDelta { - /// Encodes the sparse cell delta as the 5-element MessagePack array - /// `[ rows:u64, cols:u64, row_idx:[]u32, col_idx:[]u32, d_count:[]i64 ]` - /// — byte-identical to Rust - /// `rmp_serde::to_vec(&(u64, u64, Vec, Vec, Vec))` - /// (compact mode), the same struct-of-parallel-arrays shape the Go - /// `count_sketch_delta_sparse.go` established. Only the matrix cells - /// cross the wire; the per-row `l1`/`l2` norms and the `hh_keys` - /// heavy-hitter surface are not carried. - fn to_msgpack(&self) -> Result, MsgPackError> { - let mut row_idx: Vec = Vec::with_capacity(self.cells.len()); - let mut col_idx: Vec = Vec::with_capacity(self.cells.len()); - let mut d_count: Vec = Vec::with_capacity(self.cells.len()); - for (r, c, d) in &self.cells { - row_idx.push(*r); - col_idx.push(*c); - d_count.push(*d); - } - Ok(rmp_serde::to_vec(&( - self.rows as u64, - self.cols as u64, - row_idx, - col_idx, - d_count, - ))?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (rows, cols, row_idx, col_idx, d_count): (u64, u64, Vec, Vec, Vec) = - rmp_serde::from_slice(bytes)?; - let cells = row_idx - .into_iter() - .zip(col_idx) - .zip(d_count) - .map(|((r, c), d)| (r, c, d)) - .collect(); - Ok(CountMinSketchDelta { - rows: rows as u32, - cols: cols as u32, - cells, - ..CountMinSketchDelta::default() - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn msgpack_delta_against_empty_round_trips() { - let (rows, cols) = (5, 1024); - let mut w = CountMinSketch::new(rows, cols); - for _ in 0..200 { - w.update("hot", 1.0); - } - let empty = CountMinSketch::new(rows, cols); - let bytes = w.compute_delta_msgpack(&empty, 1.0).unwrap(); - let mut recon = CountMinSketch::new(rows, cols); - recon.apply_delta_msgpack_bytes(&bytes).unwrap(); - let (got, want) = (recon.estimate("hot"), w.estimate("hot")); - assert!((got - want).abs() <= want * 0.1, "got={got} want={want}"); - } - - #[test] - fn msgpack_delta_wire_layout_is_parallel_arrays() { - use crate::message_pack_format::MessagePackCodec; - let (rows, cols) = (3, 8); - let mut w = CountMinSketch::new(rows, cols); - w.update("x", 5.0); - let empty = CountMinSketch::new(rows, cols); - let bytes = w.compute_delta_msgpack(&empty, 1.0).unwrap(); - let cur = w.sketch(); - let (mut ri, mut ci, mut dc): (Vec, Vec, Vec) = - (Vec::new(), Vec::new(), Vec::new()); - for (r, row) in cur.iter().enumerate() { - for (c, &cell) in row.iter().enumerate() { - let d = cell as i64; - if d != 0 { - ri.push(r as u32); - ci.push(c as u32); - dc.push(d); - } - } - } - let expected = rmp_serde::to_vec(&(rows as u64, cols as u64, ri, ci, dc)).unwrap(); - assert_eq!(bytes, expected); - let d = CountMinSketchDelta::from_msgpack(&bytes).unwrap(); - assert_eq!(d.rows, rows as u32); - assert_eq!(d.cols, cols as u32); - } - - #[test] - fn test_count_min_sketch_creation() { - let cms = CountMinSketch::new(4, 1000); - assert_eq!(cms.rows, 4); - assert_eq!(cms.cols, 1000); - let sketch = cms.sketch(); - assert_eq!(sketch.len(), 4); - assert_eq!(sketch[0].len(), 1000); - - // Check all values are initialized to 0 - for row in &sketch { - for &value in row { - assert_eq!(value, 0.0); - } - } - } - - #[test] - fn test_count_min_sketch_update() { - let mut cms = CountMinSketch::new(2, 10); - cms.update("key1", 1.0); - // Query should return at least the updated value - let result = cms.estimate("key1"); - assert!(result >= 1.0); - } - - #[test] - fn test_count_min_sketch_query_empty() { - let cms = CountMinSketch::new(2, 10); - assert_eq!(cms.estimate("anything"), 0.0); - } - - #[test] - fn test_count_min_sketch_merge() { - // Use from_legacy_matrix so the test works regardless of sketchlib/legacy config - let mut sketch1 = vec![vec![0.0; 3]; 2]; - sketch1[0][0] = 5.0; - sketch1[1][2] = 10.0; - let mut cms1 = CountMinSketch::from_legacy_matrix(sketch1, 2, 3); - - let mut sketch2 = vec![vec![0.0; 3]; 2]; - sketch2[0][0] = 3.0; - sketch2[0][1] = 7.0; - let cms2 = CountMinSketch::from_legacy_matrix(sketch2, 2, 3); - - cms1.merge(&cms2).unwrap(); - let merged_sketch = cms1.sketch(); - - assert_eq!(merged_sketch[0][0], 8.0); // 5 + 3 - assert_eq!(merged_sketch[0][1], 7.0); // 0 + 7 - assert_eq!(merged_sketch[1][2], 10.0); // 10 + 0 - } - - #[test] - fn test_count_min_sketch_merge_dimension_mismatch() { - let mut cms1 = CountMinSketch::new(2, 3); - let cms2 = CountMinSketch::new(3, 3); - assert!(cms1.merge(&cms2).is_err()); - } - - #[test] - fn test_count_min_sketch_msgpack_round_trip() { - let mut cms = CountMinSketch::new(4, 256); - cms.update("apple", 5.0); - cms.update("banana", 3.0); - cms.update("apple", 2.0); // total "apple" = 7 - - let bytes = cms.to_msgpack().unwrap(); - let deserialized = CountMinSketch::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.rows, 4); - assert_eq!(deserialized.cols, 256); - assert!(deserialized.estimate("apple") >= 7.0); - assert!(deserialized.estimate("banana") >= 3.0); - } - - #[test] - fn test_aggregate_count() { - let keys = ["a", "b", "a"]; - let values = [1.0, 2.0, 3.0]; - let bytes = CountMinSketch::aggregate_count(4, 100, &keys, &values).unwrap(); - let cms = CountMinSketch::from_msgpack(&bytes).unwrap(); - // "a" was updated twice (1.0 + 3.0 = 4.0), "b" once (2.0) - assert!(cms.estimate("a") >= 4.0); - assert!(cms.estimate("b") >= 2.0); - } - - #[test] - fn test_aggregate_count_empty() { - assert!(CountMinSketch::aggregate_count(4, 100, &[], &[]).is_none()); - } - - #[test] - fn test_apply_delta_additive() { - let mut cms = CountMinSketch::from_legacy_matrix( - vec![vec![1.0, 2.0, 3.0], vec![4.0, 5.0, 6.0]], - 2, - 3, - ); - let delta = CountMinSketchDelta { - rows: 2, - cols: 3, - cells: vec![(0, 0, 10), (1, 2, 100)], - l1: vec![], - l2: vec![], - hh_keys: vec![], - }; - cms.apply_delta(&delta).unwrap(); - assert_eq!( - cms.sketch(), - vec![vec![11.0, 2.0, 3.0], vec![4.0, 5.0, 106.0]] - ); - } - - #[test] - fn test_apply_delta_matches_full_merge() { - let base = CountMinSketch::from_legacy_matrix(vec![vec![1.0, 2.0], vec![3.0, 4.0]], 2, 2); - let addition = - CountMinSketch::from_legacy_matrix(vec![vec![10.0, 0.0], vec![0.0, 20.0]], 2, 2); - let mut via_merge = base.clone(); - via_merge.merge(&addition).unwrap(); - - let delta = CountMinSketchDelta { - rows: 2, - cols: 2, - cells: vec![(0, 0, 10), (1, 1, 20)], - l1: vec![], - l2: vec![], - hh_keys: vec![], - }; - let mut via_delta = base; - via_delta.apply_delta(&delta).unwrap(); - assert_eq!(via_delta.sketch(), via_merge.sketch()); - } - - #[test] - fn test_apply_delta_out_of_range() { - let mut cms = CountMinSketch::new(2, 3); - let delta = CountMinSketchDelta { - rows: 2, - cols: 3, - cells: vec![(5, 0, 1)], - l1: vec![], - l2: vec![], - hh_keys: vec![], - }; - assert!(cms.apply_delta(&delta).is_err()); - } - - /// Cross-language byte-parity guard against `sketchlib-go`'s - /// `CountMinSketch.SerializeProtoBytesFO` output for the - /// deterministic input `goldenCmsKeys()` (10 keys "flow-0"..."flow-9", - /// each repeated 5×, 50 unweighted updates total) at dimensions - /// `(rows=4, cols=2048)` — the - /// `asap-precompute-rs::CMSWrapper::new(4, 2048)` default. The hex - /// blob below was captured from `proto.Marshal` of the Go envelope - /// with `Producer` and `HashSpec` cleared, matching the - /// `integration/parity/golden_test.go::TestGenerateGoldenFixtures` - /// recipe (and the byte-payload that the Rust wrapper's - /// `CMSWrapper::encode_envelope` emits to satisfy - /// `cross_language_parity::cms_byte_parity_with_go`). - /// - /// Frequency-Only payload structure (matches Go's - /// `SerializePortableFO`): - /// - `rows = 4`, `cols = 2048`, `counter_type = INT64` - /// - `counts_int` = packed sint64 row-major, `4*2048 = 8192` cells - /// - `sum_counts` and `sum2_counts` deliberately omitted - /// - `l1[r] = 50` (each row sees 50 unweighted inserts) - /// - `l2[r] = 250` (10 unique cells per row each holding count 5, - /// so `Σ count^2 = 10 * 25 = 250`) - /// - /// Any drift in [`CountMinSketch::update`]'s hash → column-index - /// derivation breaks this test cell-for-cell; that is the contract - /// `cross_language_parity::cms_byte_parity_with_go` in ASAPCollector - /// relies on. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - CountMinState, CounterType, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let rows = 4usize; - let cols = 2048usize; - let mut sk = CountMinSketch::new(rows, cols); - for i in 0..50u64 { - let key = format!("flow-{}", i % 10); - sk.update(&key, 1.0); - } - - // Build the Frequency-Only envelope mirroring sketchlib-go's - // `CountMinSketch.SerializePortableFO`: - // - counter_type = INT64 - // - counts_int = packed sint64 row-major, len = rows * cols - // - sum_counts / sum2_counts deliberately omitted - // - l1[r] = Σ_c count[r][c] (Go maintains this incrementally - // in InsertWithHash; equals `weight * Σ inserts in row r`, - // which collapses to 50 for an unweighted 50-insert stream) - // - l2[r] = Σ_c count[r][c]^2 (Go maintains this as - // `L2[r] += curr*curr - prev*prev`, telescoping to the same - // sum-of-squares for unweighted streams) - let matrix = sk.sketch(); - let mut counts_int: Vec = Vec::with_capacity(rows * cols); - let mut l1: Vec = Vec::with_capacity(rows); - let mut l2: Vec = Vec::with_capacity(rows); - for row in matrix.iter().take(rows) { - let mut row_l1 = 0.0f64; - let mut row_l2 = 0.0f64; - for &cell in row.iter().take(cols) { - counts_int.push(cell as i64); - row_l1 += cell; - row_l2 += cell * cell; - } - l1.push(row_l1); - l2.push(row_l2); - } - - let state = CountMinState { - rows: rows as u32, - cols: cols as u32, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - sum_counts: Vec::new(), - sum2_counts: Vec::new(), - l1, - l2, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::CountMin(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - // 8275-byte hex blob captured from - // `sketchlib-go::CountMinSketch.SerializeProtoBytesFO` for the - // same `(4, 2048) × goldenCmsKeys()` input — see - // `integration/parity/golden_test.go` and - // `cross_language_parity::cms_byte_parity_with_go` in - // ASAPCollector. - const GOLDEN_HEX: &str = include_str!("../../sketches/testdata/cms_envelope_golden.hex"); - let want = decode_hex_cms(GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "CMS envelope length differs: got {} bytes, want {} bytes", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "CMS envelope bytes diverge from sketchlib-go golden" - ); - } - - /// Cross-language byte-parity guard for CountMinDelta hh_keys: encoding the - /// canonical packed delta with heavy-hitter keys at tag 6 must be - /// byte-identical to the Go reference's `SerializeDelta` output for the same - /// state and keys. The empty-`hh_keys` path is already covered by - /// `test_compute_delta_matches_go_golden_bytes`. - #[test] - fn test_hh_keys_matches_go_golden_bytes() { - use crate::proto::sketchlib::CountMinDelta as ProtoDelta; - use prost::Message; - - let rows = 4usize; - let cols = 2048usize; - let mut current = CountMinSketch::new(rows, cols); - for i in 0..50u64 { - current.update(&format!("flow-{}", i % 10), 1.0); - } - let empty = CountMinSketch::new(rows, cols); - - // Inject heavy-hitter keys onto the canonical packed delta at tag 6. - let no_hh = current.compute_delta(&empty, 1.0).unwrap(); - let mut proto = ProtoDelta::decode(no_hh.as_slice()).unwrap(); - proto.hh_keys = vec!["flow-0".into(), "flow-3".into(), "flow-7".into()]; - let got = proto.encode_to_vec(); - - // Captured from the Go reference implementation's SerializeDelta for the - // same input with HHKeys = ["flow-0","flow-3","flow-7"]. - const GOLDEN_HEX: &str = "0804108010222000000000000049400000000000004940000000000000494000000000000049402a200000000000406f400000000000406f400000000000406f400000000000406f403206666c6f772d303206666c6f772d333206666c6f772d374a28000000000000000000000101010101010101010102020202020202020202030303030303030303035250c101b202b608b7088809bc09c20dc60d910fbb0f9f018d04a604c004d6058507a909920b900ec70ead038304bc058007c207cc0a820bd50cbe0fcd0fe2019e02cd02e702a707b809a00ac20cee0dfe0f5a280a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"; - let want = decode_hex_cms(GOLDEN_HEX); - assert_eq!( - got, - want, - "CountMinSketch delta hh_keys bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - - // Symmetric decode: apply_delta_bytes round-trips hh_keys without error. - let mut sink = CountMinSketch::new(rows, cols); - sink.apply_delta_bytes(&got).unwrap(); - } - - fn decode_hex_cms(s: &str) -> Vec { - let s = s.trim(); - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble_cms(pair[0]); - let low = hex_nibble_cms(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble_cms(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - /// `compute_delta` against an EMPTY snapshot reconstructs the window's - /// full state when its bytes are applied to a fresh empty sketch - /// (round-trip). With `threshold = 1.0` every changed cell survives, so - /// applying the delta to a zero base yields the same matrix as the - /// original window. - #[test] - fn test_compute_delta_against_empty_round_trips() { - let rows = 4usize; - let cols = 2048usize; - let mut window = CountMinSketch::new(rows, cols); - for i in 0..200u64 { - window.update(&format!("flow-{}", i % 37), 1.0); - } - let empty = CountMinSketch::new(rows, cols); - - let delta_bytes = window.compute_delta(&empty, 1.0).unwrap(); - - let mut reconstructed = CountMinSketch::new(rows, cols); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - - assert_eq!(reconstructed.sketch(), window.sketch()); - } - - /// A delta computed between two non-empty snapshots reconstructs the - /// current sketch when applied to the base — matching a direct merge of - /// the cell-wise difference. - #[test] - fn test_compute_delta_then_apply_matches_current() { - let mut base = CountMinSketch::new(2, 64); - for i in 0..40u64 { - base.update(&format!("k{}", i % 8), 1.0); - } - let mut current = base.clone(); - for i in 0..30u64 { - current.update(&format!("k{}", i % 8), 1.0); - } - - let delta_bytes = current.compute_delta(&base, 1.0).unwrap(); - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - assert_eq!(reconstructed.sketch(), current.sketch()); - } - - /// Cross-language byte-parity guard: `compute_delta` against an empty - /// snapshot must emit bytes identical to the Go reference - /// implementation's `SerializeDelta(ComputeDelta(empty, current, 1.0))` - /// for the same `(rows=4, cols=2048)` × "flow-0".."flow-9" (each 5×, - /// 50 unweighted updates) input. The golden hex was captured from a - /// `proto.Marshal` of the Go reference's `CountMinDelta` (packed-array - /// encoding) for that input. A delta-against-empty carries the window's - /// full state, so this also pins the per-row L1 (=50) / L2 (=250) norm - /// deltas and the packed cell arrays. - #[test] - fn test_compute_delta_matches_go_golden_bytes() { - let rows = 4usize; - let cols = 2048usize; - let mut current = CountMinSketch::new(rows, cols); - for i in 0..50u64 { - current.update(&format!("flow-{}", i % 10), 1.0); - } - let empty = CountMinSketch::new(rows, cols); - let got = current.compute_delta(&empty, 1.0).unwrap(); - - // Captured from the Go reference implementation's - // SerializeDelta(ComputeDelta(empty, current, 1.0)) for the same input. - const GOLDEN_HEX: &str = "0804108010222000000000000049400000000000004940000000000000494000000000000049402a200000000000406f400000000000406f400000000000406f400000000000406f404a28000000000000000000000101010101010101010102020202020202020202030303030303030303035250c101b202b608b7088809bc09c20dc60d910fbb0f9f018d04a604c004d6058507a909920b900ec70ead038304bc058007c207cc0a820bd50cbe0fcd0fe2019e02cd02e702a707b809a00ac20cee0dfe0f5a280a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"; - let want = decode_hex_cms(GOLDEN_HEX); - assert_eq!( - got, - want, - "CountMinSketch delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } -} diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs deleted file mode 100644 index 742ed7ae..00000000 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! Wire-format-aligned Count-Min sketch + top-k heap composite. -//! -//! No `sketches::*` equivalent — this combines `sketches::CMSHeap` -//! with the wire-format `CountMinSketch` shape and exposes it as a -//! single Go-interop type. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::sketches::countminsketch_heap::CountMinSketchWithHeap as NativeCountMinSketchWithHeap; -use crate::{DataInput, RegularPath, Vector2D}; - -/// Wire-format heap item (key, value) used by the dispatch helpers below. -pub struct WireHeapItem { - pub key: String, - pub value: f64, -} - -/// Concrete Count-Min-with-Heap type backing the wire-format -/// `CountMinSketchWithHeap`. -pub type SketchlibCMSHeap = NativeCountMinSketchWithHeap, RegularPath>; - -/// Creates a fresh CMSHeap with the given dimensions and heap capacity. -pub fn new_sketchlib_cms_heap( - row_num: usize, - col_num: usize, - heap_size: usize, -) -> SketchlibCMSHeap { - NativeCountMinSketchWithHeap::new(row_num, col_num, heap_size) -} - -/// Builds a CMSHeap from an existing sketch matrix and optional heap -/// items. Used when deserializing wire-format state. -pub fn sketchlib_cms_heap_from_matrix_and_heap( - row_num: usize, - col_num: usize, - heap_size: usize, - sketch: &[Vec], - topk_heap: &[WireHeapItem], -) -> SketchlibCMSHeap { - let matrix = Vector2D::from_fn(row_num, col_num, |r, c| { - sketch - .get(r) - .and_then(|row| row.get(c)) - .copied() - .unwrap_or(0.0) - .round() as i64 - }); - let mut cms_heap = NativeCountMinSketchWithHeap::from_storage(matrix, heap_size); - - for item in topk_heap { - let count = item.value.round() as i64; - if count > 0 { - let input = DataInput::Str(&item.key); - cms_heap.heap_mut().update(&input, count); - } - } - - cms_heap -} - -/// Converts a CMSHeap's storage into a `Vec>` matrix. -pub fn matrix_from_sketchlib_cms_heap(cms_heap: &SketchlibCMSHeap) -> Vec> { - let storage = cms_heap.count_min_sketch().as_storage(); - let rows = storage.rows(); - let cols = storage.cols(); - let mut sketch = vec![vec![0.0; cols]; rows]; - - for (r, row) in sketch.iter_mut().enumerate().take(rows) { - for (c, cell) in row.iter_mut().enumerate().take(cols) { - if let Some(v) = storage.get(r, c) { - *cell = *v as f64; - } - } - } - - sketch -} - -/// Converts sketchlib TopKHeap items to wire-format (key, value) pairs. -pub fn heap_to_wire(cms_heap: &SketchlibCMSHeap) -> Vec { - cms_heap - .heap() - .heap() - .iter() - .map(|hh_item| { - let key = match &hh_item.key { - crate::OwnedDataInput::String(s) => s.clone(), - other => format!("{other:?}"), - }; - WireHeapItem { - key, - value: hh_item.count as f64, - } - }) - .collect() -} - -/// Updates a CMSHeap with a weighted key. Automatically updates the heap. -pub fn sketchlib_cms_heap_update(cms_heap: &mut SketchlibCMSHeap, key: &str, value: f64) { - let many = value.round() as i64; - if many <= 0 { - return; - } - cms_heap.update_weighted(&DataInput::String(key.to_owned()), many); -} - -/// Queries a CMSHeap for a key's frequency estimate. -pub fn sketchlib_cms_heap_query(cms_heap: &SketchlibCMSHeap, key: &str) -> f64 { - cms_heap.estimate(&DataInput::String(key.to_owned())) as f64 -} - -/// Item in the top-k heap representing a key-value pair. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CmsHeapItem { - pub key: String, - pub value: f64, -} - -/// Count-Min Sketch with Heap for top-k tracking. Combines probabilistic -/// frequency counting with efficient top-k maintenance. -pub struct CountMinSketchWithHeap { - pub rows: usize, - pub cols: usize, - pub heap_size: usize, - pub(crate) backend: SketchlibCMSHeap, -} - -impl std::fmt::Debug for CountMinSketchWithHeap { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CountMinSketchWithHeap") - .field("rows", &self.rows) - .field("cols", &self.cols) - .field("heap_size", &self.heap_size) - .finish() - } -} - -impl Clone for CountMinSketchWithHeap { - fn clone(&self) -> Self { - // Structural clone: `CMSHeap` derives `Clone` (its `CountMin` + `TopKHeap` - // fields are `Clone`), so copy the backend directly instead of - // extracting the matrix + heap to wire form and rebuilding it. - Self { - rows: self.rows, - cols: self.cols, - heap_size: self.heap_size, - backend: self.backend.clone(), - } - } -} - -impl CountMinSketchWithHeap { - pub fn new(rows: usize, cols: usize, heap_size: usize) -> Self { - Self { - rows, - cols, - heap_size, - backend: new_sketchlib_cms_heap(rows, cols, heap_size), - } - } - - pub fn rows(&self) -> usize { - self.rows - } - - pub fn cols(&self) -> usize { - self.cols - } - - /// Create from a sketch matrix and heap (e.g. from JSON deserialization). - pub fn from_legacy_matrix( - sketch: Vec>, - topk_heap: Vec, - rows: usize, - cols: usize, - heap_size: usize, - ) -> Self { - let wire_heap: Vec = topk_heap - .into_iter() - .map(|h| WireHeapItem { - key: h.key, - value: h.value, - }) - .collect(); - Self { - rows, - cols, - heap_size, - backend: sketchlib_cms_heap_from_matrix_and_heap( - rows, cols, heap_size, &sketch, &wire_heap, - ), - } - } - - /// Get the top-k heap items. - pub fn topk_heap_items(&self) -> Vec { - heap_to_wire(&self.backend) - .into_iter() - .map(|w| CmsHeapItem { - key: w.key, - value: w.value, - }) - .collect() - } - - /// Get the sketch matrix. - pub fn sketch_matrix(&self) -> Vec> { - matrix_from_sketchlib_cms_heap(&self.backend) - } - - pub fn update(&mut self, key: &str, value: f64) { - sketchlib_cms_heap_update(&mut self.backend, key, value); - } - - pub fn estimate(&self, key: &str) -> f64 { - sketchlib_cms_heap_query(&self.backend, key) - } - - /// Merge another CountMinSketchWithHeap into self in place. Both - /// operands must have identical dimensions; the resulting `heap_size` - /// is the minimum of the two. - pub fn merge( - &mut self, - other: &CountMinSketchWithHeap, - ) -> Result<(), Box> { - if self.rows != other.rows || self.cols != other.cols { - return Err(format!( - "CountMinSketchWithHeap dimension mismatch: self={}x{}, other={}x{}", - self.rows, self.cols, other.rows, other.cols - ) - .into()); - } - self.backend.merge(&other.backend); - self.heap_size = self.heap_size.min(other.heap_size); - Ok(()) - } - - pub fn merge_refs( - inputs: &[&CountMinSketchWithHeap], - ) -> Result> { - let first = inputs - .first() - .ok_or("CountMinSketchWithHeap::merge_refs called with empty input")?; - let mut merged = (*first).clone(); - for h in inputs.iter().skip(1) { - merged.merge(h)?; - } - Ok(merged) - } - - pub fn aggregate_topk( - rows: usize, - cols: usize, - heap_size: usize, - keys: &[&str], - values: &[f64], - ) -> Option> { - if keys.is_empty() { - return None; - } - let mut sketch = Self::new(rows, cols, heap_size); - for (key, &value) in keys.iter().zip(values.iter()) { - sketch.update(key, value); - } - sketch.to_msgpack().ok() - } -} - -// ----- Wire format ----- - -/// Inner CMS payload nested inside [`CountMinSketchWithHeapWire`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountMinSketchInnerWire { - pub sketch: Vec>, - #[serde(rename = "row_num")] - pub rows: usize, - #[serde(rename = "col_num")] - pub cols: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountMinSketchWithHeapWire { - pub sketch: CountMinSketchInnerWire, - pub topk_heap: Vec, - pub heap_size: usize, -} - -impl MessagePackCodec for CountMinSketchWithHeap { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wire = CountMinSketchWithHeapWire { - sketch: CountMinSketchInnerWire { - sketch: self.sketch_matrix(), - rows: self.rows, - cols: self.cols, - }, - topk_heap: self.topk_heap_items(), - heap_size: self.heap_size, - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wire: CountMinSketchWithHeapWire = rmp_serde::from_slice(bytes)?; - - let mut sorted_topk_heap = wire.topk_heap; - sorted_topk_heap.sort_by(|a, b| b.value.partial_cmp(&a.value).unwrap()); - - let wire_heap: Vec = sorted_topk_heap - .iter() - .map(|h| WireHeapItem { - key: h.key.clone(), - value: h.value, - }) - .collect(); - let backend = sketchlib_cms_heap_from_matrix_and_heap( - wire.sketch.rows, - wire.sketch.cols, - wire.heap_size, - &wire.sketch.sketch, - &wire_heap, - ); - - Ok(Self { - rows: wire.sketch.rows, - cols: wire.sketch.cols, - heap_size: wire.heap_size, - backend, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_creation() { - let cms = CountMinSketchWithHeap::new(4, 1000, 20); - assert_eq!(cms.rows, 4); - assert_eq!(cms.cols, 1000); - assert_eq!(cms.heap_size, 20); - assert_eq!(cms.sketch_matrix().len(), 4); - assert_eq!(cms.sketch_matrix()[0].len(), 1000); - assert_eq!(cms.topk_heap_items().len(), 0); - } - - #[test] - fn test_query_empty() { - let cms = CountMinSketchWithHeap::new(2, 10, 5); - assert_eq!(cms.estimate("anything"), 0.0); - } - - #[test] - fn test_merge() { - let mut sketch1 = vec![vec![0.0; 10]; 2]; - sketch1[0][0] = 10.0; - sketch1[1][1] = 20.0; - let mut cms1 = CountMinSketchWithHeap::from_legacy_matrix( - sketch1, - vec![ - CmsHeapItem { - key: "key1".to_string(), - value: 100.0, - }, - CmsHeapItem { - key: "key2".to_string(), - value: 50.0, - }, - ], - 2, - 10, - 5, - ); - let mut sketch2 = vec![vec![0.0; 10]; 2]; - sketch2[0][0] = 5.0; - sketch2[1][1] = 15.0; - let cms2 = CountMinSketchWithHeap::from_legacy_matrix( - sketch2, - vec![ - CmsHeapItem { - key: "key3".to_string(), - value: 75.0, - }, - CmsHeapItem { - key: "key1".to_string(), - value: 80.0, - }, - ], - 2, - 10, - 3, - ); - - cms1.merge(&cms2).unwrap(); - - assert_eq!(cms1.sketch_matrix()[0][0], 15.0); - assert_eq!(cms1.sketch_matrix()[1][1], 35.0); - assert_eq!(cms1.heap_size, 3); - assert!(cms1.topk_heap_items().len() <= 3); - } - - #[test] - fn test_merge_dimension_mismatch() { - let mut cms1 = CountMinSketchWithHeap::new(2, 10, 5); - let cms2 = CountMinSketchWithHeap::new(3, 10, 5); - assert!(cms1.merge(&cms2).is_err()); - } - - #[test] - fn test_msgpack_round_trip() { - let mut cms = CountMinSketchWithHeap::new(4, 128, 3); - cms.update("hot", 100.0); - cms.update("cold", 1.0); - - let bytes = cms.to_msgpack().unwrap(); - let deserialized = CountMinSketchWithHeap::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.rows, 4); - assert_eq!(deserialized.cols, 128); - assert_eq!(deserialized.heap_size, 3); - let items = deserialized.topk_heap_items(); - assert!(!items.is_empty()); - let hot = items - .iter() - .find(|item| item.key == "hot") - .expect("'hot' should be in the heap"); - assert!(hot.value >= 100.0); - assert!(deserialized.estimate("hot") >= 100.0); - assert!(deserialized.estimate("cold") >= 1.0); - } - - #[test] - fn test_aggregate_topk() { - let keys = ["a", "b", "a", "c"]; - let values = [1.0, 2.0, 3.0, 0.5]; - let bytes = CountMinSketchWithHeap::aggregate_topk(4, 100, 2, &keys, &values).unwrap(); - let cms = CountMinSketchWithHeap::from_msgpack(&bytes).unwrap(); - assert_eq!(cms.heap_size, 2); - assert!(cms.topk_heap_items().len() <= 2); - } - - #[test] - fn test_aggregate_topk_empty() { - assert!(CountMinSketchWithHeap::aggregate_topk(4, 100, 10, &[], &[]).is_none()); - } -} diff --git a/src/message_pack_format/portable/countsketch.rs b/src/message_pack_format/portable/countsketch.rs deleted file mode 100644 index 7d5ed82f..00000000 --- a/src/message_pack_format/portable/countsketch.rs +++ /dev/null @@ -1,972 +0,0 @@ -//! Wire-format-aligned Count Sketch types. The wire DTO and runtime -//! ops live together here. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; - -// ===================================================================== -// ASAP runtime wire-format-aligned variant . -// -// `CountSketch` and `CountSketchDelta` below are the public-field, -// proto-decode-friendly types consumed by the ASAP query engine -// accumulators. The high-throughput in-process variant is -// `crate::sketches::countsketch::CountSketch`, which keeps its own design. -// ===================================================================== - -// Count Sketch (a.k.a. Count-Min-style signed-counter sketch) — -// element-wise mergeable frequency estimator. -// -// Parallel to `count_min::CountMinSketch` but with **signed** counters, -// matching the `asap_sketchlib::proto::sketchlib::CountSketchState` wire -// format that DataCollector's `countsketchprocessor` emits via the -// modified OTLP `Metric.data = CountSketch{…}` variant. -// -// The surface is: construct from a decoded proto state, merge -// element-wise with another sketch, emit the matrix for queries and -// serialization. The richer query semantics of Count Sketch (median- -// of-estimators heavy-hitter tracking, `TopKState` integration) are not -// implemented here — the wire format carries the matrix losslessly, so -// the merge/store round-trip works with just a matrix. - -/// Default Top-K capacity. Mirrors sketchlib-go `TOPK_SIZE = 100`. -pub const COUNT_SKETCH_TOPK_CAPACITY: usize = 100; - -/// Sparse delta between two consecutive CountSketch snapshots — -/// the input shape for [`CountSketch::apply_delta`]. Mirrors the -/// `CountSketchDelta` proto in -/// `sketchlib-go/proto/countsketch/countsketch.proto` and the native -/// Go `Delta` in `sketchlib-go/sketches/CountSketch/delta.go`. -/// -/// Cells apply additively: `matrix[row][col] += d_count` for each -/// `(row, col, d_count)` triple. Per-row L2 norm deltas apply -/// additively. Heavy-hitter candidate keys (`hh_keys`) are queried -/// against the post-merge matrix and used to rebuild the receiver's -/// Top-K heap. -#[derive(Debug, Clone, Default)] -pub struct CountSketchDelta { - pub rows: u32, - pub cols: u32, - /// `(row, col, d_count)` cell updates, additive on the CS matrix. - pub cells: Vec<(u32, u32, i64)>, - /// Per-row L2 norm deltas. Additive, one scalar per row of the - /// base sketch. Kept on the delta surface for downstream - /// error-accounting; `apply_delta` itself ignores L2. - pub l2: Vec, - /// Heavy-hitter candidate keys forwarded by the upstream - /// Space-Saving tracker. The receiver re-queries the merged CS - /// matrix for each key and updates its Top-K heap with the - /// resulting estimate. Mirrors Go's `Delta.HHKeys`. - pub hh_keys: Vec, -} - -/// Minimal Count Sketch state — a flat `rows × cols` matrix of signed -/// counts. Element-wise mergeable (sum over aligned cells). Mirrors -/// sketchlib-go's `CountSketch.Count`/`TopK` pair (the on-the-wire -/// `L2` field is a derived value and is recomputed on load). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountSketch { - #[serde(rename = "row_num")] - pub rows: usize, - #[serde(rename = "col_num")] - pub cols: usize, - /// Row-major matrix of signed counts. `matrix[r][c]` is the value of - /// hash row `r`, column `c`. - pub matrix: Vec>, - /// Top-K heavy hitters as `(key, count)` pairs, capped at - /// [`COUNT_SKETCH_TOPK_CAPACITY`]. Order is not guaranteed (heap - /// shape is not preserved on the wire). Mirrors Go's - /// `CountSketch.TopK` slot. Defaults to empty on legacy payloads. - #[serde(default)] - pub topk: Vec<(String, f64)>, -} - -impl CountSketch { - /// Construct an all-zero sketch with the given dimensions. - pub fn new(rows: usize, cols: usize) -> Self { - Self { - rows, - cols, - matrix: vec![vec![0.0; cols]; rows], - topk: Vec::new(), - } - } - - /// Construct from a pre-built matrix (used by the modified-OTLP - /// proto-decode path). `topk` is zero-initialised; callers that - /// need non-zero auxiliary state should use the msgpack/proto path. - pub fn from_legacy_matrix(matrix: Vec>, rows: usize, cols: usize) -> Self { - debug_assert_eq!(matrix.len(), rows, "row count mismatch"); - debug_assert!( - matrix.iter().all(|r| r.len() == cols), - "column count mismatch in at least one row" - ); - Self { - rows, - cols, - matrix, - topk: Vec::new(), - } - } - - /// Borrow the inner matrix. - pub fn sketch(&self) -> &Vec> { - &self.matrix - } - - /// Update the in-memory Top-K heap with `(key, count)`. Keeps the - /// heap bounded by [`COUNT_SKETCH_TOPK_CAPACITY`]; on overflow, - /// drops the smallest-count entry. If `key` is already present, - /// the new count replaces the old (max semantics). Used by - /// `apply_delta` to rebuild Top-K from `hh_keys`. - fn topk_update(&mut self, key: &str, count: f64) { - if let Some(slot) = self.topk.iter_mut().find(|(k, _)| k == key) { - if count > slot.1 { - slot.1 = count; - } - return; - } - if self.topk.len() < COUNT_SKETCH_TOPK_CAPACITY { - self.topk.push((key.to_owned(), count)); - return; - } - // Capacity hit: replace the minimum if `count` exceeds it. - if let Some((min_idx, min_count)) = self - .topk - .iter() - .enumerate() - .min_by(|a, b| { - a.1.1 - .partial_cmp(&b.1.1) - .unwrap_or(std::cmp::Ordering::Equal) - }) - .map(|(i, e)| (i, e.1)) - { - if count > min_count { - self.topk[min_idx] = (key.to_owned(), count); - } - } - } - - /// Insert a single weighted observation. Uses the shared - /// [`crate::MatrixHash::Packed64`] fast-path hash so the matrix-cell - /// layout matches `sketchlib-go::CountSketch.UpdateString` - /// bit-for-bit. Locked in by - /// `tests/sketches_go_parity_probe.rs`. - /// - /// Constraints inherited from Go's Packed64 mode: - /// - `cols` must be a power of two; the column mask is `cols - 1`. - /// - `rows * (log2(cols) + 1) ≤ 64` so the packed hash holds all - /// row indices and sign bits. - pub fn update(&mut self, key: &str, value: f64) { - if self.rows == 0 || self.cols == 0 { - return; - } - let input = crate::common::DataInput::String(key.to_owned()); - let hashed = crate::common::MatrixHash::Packed64( - ::hash64_seeded(0, &input), - ); - for r in 0..self.rows { - let col = ::col_for_row( - &hashed, r, self.cols, - ); - let sign = ::sign_for_row( - &hashed, r, - ) as f64; - self.matrix[r][col] += sign * value; - } - } - - /// Estimate the frequency of `key` via the standard median-of-rows - /// CountSketch query. Returns 0 for an empty sketch. Inverts the - /// signed-counter projection that [`Self::update`] applied. - pub fn estimate(&self, key: &str) -> f64 { - if self.rows == 0 || self.cols == 0 { - return 0.0; - } - let input = crate::common::DataInput::String(key.to_owned()); - let hashed = crate::common::MatrixHash::Packed64( - ::hash64_seeded(0, &input), - ); - let mut estimates: Vec = Vec::with_capacity(self.rows); - for r in 0..self.rows { - let col = ::col_for_row( - &hashed, r, self.cols, - ); - let sign = ::sign_for_row( - &hashed, r, - ) as f64; - estimates.push(sign * self.matrix[r][col]); - } - estimates.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let mid = estimates.len() / 2; - if estimates.len() % 2 == 1 { - estimates[mid] - } else { - (estimates[mid - 1] + estimates[mid]) / 2.0 - } - } - - /// Merge one other sketch into self via element-wise addition. Both - /// operands must have identical dimensions. - pub fn merge( - &mut self, - other: &CountSketch, - ) -> Result<(), Box> { - if self.rows != other.rows || self.cols != other.cols { - return Err(format!( - "CountSketch dimension mismatch: self={}x{}, other={}x{}", - self.rows, self.cols, other.rows, other.cols - ) - .into()); - } - for r in 0..self.rows { - for c in 0..self.cols { - self.matrix[r][c] += other.matrix[r][c]; - } - } - Ok(()) - } - - /// Apply a sparse delta in place. Matches the `ApplyDelta` - /// semantics in `sketchlib-go/sketches/CountSketch/delta.go`: - /// * each `(row, col, d_count)` triple updates the count matrix - /// additively (`matrix[r][c] += d_count`); - /// * each `hh_key` is re-queried against the post-update matrix - /// and pushed into the receiver's Top-K with the merged-estimate - /// count (mirrors Go's `Delta.HHKeys` heavy-hitter rebuild). - /// - /// Returns `Err` if any `(row, col)` is out of range — indicating - /// a dimension mismatch between the snapshot this sketch was - /// built from and the delta sender. - pub fn apply_delta( - &mut self, - delta: &CountSketchDelta, - ) -> Result<(), Box> { - // 1. Cell additions. - for (row, col, d_count) in &delta.cells { - let r = *row as usize; - let c = *col as usize; - if r >= self.rows || c >= self.cols { - return Err(format!( - "CountSketchDelta cell ({r},{c}) out of range (matrix={}x{})", - self.rows, self.cols - ) - .into()); - } - // `d_count` is signed on the wire; CS counts are signed - // too (can go negative under adversarial keys). - self.matrix[r][c] += *d_count as f64; - } - // 2. Heavy-hitter rebuild from `hh_keys`. Re-estimate against - // the freshly-updated matrix and push into Top-K with the - // merged count. Mirrors sketchlib-go's `Delta.HHKeys` path. - for key in &delta.hh_keys { - let est = self.estimate(key); - self.topk_update(key, est); - } - Ok(()) - } - - /// Compute a sparse, proto-marshalled `CountSketchDelta` of `self` - /// against a `snapshot`. A cell is included when its signed count - /// delta `Δcount` (self − snapshot) is non-zero and its magnitude is - /// `>= threshold` (the threshold is truncated to an integer, matching - /// the Go reference). The full per-row L2 norm deltas are always - /// carried (one entry per row). - /// - /// This is the Rust twin of the Go reference implementation's - /// `ComputeDelta` + `SerializeDelta`: it iterates the matrix in - /// row-major order, subtracts the snapshot's value for each cell, and - /// emits the surviving signed cell deltas using the packed-array proto - /// encoding (`cell_rows`/`cell_cols`/`d_counts`). L2 row deltas are - /// derived from the matrices — `l2[r] = Σ_c count[r][c]^2` — matching - /// the Go producer's incrementally-maintained norm. Heavy-hitter - /// candidate keys (`hh_keys`) are sourced from an upstream Space-Saving - /// tracker that this minimal wrapper does not maintain, so the field is - /// left empty (an empty repeated field encodes to nothing on the wire). - /// The returned bytes are a `prost`-encoded - /// [`crate::proto::sketchlib::CountSketchDelta`], byte-identical to the - /// Go `proto.Marshal(CountSketchDelta)` output for the same inputs when - /// no heavy-hitter keys are forwarded (cross-language byte parity). - /// - /// Delta-against-empty: when `snapshot` is the all-zero sketch, every - /// surviving cell delta equals this window's own (signed) cell count, - /// so the result is this window's full state encoded as a delta (no - /// cross-window subtraction). CountSketch deltas carry only - /// sketch-internal cells/norms — there are no DataPoint-level metric - /// scalars to drop. - pub fn compute_delta( - &self, - snapshot: &CountSketch, - threshold: f64, - ) -> Result, Box> { - use crate::proto::sketchlib::CountSketchDelta as ProtoDelta; - use prost::Message; - - if self.rows != snapshot.rows || self.cols != snapshot.cols { - return Err(format!( - "CountSketch dimension mismatch: self={}x{}, snapshot={}x{}", - self.rows, self.cols, snapshot.rows, snapshot.cols - ) - .into()); - } - - // Threshold is compared against |Δ| after truncation to an - // integer, mirroring the Go reference's `int64(threshold)` cast. - let thresh = threshold as i64; - - let mut cell_rows: Vec = Vec::new(); - let mut cell_cols: Vec = Vec::new(); - let mut d_counts: Vec = Vec::new(); - let mut l2: Vec = Vec::with_capacity(self.rows); - - for r in 0..self.rows { - let mut cur_l2 = 0.0f64; - let mut snap_l2 = 0.0f64; - for c in 0..self.cols { - let cv = self.matrix[r][c]; - let sv = snapshot.matrix[r][c]; - cur_l2 += cv * cv; - snap_l2 += sv * sv; - let dc = (cv - sv) as i64; - if dc != 0 && (dc <= -thresh || dc >= thresh) { - cell_rows.push(r as u32); - cell_cols.push(c as u32); - d_counts.push(dc); - } - } - l2.push(cur_l2 - snap_l2); - } - - let delta = ProtoDelta { - rows: self.rows as u32, - cols: self.cols as u32, - cells_legacy: Vec::new(), - l2_legacy: Vec::new(), - topk: None, - hh_keys: Vec::new(), - cell_rows, - cell_cols, - d_counts, - l2, - }; - Ok(delta.encode_to_vec()) - } - - /// Apply a `prost`-encoded [`crate::proto::sketchlib::CountSketchDelta`] - /// to this sketch in place (additive signed-cell merge + heavy-hitter - /// rebuild). The Rust twin of the Go reference implementation's - /// `DeserializeDelta` + `ApplyDelta`. Reads the packed cell arrays - /// (`cell_rows`/`cell_cols`/`d_counts`) and the heavy-hitter keys - /// (`hh_keys`), falling back to the legacy per-cell records and TopK - /// entries for payloads from older producers. - /// - /// Returns `Err` if `bytes` is not a valid `CountSketchDelta` proto or - /// a cell is out of range for this sketch's dimensions. - pub fn apply_delta_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - use crate::proto::sketchlib::CountSketchDelta as ProtoDelta; - use prost::Message; - - let proto = ProtoDelta::decode(bytes)?; - let cells: Vec<(u32, u32, i64)> = if !proto.cell_rows.is_empty() { - proto - .cell_rows - .iter() - .zip(proto.cell_cols.iter()) - .zip(proto.d_counts.iter()) - .map(|((&r, &c), &d)| (r, c, d)) - .collect() - } else { - proto - .cells_legacy - .iter() - .map(|c| (c.row, c.col, c.d_count as i64)) - .collect() - }; - let l2 = if !proto.l2.is_empty() { - proto.l2 - } else { - proto.l2_legacy - }; - let hh_keys = if !proto.hh_keys.is_empty() { - proto.hh_keys - } else if let Some(topk) = proto.topk { - topk.entries.into_iter().map(|e| e.key).collect() - } else { - Vec::new() - }; - let delta = CountSketchDelta { - rows: proto.rows, - cols: proto.cols, - cells, - l2, - hh_keys, - }; - self.apply_delta(&delta) - } - - /// MessagePack twin of [`Self::compute_delta`]. Computes the same - /// sparse signed-cell delta of `self` against `snapshot` (a cell is - /// carried when `Δcount` is non-zero and `|Δcount| >= threshold`, the - /// threshold truncated to an integer), but serializes it as the - /// cells-only parallel-array MessagePack layout via - /// [`CountSketchDelta`]'s [`MessagePackCodec`] instead of proto. The - /// per-row `l2` norms and `hh_keys` the proto delta carries are dropped - /// — the plain-sketch estimate path does not consume them. - /// Delta-against-empty carries this window's own signed cell counts. - pub fn compute_delta_msgpack( - &self, - snapshot: &CountSketch, - threshold: f64, - ) -> Result, Box> { - if self.rows != snapshot.rows || self.cols != snapshot.cols { - return Err(format!( - "CountSketch dimension mismatch: self={}x{}, snapshot={}x{}", - self.rows, self.cols, snapshot.rows, snapshot.cols - ) - .into()); - } - let thresh = threshold as i64; - let mut delta = CountSketchDelta { - rows: self.rows as u32, - cols: self.cols as u32, - ..CountSketchDelta::default() - }; - for r in 0..self.rows { - for c in 0..self.cols { - let dc = (self.matrix[r][c] - snapshot.matrix[r][c]) as i64; - if dc != 0 && (dc <= -thresh || dc >= thresh) { - delta.cells.push((r as u32, c as u32, dc)); - } - } - } - Ok(delta.to_msgpack()?) - } - - /// MessagePack twin of [`Self::apply_delta_bytes`]. Decodes the - /// cells-only parallel-array MessagePack [`CountSketchDelta`] and - /// applies it in place (additive signed-cell merge). - /// - /// Returns `Err` if `bytes` is not a valid MessagePack - /// `CountSketchDelta` or a cell is out of range for this sketch's - /// dimensions. - pub fn apply_delta_msgpack_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - let delta = CountSketchDelta::from_msgpack(bytes)?; - self.apply_delta(&delta) - } - - /// Merge a slice of references into a single new sketch. All inputs - /// must share the same dimensions; returns `Err` on mismatch or an - /// empty input. - pub fn merge_refs( - inputs: &[&CountSketch], - ) -> Result> { - let first = inputs - .first() - .ok_or("CountSketch::merge_refs called with empty input")?; - let mut merged = CountSketch::new(first.rows, first.cols); - for cs in inputs { - merged.merge(cs)?; - } - Ok(merged) - } -} - -impl MessagePackCodec for CountSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) - } -} - -impl MessagePackCodec for CountSketchDelta { - /// Encodes the sparse signed-cell delta as the 5-element MessagePack - /// array - /// `[ rows:u64, cols:u64, row_idx:[]u32, col_idx:[]u32, d_count:[]i64 ]` - /// — byte-identical to Rust - /// `rmp_serde::to_vec(&(u64, u64, Vec, Vec, Vec))` - /// (compact mode), the layout the Go `count_sketch_delta_sparse.go` - /// established (with signed `d_count` in place of the F2-broadcast's - /// `f64` vals). Only the matrix cells cross the wire; the per-row `l2` - /// norms and `hh_keys` heavy-hitter surface are not carried. - fn to_msgpack(&self) -> Result, MsgPackError> { - let mut row_idx: Vec = Vec::with_capacity(self.cells.len()); - let mut col_idx: Vec = Vec::with_capacity(self.cells.len()); - let mut d_count: Vec = Vec::with_capacity(self.cells.len()); - for (r, c, d) in &self.cells { - row_idx.push(*r); - col_idx.push(*c); - d_count.push(*d); - } - Ok(rmp_serde::to_vec(&( - self.rows as u64, - self.cols as u64, - row_idx, - col_idx, - d_count, - ))?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (rows, cols, row_idx, col_idx, d_count): (u64, u64, Vec, Vec, Vec) = - rmp_serde::from_slice(bytes)?; - let cells = row_idx - .into_iter() - .zip(col_idx) - .zip(d_count) - .map(|((r, c), d)| (r, c, d)) - .collect(); - Ok(CountSketchDelta { - rows: rows as u32, - cols: cols as u32, - cells, - ..CountSketchDelta::default() - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn msgpack_delta_against_empty_round_trips() { - let (rows, cols) = (5, 1024); - let mut w = CountSketch::new(rows, cols); - for _ in 0..200 { - w.update("hot", 1.0); - } - let empty = CountSketch::new(rows, cols); - let bytes = w.compute_delta_msgpack(&empty, 1.0).unwrap(); - let mut recon = CountSketch::new(rows, cols); - recon.apply_delta_msgpack_bytes(&bytes).unwrap(); - // CountSketch round-trips its cells exactly under delta-against-empty. - assert_eq!(recon.sketch(), w.sketch()); - } - - #[test] - fn msgpack_delta_wire_layout_is_parallel_arrays() { - use crate::message_pack_format::MessagePackCodec; - let (rows, cols) = (3, 8); - let mut w = CountSketch::new(rows, cols); - w.update("x", 5.0); - let empty = CountSketch::new(rows, cols); - let bytes = w.compute_delta_msgpack(&empty, 1.0).unwrap(); - let cur = w.sketch(); - let (mut ri, mut ci, mut dc): (Vec, Vec, Vec) = - (Vec::new(), Vec::new(), Vec::new()); - for (r, row) in cur.iter().enumerate() { - for (c, &cell) in row.iter().enumerate() { - let d = cell as i64; - if d != 0 { - ri.push(r as u32); - ci.push(c as u32); - dc.push(d); - } - } - } - let expected = rmp_serde::to_vec(&(rows as u64, cols as u64, ri, ci, dc)).unwrap(); - assert_eq!(bytes, expected); - let d = CountSketchDelta::from_msgpack(&bytes).unwrap(); - assert_eq!(d.rows, rows as u32); - assert_eq!(d.cols, cols as u32); - } - - #[test] - fn test_new_empty() { - let cs = CountSketch::new(2, 3); - assert_eq!(cs.rows, 2); - assert_eq!(cs.cols, 3); - assert_eq!(cs.sketch(), &vec![vec![0.0, 0.0, 0.0], vec![0.0, 0.0, 0.0]]); - } - - #[test] - fn test_from_legacy_matrix() { - let m = vec![vec![1.0, -2.0, 3.0], vec![-4.0, 5.0, -6.0]]; - let cs = CountSketch::from_legacy_matrix(m.clone(), 2, 3); - assert_eq!(cs.sketch(), &m); - } - - #[test] - fn test_merge_element_wise() { - let mut a = CountSketch::from_legacy_matrix(vec![vec![1.0, 2.0], vec![3.0, 4.0]], 2, 2); - let b = CountSketch::from_legacy_matrix(vec![vec![-1.0, -2.0], vec![-3.0, -4.0]], 2, 2); - a.merge(&b).unwrap(); - assert_eq!(a.sketch(), &vec![vec![0.0, 0.0], vec![0.0, 0.0]]); - } - - #[test] - fn test_merge_dimension_mismatch() { - let mut a = CountSketch::new(2, 3); - let b = CountSketch::new(3, 3); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_merge_refs() { - let a = CountSketch::from_legacy_matrix(vec![vec![1.0, 2.0]], 1, 2); - let b = CountSketch::from_legacy_matrix(vec![vec![3.0, 4.0]], 1, 2); - let c = CountSketch::from_legacy_matrix(vec![vec![5.0, 6.0]], 1, 2); - let merged = CountSketch::merge_refs(&[&a, &b, &c]).unwrap(); - assert_eq!(merged.sketch(), &vec![vec![9.0, 12.0]]); - } - - #[test] - fn test_apply_delta_additive() { - let mut cs = CountSketch::from_legacy_matrix( - vec![vec![1.0, -2.0, 3.0], vec![-4.0, 5.0, -6.0]], - 2, - 3, - ); - let delta = CountSketchDelta { - rows: 2, - cols: 3, - cells: vec![ - (0, 0, 10), // 1 + 10 = 11 - (0, 2, -3), // 3 - 3 = 0 - (1, 1, -15), // 5 - 15 = -10 - ], - l2: vec![], - hh_keys: vec![], - }; - cs.apply_delta(&delta).unwrap(); - assert_eq!( - cs.sketch(), - &vec![vec![11.0, -2.0, 0.0], vec![-4.0, -10.0, -6.0]] - ); - } - - #[test] - fn test_apply_delta_matches_full_merge() { - let base = CountSketch::from_legacy_matrix(vec![vec![1.0, 2.0], vec![3.0, 4.0]], 2, 2); - let addition = - CountSketch::from_legacy_matrix(vec![vec![10.0, 0.0], vec![0.0, 20.0]], 2, 2); - let mut via_merge = base.clone(); - via_merge.merge(&addition).unwrap(); - - let delta = CountSketchDelta { - rows: 2, - cols: 2, - cells: vec![(0, 0, 10), (1, 1, 20)], - l2: vec![], - hh_keys: vec![], - }; - let mut via_delta = base; - via_delta.apply_delta(&delta).unwrap(); - assert_eq!(via_delta.sketch(), via_merge.sketch()); - } - - #[test] - fn test_apply_delta_out_of_range() { - let mut cs = CountSketch::new(2, 3); - let delta = CountSketchDelta { - rows: 2, - cols: 3, - cells: vec![(2, 0, 1)], // row 2 out of range for 2-row matrix - l2: vec![], - hh_keys: vec![], - }; - assert!(cs.apply_delta(&delta).is_err()); - } - - #[test] - fn test_apply_delta_rebuilds_topk_from_hh_keys() { - // Build a sketch with two known keys via the in-process - // `update` path so the matrix has a coherent shape, then - // send a delta that only carries `hh_keys` entries. The - // receiver should re-query the merged matrix and populate - // `topk` with the resulting estimates. Mirrors sketchlib-go's - // `Delta.HHKeys` heavy-hitter rebuild path. - let mut cs = CountSketch::new(3, 16); - cs.update("alpha", 5.0); - cs.update("beta", 3.0); - let delta = CountSketchDelta { - rows: 3, - cols: 16, - cells: vec![], - l2: vec![], - hh_keys: vec!["alpha".to_string(), "beta".to_string()], - }; - cs.apply_delta(&delta).unwrap(); - assert_eq!(cs.topk.len(), 2); - let alpha_count = cs - .topk - .iter() - .find(|(k, _)| k == "alpha") - .map(|(_, v)| *v) - .expect("alpha should be in topk"); - let beta_count = cs - .topk - .iter() - .find(|(k, _)| k == "beta") - .map(|(_, v)| *v) - .expect("beta should be in topk"); - // Alpha was inserted with weight 5; the median estimate - // should exceed beta's (weight 3) modulo signed-counter - // cancellation in this small 3x16 matrix. - assert!( - alpha_count > beta_count, - "alpha={alpha_count} beta={beta_count}" - ); - } - - #[test] - fn test_apply_delta_hh_keys_topk_capacity() { - // Verify the Top-K heap is bounded by COUNT_SKETCH_TOPK_CAPACITY - // and that on overflow, the smallest-count entry is evicted in - // favor of a larger-count newcomer. - let mut cs = CountSketch::new(3, 1024); - let n = COUNT_SKETCH_TOPK_CAPACITY + 5; - let keys: Vec = (0..n).map(|i| format!("k{i:04}")).collect(); - // Fill all keys into the matrix so estimates are non-zero. - for (i, k) in keys.iter().enumerate() { - cs.update(k, (i + 1) as f64); - } - let delta = CountSketchDelta { - rows: 3, - cols: 1024, - cells: vec![], - l2: vec![], - hh_keys: keys.clone(), - }; - cs.apply_delta(&delta).unwrap(); - assert_eq!(cs.topk.len(), COUNT_SKETCH_TOPK_CAPACITY); - } - - #[test] - fn test_msgpack_round_trip() { - let original = - CountSketch::from_legacy_matrix(vec![vec![1.5, -2.5], vec![3.5, -4.5]], 2, 2); - let bytes = original.to_msgpack().unwrap(); - let decoded = CountSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(decoded.sketch(), original.sketch()); - assert_eq!(decoded.rows, original.rows); - assert_eq!(decoded.cols, original.cols); - } - - /// Cross-language byte-parity guard against `sketchlib-go`'s - /// `CountSketch.SerializeProtoBytes` output for the deterministic - /// input `goldenCsKeys()` (25 keys "k-a"..."k-e", each repeated 5×) - /// at dimensions `(rows=3, cols=512)`. The hex blob below was - /// captured from a `proto.Marshal` of the Go envelope with - /// `Producer` and `HashSpec` cleared (matching the - /// `integration/parity/golden_test.go::TestGenerateGoldenFixtures` - /// recipe used for DDSketch and KLL). - /// - /// Any drift in [`CountSketch::update`]'s hash → (col, sign) - /// derivation breaks this test cell-for-cell; that is the contract - /// `cross_language_parity::countsketch_byte_parity_with_go` in - /// ASAPCollector relies on. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - CountSketchState, CounterType, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let rows = 3usize; - let cols = 512usize; - let mut sk = CountSketch::new(rows, cols); - for i in 0..25 { - let key = format!("k-{}", (b'a' + (i % 5) as u8) as char); - sk.update(&key, 1.0); - } - - // Build envelope mirroring sketchlib-go's CountSketch.SerializePortable: - // counter_type = INT64, counts packed sint64 in row-major - // order, l2 = per-row sum of squared cells, no TopK. - let mut counts_int: Vec = Vec::with_capacity(rows * cols); - let mut l2: Vec = Vec::with_capacity(rows); - for row in sk.matrix.iter().take(rows) { - let mut row_l2 = 0.0f64; - for &cell in row.iter().take(cols) { - counts_int.push(cell as i64); - row_l2 += cell * cell; - } - l2.push(row_l2); - } - - let state = CountSketchState { - rows: rows as u32, - cols: cols as u32, - counter_type: CounterType::Int64 as i32, - counts_int, - counts_float: Vec::new(), - l2, - topk: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::CountSketch(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - // Byte string captured from sketchlib-go for the same - // (3,512) × `goldenCsKeys()` input — see - // `integration/parity/golden_test.go` and - // `cross_language_parity::countsketch_byte_parity_with_go` - // in ASAPCollector. 1577 bytes total: a `SketchEnvelope` proto - // wrapping a `CountSketchState` whose `counts_int` is a packed - // sint64 (zigzag) row-major matrix and `l2` is `[125.0;3]` - // (each row's 5 hot cells hold ±5 → l2 = 5·25 = 125). - const GOLDEN_HEX: &str = "08015aa40c0803108004180222800c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000090000000000000a000000000000000900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032180000000000405f400000000000405f400000000000405f40"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "CountSketch envelope bytes diverge from sketchlib-go golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - /// Cross-language byte-parity guard for CountSketchDelta hh_keys: encoding - /// the canonical packed delta with heavy-hitter keys at tag 6 must be - /// byte-identical to the Go reference's `SerializeDelta` output for the same - /// state and keys. The empty-`hh_keys` path is already covered by - /// `test_compute_delta_matches_go_golden_bytes`. - #[test] - fn test_hh_keys_matches_go_golden_bytes() { - use crate::proto::sketchlib::CountSketchDelta as ProtoDelta; - use prost::Message; - - let rows = 3usize; - let cols = 512usize; - let mut current = CountSketch::new(rows, cols); - for i in 0..25 { - let key = format!("k-{}", (b'a' + (i % 5) as u8) as char); - current.update(&key, 1.0); - } - let empty = CountSketch::new(rows, cols); - - // Inject heavy-hitter keys onto the canonical packed delta at tag 6. - let no_hh = current.compute_delta(&empty, 1.0).unwrap(); - let mut proto = ProtoDelta::decode(no_hh.as_slice()).unwrap(); - proto.hh_keys = vec!["flow-0".into(), "flow-3".into(), "flow-7".into()]; - let got = proto.encode_to_vec(); - - // Captured from the Go reference implementation's SerializeDelta for the - // same input with HHKeys = ["flow-0","flow-3","flow-7"]. - const GOLDEN_HEX: &str = "08031080043206666c6f772d303206666c6f772d333206666c6f772d374a0f000000000001010101010202020202521a3d7fd9019402d3034eb9019602f703fe0306a501c601d201ba025a0f090a0909090a090a090a09090a090962180000000000405f400000000000405f400000000000405f40"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "CountSketch delta hh_keys bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - fn decode_hex(s: &str) -> Vec { - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - /// `compute_delta` against an EMPTY snapshot reconstructs the window's - /// full (signed) state when its bytes are applied to a fresh empty - /// sketch (round-trip). With `threshold = 1.0` every changed cell - /// survives. - #[test] - fn test_compute_delta_against_empty_round_trips() { - let rows = 3usize; - let cols = 512usize; - let mut window = CountSketch::new(rows, cols); - for i in 0..200u64 { - window.update(&format!("k-{}", i % 23), 1.0); - } - let empty = CountSketch::new(rows, cols); - - let delta_bytes = window.compute_delta(&empty, 1.0).unwrap(); - - let mut reconstructed = CountSketch::new(rows, cols); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - - assert_eq!(reconstructed.sketch(), window.sketch()); - } - - /// A delta computed between two non-empty snapshots reconstructs the - /// current sketch when applied to the base. - #[test] - fn test_compute_delta_then_apply_matches_current() { - let mut base = CountSketch::new(3, 64); - for i in 0..40u64 { - base.update(&format!("x{}", i % 8), 1.0); - } - let mut current = base.clone(); - for i in 0..30u64 { - current.update(&format!("x{}", i % 8), 1.0); - } - - let delta_bytes = current.compute_delta(&base, 1.0).unwrap(); - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - assert_eq!(reconstructed.sketch(), current.sketch()); - } - - /// Cross-language byte-parity guard: `compute_delta` against an empty - /// snapshot must emit bytes identical to the Go reference - /// implementation's `SerializeDelta(ComputeDelta(empty, current, 1.0))` - /// for the same `(rows=3, cols=512)` × "k-a".."k-e" (each 5×, 25 - /// unweighted updates) input. The golden hex was captured from a - /// `proto.Marshal` of the Go reference's `CountSketchDelta` - /// (packed-array encoding). No heavy-hitter keys are forwarded by this - /// minimal wrapper, so `hh_keys` is empty in both producers; the golden - /// pins the packed cell arrays and the per-row L2 (=125) norm deltas. - #[test] - fn test_compute_delta_matches_go_golden_bytes() { - let rows = 3usize; - let cols = 512usize; - let mut current = CountSketch::new(rows, cols); - for i in 0..25 { - let key = format!("k-{}", (b'a' + (i % 5) as u8) as char); - current.update(&key, 1.0); - } - let empty = CountSketch::new(rows, cols); - let got = current.compute_delta(&empty, 1.0).unwrap(); - - // Captured from the Go reference implementation's - // SerializeDelta(ComputeDelta(empty, current, 1.0)) for the same input. - const GOLDEN_HEX: &str = "08031080044a0f000000000001010101010202020202521a3d7fd9019402d3034eb9019602f703fe0306a501c601d201ba025a0f090a0909090a090a090a09090a090962180000000000405f400000000000405f400000000000405f40"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "CountSketch delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } -} diff --git a/src/message_pack_format/portable/countsketch_topk.rs b/src/message_pack_format/portable/countsketch_topk.rs deleted file mode 100644 index a153143c..00000000 --- a/src/message_pack_format/portable/countsketch_topk.rs +++ /dev/null @@ -1,471 +0,0 @@ -//! Wire-format-aligned Count Sketch + top-k heap composite. -//! -//! No `sketches::*` equivalent — this combines `sketches::CSHeap` with -//! the wire-format `CountSketch` shape and exposes it as a single -//! Go-interop type. Mirrors [`crate::message_pack_format::portable::countminsketch_topk::CountMinSketchWithHeap`] -//! structurally, but backed by `CSHeap` (median estimator, signed rows) -//! instead of `CMSHeap` (min-over-rows estimator) — the two are -//! different sketch algorithms that happen to produce a same-shaped -//! `rows x cols` matrix, not interchangeable data. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::sketches::countsketch_heap::CountSketchWithHeap as NativeCountSketchWithHeap; -use crate::{DataInput, RegularPath, Vector2D}; - -/// Wire-format heap item (key, value) used by the dispatch helpers below. -pub struct WireHeapItem { - pub key: String, - pub value: f64, -} - -/// Concrete Count-Sketch-with-Heap type backing the wire-format -/// `CountSketchWithHeap`. -pub type SketchlibCSHeap = NativeCountSketchWithHeap, RegularPath>; - -/// Creates a fresh CSHeap with the given dimensions and heap capacity. -pub fn new_sketchlib_cs_heap(row_num: usize, col_num: usize, heap_size: usize) -> SketchlibCSHeap { - NativeCountSketchWithHeap::new(row_num, col_num, heap_size) -} - -/// Builds a CSHeap from an existing sketch matrix and optional heap -/// items. Used when deserializing wire-format state. -pub fn sketchlib_cs_heap_from_matrix_and_heap( - row_num: usize, - col_num: usize, - heap_size: usize, - sketch: &[Vec], - topk_heap: &[WireHeapItem], -) -> SketchlibCSHeap { - let matrix = Vector2D::from_fn(row_num, col_num, |r, c| { - sketch - .get(r) - .and_then(|row| row.get(c)) - .copied() - .unwrap_or(0.0) - .round() as i64 - }); - let mut cs_heap = NativeCountSketchWithHeap::from_storage(matrix, heap_size); - - for item in topk_heap { - let count = item.value.round() as i64; - if count > 0 { - let input = DataInput::Str(&item.key); - cs_heap.heap_mut().update(&input, count); - } - } - - cs_heap -} - -/// Converts a CSHeap's storage into a `Vec>` matrix. -pub fn matrix_from_sketchlib_cs_heap(cs_heap: &SketchlibCSHeap) -> Vec> { - let storage = cs_heap.count_sketch().as_storage(); - let rows = storage.rows(); - let cols = storage.cols(); - let mut sketch = vec![vec![0.0; cols]; rows]; - - for (r, row) in sketch.iter_mut().enumerate().take(rows) { - for (c, cell) in row.iter_mut().enumerate().take(cols) { - if let Some(v) = storage.get(r, c) { - *cell = *v as f64; - } - } - } - - sketch -} - -/// Converts sketchlib TopKHeap items to wire-format (key, value) pairs. -pub fn heap_to_wire(cs_heap: &SketchlibCSHeap) -> Vec { - cs_heap - .heap() - .heap() - .iter() - .map(|hh_item| { - let key = match &hh_item.key { - crate::OwnedDataInput::String(s) => s.clone(), - other => format!("{other:?}"), - }; - WireHeapItem { - key, - value: hh_item.count as f64, - } - }) - .collect() -} - -/// Updates a CSHeap with a weighted key. Automatically updates the heap. -pub fn sketchlib_cs_heap_update(cs_heap: &mut SketchlibCSHeap, key: &str, value: f64) { - let many = value.round() as i64; - if many <= 0 { - return; - } - cs_heap.update_weighted(&DataInput::String(key.to_owned()), many); -} - -/// Queries a CSHeap for a key's frequency estimate (median-of-signed-rows, -/// not CMS's min-over-rows). -pub fn sketchlib_cs_heap_query(cs_heap: &SketchlibCSHeap, key: &str) -> f64 { - cs_heap.estimate(&DataInput::String(key.to_owned())) -} - -/// Item in the top-k heap representing a key-value pair. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CsHeapItem { - pub key: String, - pub value: f64, -} - -/// Count Sketch with Heap for top-k tracking. Combines the balanced/ -/// zero-mean-error frequency estimator with efficient top-k maintenance. -pub struct CountSketchWithHeap { - pub rows: usize, - pub cols: usize, - pub heap_size: usize, - pub(crate) backend: SketchlibCSHeap, -} - -impl std::fmt::Debug for CountSketchWithHeap { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CountSketchWithHeap") - .field("rows", &self.rows) - .field("cols", &self.cols) - .field("heap_size", &self.heap_size) - .finish() - } -} - -impl Clone for CountSketchWithHeap { - fn clone(&self) -> Self { - // Structural clone: `NativeCountSketchWithHeap` derives `Clone` (its `CountSketch` + - // `TopKHeap` fields are `Clone`), so copy the backend directly - // instead of extracting the matrix + heap to wire form and - // rebuilding it. - Self { - rows: self.rows, - cols: self.cols, - heap_size: self.heap_size, - backend: self.backend.clone(), - } - } -} - -impl CountSketchWithHeap { - pub fn new(rows: usize, cols: usize, heap_size: usize) -> Self { - Self { - rows, - cols, - heap_size, - backend: new_sketchlib_cs_heap(rows, cols, heap_size), - } - } - - pub fn rows(&self) -> usize { - self.rows - } - - pub fn cols(&self) -> usize { - self.cols - } - - /// Create from a sketch matrix and heap (e.g. from JSON deserialization). - pub fn from_legacy_matrix( - sketch: Vec>, - topk_heap: Vec, - rows: usize, - cols: usize, - heap_size: usize, - ) -> Self { - let wire_heap: Vec = topk_heap - .into_iter() - .map(|h| WireHeapItem { - key: h.key, - value: h.value, - }) - .collect(); - Self { - rows, - cols, - heap_size, - backend: sketchlib_cs_heap_from_matrix_and_heap( - rows, cols, heap_size, &sketch, &wire_heap, - ), - } - } - - /// Get the top-k heap items. - pub fn topk_heap_items(&self) -> Vec { - heap_to_wire(&self.backend) - .into_iter() - .map(|w| CsHeapItem { - key: w.key, - value: w.value, - }) - .collect() - } - - /// Get the sketch matrix. - pub fn sketch_matrix(&self) -> Vec> { - matrix_from_sketchlib_cs_heap(&self.backend) - } - - pub fn update(&mut self, key: &str, value: f64) { - sketchlib_cs_heap_update(&mut self.backend, key, value); - } - - pub fn estimate(&self, key: &str) -> f64 { - sketchlib_cs_heap_query(&self.backend, key) - } - - /// Merge another CountSketchWithHeap into self in place. Both - /// operands must have identical dimensions; the resulting - /// `heap_size` is the minimum of the two. - pub fn merge( - &mut self, - other: &CountSketchWithHeap, - ) -> Result<(), Box> { - if self.rows != other.rows || self.cols != other.cols { - return Err(format!( - "CountSketchWithHeap dimension mismatch: self={}x{}, other={}x{}", - self.rows, self.cols, other.rows, other.cols - ) - .into()); - } - self.backend.merge(&other.backend); - self.heap_size = self.heap_size.min(other.heap_size); - Ok(()) - } - - pub fn merge_refs( - inputs: &[&CountSketchWithHeap], - ) -> Result> { - let first = inputs - .first() - .ok_or("CountSketchWithHeap::merge_refs called with empty input")?; - let mut merged = (*first).clone(); - for h in inputs.iter().skip(1) { - merged.merge(h)?; - } - Ok(merged) - } - - pub fn aggregate_topk( - rows: usize, - cols: usize, - heap_size: usize, - keys: &[&str], - values: &[f64], - ) -> Option> { - if keys.is_empty() { - return None; - } - let mut sketch = Self::new(rows, cols, heap_size); - for (key, &value) in keys.iter().zip(values.iter()) { - sketch.update(key, value); - } - sketch.to_msgpack().ok() - } -} - -// ----- Wire format ----- - -/// Inner CS payload nested inside [`CountSketchWithHeapWire`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountSketchInnerWire { - pub sketch: Vec>, - #[serde(rename = "row_num")] - pub rows: usize, - #[serde(rename = "col_num")] - pub cols: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountSketchWithHeapWire { - pub sketch: CountSketchInnerWire, - pub topk_heap: Vec, - pub heap_size: usize, -} - -impl MessagePackCodec for CountSketchWithHeap { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wire = CountSketchWithHeapWire { - sketch: CountSketchInnerWire { - sketch: self.sketch_matrix(), - rows: self.rows, - cols: self.cols, - }, - topk_heap: self.topk_heap_items(), - heap_size: self.heap_size, - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wire: CountSketchWithHeapWire = rmp_serde::from_slice(bytes)?; - - let mut sorted_topk_heap = wire.topk_heap; - sorted_topk_heap.sort_by(|a, b| b.value.partial_cmp(&a.value).unwrap()); - - let wire_heap: Vec = sorted_topk_heap - .iter() - .map(|h| WireHeapItem { - key: h.key.clone(), - value: h.value, - }) - .collect(); - let backend = sketchlib_cs_heap_from_matrix_and_heap( - wire.sketch.rows, - wire.sketch.cols, - wire.heap_size, - &wire.sketch.sketch, - &wire_heap, - ); - - Ok(Self { - rows: wire.sketch.rows, - cols: wire.sketch.cols, - heap_size: wire.heap_size, - backend, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_creation() { - let cs = CountSketchWithHeap::new(4, 1000, 20); - assert_eq!(cs.rows, 4); - assert_eq!(cs.cols, 1000); - assert_eq!(cs.heap_size, 20); - assert_eq!(cs.sketch_matrix().len(), 4); - assert_eq!(cs.sketch_matrix()[0].len(), 1000); - assert_eq!(cs.topk_heap_items().len(), 0); - } - - #[test] - fn test_query_empty() { - let cs = CountSketchWithHeap::new(2, 10, 5); - assert_eq!(cs.estimate("anything"), 0.0); - } - - #[test] - fn test_merge() { - let mut sketch1 = vec![vec![0.0; 10]; 2]; - sketch1[0][0] = 10.0; - sketch1[1][1] = 20.0; - let mut cs1 = CountSketchWithHeap::from_legacy_matrix( - sketch1, - vec![ - CsHeapItem { - key: "key1".to_string(), - value: 100.0, - }, - CsHeapItem { - key: "key2".to_string(), - value: 50.0, - }, - ], - 2, - 10, - 5, - ); - let mut sketch2 = vec![vec![0.0; 10]; 2]; - sketch2[0][0] = 5.0; - sketch2[1][1] = 15.0; - let cs2 = CountSketchWithHeap::from_legacy_matrix( - sketch2, - vec![ - CsHeapItem { - key: "key3".to_string(), - value: 75.0, - }, - CsHeapItem { - key: "key1".to_string(), - value: 80.0, - }, - ], - 2, - 10, - 3, - ); - - cs1.merge(&cs2).unwrap(); - - assert_eq!(cs1.sketch_matrix()[0][0], 15.0); - assert_eq!(cs1.sketch_matrix()[1][1], 35.0); - assert_eq!(cs1.heap_size, 3); - assert!(cs1.topk_heap_items().len() <= 3); - } - - #[test] - fn test_merge_dimension_mismatch() { - let mut cs1 = CountSketchWithHeap::new(2, 10, 5); - let cs2 = CountSketchWithHeap::new(3, 10, 5); - assert!(cs1.merge(&cs2).is_err()); - } - - #[test] - fn test_msgpack_round_trip() { - let mut cs = CountSketchWithHeap::new(4, 128, 3); - cs.update("hot", 100.0); - cs.update("cold", 1.0); - - let bytes = cs.to_msgpack().unwrap(); - let deserialized = CountSketchWithHeap::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.rows, 4); - assert_eq!(deserialized.cols, 128); - assert_eq!(deserialized.heap_size, 3); - let items = deserialized.topk_heap_items(); - assert!(!items.is_empty()); - let hot = items - .iter() - .find(|item| item.key == "hot") - .expect("'hot' should be in the heap"); - assert!(hot.value >= 100.0); - assert!(deserialized.estimate("hot") >= 100.0); - assert!(deserialized.estimate("cold") >= 1.0); - } - - #[test] - fn test_aggregate_topk() { - let keys = ["a", "b", "a", "c"]; - let values = [1.0, 2.0, 3.0, 0.5]; - let bytes = CountSketchWithHeap::aggregate_topk(4, 100, 2, &keys, &values).unwrap(); - let cs = CountSketchWithHeap::from_msgpack(&bytes).unwrap(); - assert_eq!(cs.heap_size, 2); - assert!(cs.topk_heap_items().len() <= 2); - } - - #[test] - fn test_aggregate_topk_empty() { - assert!(CountSketchWithHeap::aggregate_topk(4, 100, 10, &[], &[]).is_none()); - } - - /// The whole point of splitting this from `CountMinSketchWithHeap`: - /// CS's median estimator must behave differently from CMS's - /// min-over-rows estimator on the same kind of skewed insert - /// pattern, not just share a struct shape. - #[test] - fn test_median_estimator_differs_from_cms_style_min() { - let mut cs = CountSketchWithHeap::new(5, 64, 10); - for _ in 0..50 { - cs.update("heavy", 1.0); - } - for i in 0..20 { - cs.update(&format!("light-{i}"), 1.0); - } - let est = cs.estimate("heavy"); - assert!( - (40.0..=60.0).contains(&est), - "median estimate {est} should track the true count (50) reasonably \ - closely, not collide-and-inflate the way a single min-over-rows \ - outlier could" - ); - } -} diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs deleted file mode 100644 index e8961d6f..00000000 --- a/src/message_pack_format/portable/ddsketch.rs +++ /dev/null @@ -1,1364 +0,0 @@ -//! Wire-format-aligned DDSketch types. The wire DTO + runtime ops -//! live together here. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; - -/// Bucket-store growth chunk for the wire-format-aligned [`DdSketch`] -/// variant. Matches the Go reference implementation's `DDSketch.GrowChunk` -/// so the `store_counts` / `store_offset` layout written by -/// [`DdSketch::update`] is byte-identical to the Go `SerializePortable` -/// output for the same input stream. -pub const DDSKETCH_GROW_CHUNK: usize = 128; - -/// Safety limit for a single [`DdSketchDelta`] application: deltas arrive -/// over the wire untrusted, so a span beyond this many buckets is rejected -/// with an error instead of padding the dense store toward a multi-gigabyte -/// allocation. Generous relative to any legitimate producer: at α = 0.01 the -/// entire indexable range spans ~35k buckets. -pub const MAX_APPLY_DELTA_SPAN_BUCKETS: i64 = 1 << 22; - -// ===================================================================== -// Wire-format-aligned variant. -// -// `DdSketch` and `DdSketchDelta` below are the public-field, -// proto-decode-friendly types consumed by the query-engine -// accumulators. The high-throughput in-process variant is -// `crate::sketches::ddsketch::DDSketch`, which keeps its own design. -// ===================================================================== - -/// Sparse delta between two consecutive DDSketch snapshots — the -/// input shape for [`DdSketch::apply_delta`]. Mirrors the -/// `DDSketchDelta` proto (and its Rust bindings). Kept as a plain -/// struct so this crate doesn't need a tonic/prost dependency; proto -/// decode lives in the accumulator. -#[derive(Debug, Clone, Default)] -pub struct DdSketchDelta { - /// `(absolute_bucket_index, Δcount)` pairs, additive. - pub buckets: Vec<(i32, u64)>, - pub negative_buckets: Vec<(i32, u64)>, - pub zero_count: u64, - /// Δ total count. May be negative (signed on the wire). - pub d_count: i64, - /// Δ sum. - pub d_sum: f64, - /// Whether `new_min` carries a meaningful value. Min can only - /// decrease; a delta that didn't lower min sends `false`. - pub min_changed: bool, - pub new_min: f64, - /// Whether `new_max` carries a meaningful value. Max can only - /// increase. - pub max_changed: bool, - pub new_max: f64, -} - -/// Signed DDSketch state: separate magnitude stores and a zero count. -/// Positive-only MessagePack retains `[alpha, counts, offset]`; signed state -/// appends `[negative_counts, negative_offset, zero_count]` to that array. -#[derive(Debug, Clone, Deserialize)] -pub struct DdSketch { - /// Relative accuracy parameter; must satisfy `0 < alpha < 1`. - pub alpha: f64, - /// Bucket counts in absolute-index order. The absolute index of - /// `store_counts[i]` is `i + store_offset`. - pub store_counts: Vec, - /// Absolute bucket index corresponding to `store_counts[0]`. May - /// be negative. - pub store_offset: i32, - /// Negative-value magnitudes, indexed with the same logarithmic mapping. - #[serde(default)] - pub negative_store_counts: Vec, - #[serde(default)] - pub negative_store_offset: i32, - #[serde(default)] - pub zero_count: u64, -} - -impl Serialize for DdSketch { - fn serialize(&self, serializer: S) -> Result { - use serde::ser::SerializeStruct; - let signed = !self.negative_store_counts.is_empty() || self.zero_count != 0; - let mut state = serializer.serialize_struct("DdSketch", if signed { 6 } else { 3 })?; - state.serialize_field("alpha", &self.alpha)?; - state.serialize_field("store_counts", &self.store_counts)?; - state.serialize_field("store_offset", &self.store_offset)?; - if signed { - state.serialize_field("negative_store_counts", &self.negative_store_counts)?; - state.serialize_field("negative_store_offset", &self.negative_store_offset)?; - state.serialize_field("zero_count", &self.zero_count)?; - } - state.end() - } -} - -impl DdSketch { - /// Construct an empty sketch. - pub fn new(alpha: f64) -> Self { - assert!( - alpha > 0.0 && alpha < 1.0, - "alpha must be in (0,1); alpha=0 makes ln(gamma)=0 and every guard degenerate" - ); - Self { - alpha, - store_counts: Vec::new(), - store_offset: 0, - negative_store_counts: Vec::new(), - negative_store_offset: 0, - zero_count: 0, - } - } - - /// Construct from the decoded wire fields. - pub fn from_raw(alpha: f64, store_counts: Vec, store_offset: i32) -> Self { - Self { - alpha, - store_counts, - store_offset, - negative_store_counts: Vec::new(), - negative_store_offset: 0, - zero_count: 0, - } - } - - /// Convert every store to protobuf, including negative and zero observations. - pub fn to_proto(&self) -> crate::proto::sketchlib::DdSketchState { - crate::proto::sketchlib::DdSketchState { - alpha: self.wire_alpha(), - store_counts: self.store_counts.clone(), - store_offset: self.store_offset, - negative_store_counts: self.negative_store_counts.clone(), - negative_store_offset: self.negative_store_offset, - zero_count: self.zero_count, - } - } - - /// Restore all stores from a decoded protobuf state. - pub fn from_proto(state: crate::proto::sketchlib::DdSketchState) -> Self { - Self { - alpha: state.alpha, - store_counts: state.store_counts, - store_offset: state.store_offset, - negative_store_counts: state.negative_store_counts, - negative_store_offset: state.negative_store_offset, - zero_count: state.zero_count, - } - } - - /// Total number of values added, recovered by summing the bucket - /// counts. The wire format carries no `count` scalar, so this is the - /// authoritative count for quantile-rank computation. - pub fn total_count(&self) -> u64 { - self.store_counts - .iter() - .chain(&self.negative_store_counts) - .copied() - .fold(self.zero_count, u64::saturating_add) - } - - /// Merge one other sketch into self by aligning bucket arrays on - /// absolute indices. Both operands must share the same `alpha`. - /// - /// Like [`Self::apply_delta`], the operand is treated as untrusted wire - /// state: if the union span would exceed - /// [`MAX_APPLY_DELTA_SPAN_BUCKETS`] *and* grow beyond both operands' - /// existing stores, `Err` is returned before any allocation. Stores that - /// already legitimately span more than the cap (exotic small-α - /// configurations) merge normally as long as the union stays within - /// what either side already holds. - pub fn merge( - &mut self, - other: &DdSketch, - ) -> Result<(), Box> { - // Validate both store unions before publishing any of the merged state. - let mut merged = self.clone(); - merged.merge_positive(other)?; - let mut negative = Self::from_raw( - self.alpha, - self.negative_store_counts.clone(), - self.negative_store_offset, - ); - negative.merge_positive(&Self::from_raw( - other.alpha, - other.negative_store_counts.clone(), - other.negative_store_offset, - ))?; - merged.negative_store_counts = negative.store_counts; - merged.negative_store_offset = negative.store_offset; - merged.zero_count = self - .zero_count - .checked_add(other.zero_count) - .ok_or("DDSketch zero count overflow")?; - *self = merged; - Ok(()) - } - - fn merge_positive( - &mut self, - other: &DdSketch, - ) -> Result<(), Box> { - if (self.alpha - other.alpha).abs() > f64::EPSILON { - return Err(format!( - "DdSketch alpha mismatch: self={}, other={}", - self.alpha, other.alpha - ) - .into()); - } - - if other.store_counts.is_empty() { - return Ok(()); - } - if self.store_counts.is_empty() { - self.store_counts = other.store_counts.clone(); - self.store_offset = other.store_offset; - } else { - let self_start = self.store_offset as i64; - let self_end = self_start + self.store_counts.len() as i64; - let other_start = other.store_offset as i64; - let other_end = other_start + other.store_counts.len() as i64; - let new_start = self_start.min(other_start); - let new_end = self_end.max(other_end); - let new_len = new_end - new_start; - // Hostile-span guard: a decoded snapshot with store_offset near - // i32::MIN and a short count vector must not allocate a union - // spanning ~2^31 buckets (~17 GiB) in one call. Growth beyond - // both operands' existing spans is capped; stores that already - // legitimately exceed the cap pass through untouched. - let max_existing = - (self.store_counts.len() as i64).max(other.store_counts.len() as i64); - if new_len > MAX_APPLY_DELTA_SPAN_BUCKETS && new_len > max_existing { - return Err(format!( - "DdSketch merge spans {} buckets, exceeding the {}-bucket safety limit", - new_len, MAX_APPLY_DELTA_SPAN_BUCKETS - ) - .into()); - } - let new_len = new_len as usize; - let mut merged = vec![0u64; new_len]; - for (start, counts) in [ - (self_start, &self.store_counts), - (other_start, &other.store_counts), - ] { - for (i, &c) in counts.iter().enumerate() { - let idx = (start + i as i64 - new_start) as usize; - merged[idx] = merged[idx].saturating_add(c); - } - } - self.store_counts = merged; - self.store_offset = new_start as i32; - } - Ok(()) - } - - /// Apply a sparse delta to this sketch in place. Matches the - /// `ApplyDelta` logic in the Go reference implementation: - /// bucket counts add, total count + sum add, min can only decrease - /// and max can only increase. Used by the backend ingest path to - /// reconstitute a full sketch from a base snapshot + subsequent - /// delta-transmission frames. - /// - /// Deltas arrive over the wire and are treated as untrusted: before - /// mutating anything, the union span the delta would require is - /// checked against [`MAX_APPLY_DELTA_SPAN_BUCKETS`]. A hostile or - /// corrupt delta carrying an index near `i32::MAX` would otherwise - /// pad the dense store by ~2·10⁹ buckets (~17 GiB) in one call — - /// independent of α, unlike the `update()` path whose worst case is - /// bounded by the indexable-range guard. - pub fn apply_delta( - &mut self, - delta: &DdSketchDelta, - ) -> Result<(), Box> { - let mut merged = self.clone(); - merged.apply_positive_delta(delta)?; - let mut negative = Self::from_raw( - self.alpha, - self.negative_store_counts.clone(), - self.negative_store_offset, - ); - negative.apply_positive_delta(&DdSketchDelta { - buckets: delta.negative_buckets.clone(), - ..Default::default() - })?; - merged.negative_store_counts = negative.store_counts; - merged.negative_store_offset = negative.store_offset; - merged.zero_count = self - .zero_count - .checked_add(delta.zero_count) - .ok_or("DDSketch zero count overflow")?; - *self = merged; - Ok(()) - } - - fn apply_positive_delta( - &mut self, - delta: &DdSketchDelta, - ) -> Result<(), Box> { - // Pre-validate span before any mutation so a rejected delta - // leaves state untouched. - if !delta.buckets.is_empty() { - let mut min_d = i64::MAX; - let mut max_d = i64::MIN; - for (abs_idx, _) in &delta.buckets { - min_d = min_d.min(*abs_idx as i64); - max_d = max_d.max(*abs_idx as i64); - } - let cur_start = if self.store_counts.is_empty() { - min_d - } else { - self.store_offset as i64 - }; - let cur_end = cur_start + self.store_counts.len() as i64; - let new_start = cur_start.min(min_d); - let new_end = cur_end.max(max_d + 1); - if new_end - new_start > MAX_APPLY_DELTA_SPAN_BUCKETS { - return Err(format!( - "DdSketch delta spans {} buckets, exceeding the {}-bucket safety limit", - new_end - new_start, - MAX_APPLY_DELTA_SPAN_BUCKETS - ) - .into()); - } - } - - for (abs_idx, d_count) in &delta.buckets { - if self.store_counts.is_empty() { - self.store_counts = vec![0u64; 1]; - self.store_offset = *abs_idx; - } - let cur_start = self.store_offset as i64; - let cur_end = cur_start + self.store_counts.len() as i64; - let k = *abs_idx as i64; - if k < cur_start { - // Prepend zeros. - let pad = (cur_start - k) as usize; - let mut buf = vec![0u64; pad]; - buf.append(&mut self.store_counts); - self.store_counts = buf; - self.store_offset = *abs_idx; - } else if k >= cur_end { - let pad = (k - cur_end + 1) as usize; - self.store_counts.extend(std::iter::repeat_n(0u64, pad)); - } - let arr_idx = (k - self.store_offset as i64) as usize; - self.store_counts[arr_idx] = self.store_counts[arr_idx].saturating_add(*d_count); - } - Ok(()) - } - - /// Compute a sparse, proto-marshalled `DDSketchDelta` of `self` - /// against a `snapshot`. A bucket is included when its `Δcount` - /// (self − snapshot, clamped at 0) is `>= threshold`. - /// - /// This is the Rust twin of the Go reference implementation's - /// `ComputeDelta`: it iterates every non-empty bucket in `self`, - /// subtracts the snapshot's count for the same absolute index, and - /// emits the surviving bucket deltas. The returned bytes are a - /// `prost`-encoded [`crate::proto::sketchlib::DdSketchDelta`], - /// byte-identical to the Go `proto.Marshal(DDSketchDelta)` output - /// for the same inputs (cross-language byte parity). - /// - /// Delta-against-empty: when `snapshot` is the empty sketch, every - /// surviving bucket delta equals the window's own bucket count, so - /// the result is this window's full state encoded as a delta (no - /// cross-window subtraction). The DataPoint-level metric scalars are - /// not carried — the count delta is recoverable by summing bucket - /// deltas. - pub fn compute_delta(&self, snapshot: &DdSketch, threshold: u64) -> Vec { - use crate::proto::sketchlib::{DdSketchBucketDelta, DdSketchDelta as ProtoDelta}; - use prost::Message; - - let mut delta = ProtoDelta::default(); - if !self.store_counts.is_empty() { - for (i, &c) in self.store_counts.iter().enumerate() { - if c == 0 { - continue; - } - let k = self.store_offset + i as i32; - let snap_count: u64 = if !snapshot.store_counts.is_empty() { - let idx = k as i64 - snapshot.store_offset as i64; - if idx >= 0 && (idx as usize) < snapshot.store_counts.len() { - snapshot.store_counts[idx as usize] - } else { - 0 - } - } else { - 0 - }; - let dc = c.saturating_sub(snap_count); - if dc >= threshold { - delta.buckets.push(DdSketchBucketDelta { - index: k, - d_count: dc, - }); - } - } - } - let negative = Self::from_raw( - self.alpha, - self.negative_store_counts.clone(), - self.negative_store_offset, - ); - let previous = Self::from_raw( - snapshot.alpha, - snapshot.negative_store_counts.clone(), - snapshot.negative_store_offset, - ); - delta.negative_buckets = sparse_difference(&negative, &previous, threshold) - .into_iter() - .map(|(index, d_count)| DdSketchBucketDelta { index, d_count }) - .collect(); - delta.zero_count = self.zero_count.saturating_sub(snapshot.zero_count); - if delta.zero_count < threshold { - delta.zero_count = 0; - } - delta.encode_to_vec() - } - - /// Apply a `prost`-encoded [`crate::proto::sketchlib::DdSketchDelta`] - /// to this sketch in place (additive bucket merge). The Rust twin of - /// the Go reference implementation's `ApplyDelta`. - /// - /// Returns `Err` if `bytes` is not a valid `DDSketchDelta` proto. - pub fn apply_delta_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - use crate::proto::sketchlib::DdSketchDelta as ProtoDelta; - use prost::Message; - - let proto = ProtoDelta::decode(bytes)?; - let delta = DdSketchDelta { - negative_buckets: proto - .negative_buckets - .into_iter() - .map(|b| (b.index, b.d_count)) - .collect(), - zero_count: proto.zero_count, - buckets: proto - .buckets - .into_iter() - .map(|b| (b.index, b.d_count)) - .collect(), - ..DdSketchDelta::default() - }; - self.apply_delta(&delta) - } - - /// MessagePack twin of [`Self::compute_delta`]. Computes the same - /// sparse bucket delta of `self` against `snapshot` (a bucket is - /// carried when its `Δcount = self − snapshot` clamped at 0 is - /// `>= threshold`), but serializes it as the parallel-array - /// MessagePack layout via [`DdSketchDelta`]'s [`MessagePackCodec`] - /// instead of proto. Same delta-against-empty semantics: against the - /// empty sketch every surviving delta equals this window's own bucket - /// count. - pub fn compute_delta_msgpack(&self, snapshot: &DdSketch, threshold: u64) -> Vec { - let mut delta = DdSketchDelta::default(); - if !self.store_counts.is_empty() { - for (i, &c) in self.store_counts.iter().enumerate() { - if c == 0 { - continue; - } - let k = self.store_offset + i as i32; - let snap_count: u64 = if !snapshot.store_counts.is_empty() { - let idx = k as i64 - snapshot.store_offset as i64; - if idx >= 0 && (idx as usize) < snapshot.store_counts.len() { - snapshot.store_counts[idx as usize] - } else { - 0 - } - } else { - 0 - }; - let dc = c.saturating_sub(snap_count); - if dc >= threshold { - delta.buckets.push((k, dc)); - } - } - } - delta.negative_buckets = sparse_difference( - &Self::from_raw( - self.alpha, - self.negative_store_counts.clone(), - self.negative_store_offset, - ), - &Self::from_raw( - snapshot.alpha, - snapshot.negative_store_counts.clone(), - snapshot.negative_store_offset, - ), - threshold, - ); - delta.zero_count = self.zero_count.saturating_sub(snapshot.zero_count); - if delta.zero_count < threshold { - delta.zero_count = 0; - } - delta - .to_msgpack() - .expect("DdSketchDelta msgpack encode is infallible for owned bucket arrays") - } - - /// MessagePack twin of [`Self::apply_delta_bytes`]. Decodes the - /// parallel-array MessagePack [`DdSketchDelta`] and applies it in - /// place (additive bucket merge). - /// - /// Returns `Err` if `bytes` is not a valid MessagePack `DdSketchDelta`. - pub fn apply_delta_msgpack_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - let delta = DdSketchDelta::from_msgpack(bytes)?; - self.apply_delta(&delta) - } - - /// Merge a slice of references into a single new sketch. Returns - /// `Err` on alpha mismatch or an empty input. - pub fn merge_refs( - inputs: &[&DdSketch], - ) -> Result> { - let first = inputs - .first() - .ok_or("DdSketch::merge_refs called with empty input")?; - let mut merged = DdSketch::new(first.alpha); - for d in inputs { - merged.merge(d)?; - } - Ok(merged) - } - - /// Insert a single positive value. Updates count, sum, min/max - /// and increments the bucket for `floor(ln(v) / ln(gamma))` - /// where `gamma = (1+α)/(1-α)`. Provided primarily so tests can - /// build a ground-truth sketch to compare delta-apply output - /// against. - /// - /// Bucket-store growth mirrors the Go reference implementation's - /// `Buckets.ensure`: the first allocation is a half-chunk-centered - /// `GROW_CHUNK` of zeros, and subsequent expansions extend by - /// `max(needed, GROW_CHUNK)` so the on-the-wire `store_counts` / - /// `store_offset` layout is byte-identical to the Go - /// `SerializePortable` output. Without this chunked layout the - /// `DDSketchState` proto bytes would diverge from the Go producer's - /// payload (cross-language byte parity). - pub fn update(&mut self, value: f64) { - let _ = self.try_update(value); - } - - /// Reject NaN/infinity and values above the mapping range explicitly. - /// Like DataDog, magnitudes below the mapping range enter the zero bucket; - /// the relative-error guarantee does not cover those nonzero tiny values. - pub fn try_update(&mut self, value: f64) -> Result<(), &'static str> { - if !value.is_finite() { - return Err("DDSketch requires finite values"); - } - let (min_v, max_v) = crate::sketches::ddsketch::ddsketch_indexable_bounds(self.alpha); - if value.abs() > max_v { - return Err("DDSketch value exceeds indexable range"); - } - if value.abs() < min_v { - self.zero_count = self - .zero_count - .checked_add(1) - .ok_or("DDSketch zero count overflow")?; - return Ok(()); - } - let negative = value < 0.0; - let value = value.abs(); - let gamma = (1.0 + self.alpha) / (1.0 - self.alpha); - let ln_gamma = gamma.ln(); - // Reuse the positive-store growth routine for the magnitude store. - if negative { - std::mem::swap(&mut self.store_counts, &mut self.negative_store_counts); - std::mem::swap(&mut self.store_offset, &mut self.negative_store_offset); - } - let idx = (value.ln() / ln_gamma).floor() as i32; - self.ensure_bucket(idx); - let arr_idx = (idx as i64 - self.store_offset as i64) as usize; - self.store_counts[arr_idx] = self.store_counts[arr_idx].saturating_add(1); - if negative { - std::mem::swap(&mut self.store_counts, &mut self.negative_store_counts); - std::mem::swap(&mut self.store_offset, &mut self.negative_store_offset); - } - Ok(()) - } - - /// Ensure bucket `k` is addressable in `store_counts`, growing in - /// chunks of [`DDSKETCH_GROW_CHUNK`] to match the Go reference - /// implementation's `Buckets.ensure`. Empty stores are seeded with - /// a half-chunk of zeros centered on `k` (`store_offset = k - - /// GROW_CHUNK/2`); out-of-range expansions extend by `max(needed, - /// GROW_CHUNK)` on either side. The on-the-wire byte layout is - /// therefore identical between the Go and Rust producers fed the - /// same input stream — required for cross-language byte parity. - fn ensure_bucket(&mut self, k: i32) { - if self.store_counts.is_empty() { - self.store_counts = vec![0u64; DDSKETCH_GROW_CHUNK]; - self.store_offset = k - (DDSKETCH_GROW_CHUNK as i32 / 2); - return; - } - let cur_start = self.store_offset as i64; - let cur_end = cur_start + self.store_counts.len() as i64; - let kk = k as i64; - if kk < cur_start { - let needed = (cur_start - kk) as usize; - let grow = needed.max(DDSKETCH_GROW_CHUNK); - let mut buf = vec![0u64; grow]; - buf.extend_from_slice(&self.store_counts); - self.store_counts = buf; - self.store_offset -= grow as i32; - } else if kk >= cur_end { - let needed = (kk - cur_end + 1) as usize; - let grow = needed.max(DDSKETCH_GROW_CHUNK); - self.store_counts.extend(std::iter::repeat_n(0u64, grow)); - } - } - - /// Estimate the quantile at rank `q` ∈ [0, 1]. Walks the bucket - /// array in ascending absolute-index order, accumulating counts - /// until the target rank; returns the bucket's representative - /// value `gamma^k * (1 + alpha)` where `k` is the bucket's absolute - /// index. Returns `None` if the sketch is empty. - /// - /// For indexable nonzero order statistics, absolute error is at most - /// `alpha * abs(true_value)`. Tiny values mapped to zero are excluded. - pub fn quantile(&self, q: f64) -> Option { - let count = self.total_count(); - if count == 0 { - return None; - } - let target = (q * (count.saturating_sub(1)) as f64).floor() as u64; - self.value_at_rank(target) - } - - /// Estimate the linearly interpolated quantile at `q * (n - 1)`. - /// - /// Interpolation across opposite signs can cancel: its absolute error is - /// bounded by alpha times the weighted absolute endpoints, but its relative - /// error need not be bounded by alpha. Zero-mapped tiny values are excluded. - /// This interpolates the two adjacent order-statistic bucket estimates, - /// as required by PromQL quantile and continuous percentiles. The existing - /// `quantile` method retains its lower-order-statistic convention. - /// Returns `None` for invalid q/alpha, empty or overflowing counts, counts - /// above exact Float64 integer precision, or nonfinite bucket estimates. - pub fn quantile_interpolated(&self, q: f64) -> Option { - if !q.is_finite() - || !(0.0..=1.0).contains(&q) - || !self.alpha.is_finite() - || !(0.0..1.0).contains(&self.alpha) - || self.alpha == 0.0 - { - return None; - } - let count = self - .store_counts - .iter() - .chain(&self.negative_store_counts) - .try_fold(self.zero_count, |sum, count| sum.checked_add(*count))?; - if count == 0 || count > (1u64 << 53) { - return None; - } - let rank = q * (count - 1) as f64; - let lower_rank = rank.floor() as u64; - let upper_rank = rank.ceil() as u64; - let lower = self.value_at_rank(lower_rank)?; - let upper = self.value_at_rank(upper_rank)?; - if !lower.is_finite() || !upper.is_finite() { - return None; - } - let fraction = rank - lower_rank as f64; - let result = lower * (1.0 - fraction) + upper * fraction; - result.is_finite().then_some(result) - } - - fn value_at_rank(&self, target: u64) -> Option { - let mut target = target; - let gamma = (1.0 + self.alpha) / (1.0 - self.alpha); - for (i, &count) in self.negative_store_counts.iter().enumerate().rev() { - if target < count { - return Some( - -gamma.powf((self.negative_store_offset as i64 + i as i64) as f64) - * (1.0 + self.alpha), - ); - } - target = target.checked_sub(count)?; - } - if target < self.zero_count { - return Some(0.0); - } - target = target.checked_sub(self.zero_count)?; - let mut cumulative: u64 = 0; - let gamma = (1.0 + self.alpha) / (1.0 - self.alpha); - let mut last_nonempty: Option = None; - for (i, &c) in self.store_counts.iter().enumerate() { - if c > 0 { - last_nonempty = Some(i); - } - cumulative = cumulative.saturating_add(c); - if cumulative > target { - let k = self.store_offset as i64 + i as i64; - return Some(gamma.powf(k as f64) * (1.0 + self.alpha)); - } - } - // Numerical edge case: if we fall off the end (e.g. q == 1.0 and - // rounding lands past the final increment), estimate from the - // highest non-empty bucket. The wire format carries no `max` - // scalar; the representative is within DDSketch's α - // relative-accuracy bound of the true max. - last_nonempty.map(|i| { - let k = (self.store_offset as i64 + i as i64) as f64; - gamma.powf(k) * (1.0 + self.alpha) - }) - } - - /// Return the alpha value as it appears on the wire — round-tripped - /// through gamma exactly the way the Go reference implementation's - /// `DDSketch.SerializePortable` does it (`alpha = (gamma - 1) / - /// (gamma + 1)`, where `gamma = (1+α)/(1-α)`). The roundtrip - /// introduces a small floating-point drift; without applying it the - /// Rust producer's `DDSketchState.alpha` field bytes would diverge - /// from the Go producer's, and cross-language byte parity would fail - /// on the very first proto field. - #[inline] - pub fn wire_alpha(&self) -> f64 { - let gamma = (1.0 + self.alpha) / (1.0 - self.alpha); - (gamma - 1.0) / (gamma + 1.0) - } -} - -fn sparse_difference(current: &DdSketch, previous: &DdSketch, threshold: u64) -> Vec<(i32, u64)> { - current - .store_counts - .iter() - .enumerate() - .filter_map(|(i, count)| { - let key = i64::from(current.store_offset) + i as i64; - let old = key - i64::from(previous.store_offset); - let old_count = usize::try_from(old) - .ok() - .and_then(|i| previous.store_counts.get(i)) - .copied() - .unwrap_or(0); - let delta = count.saturating_sub(old_count); - (delta > 0 && delta >= threshold).then_some((key as i32, delta)) - }) - .collect() -} - -#[derive(Deserialize)] -struct DeltaWire( - Vec, - Vec, - #[serde(default)] Vec, - #[serde(default)] Vec, - #[serde(default)] u64, -); - -impl MessagePackCodec for DdSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) - } -} - -impl MessagePackCodec for DdSketchDelta { - /// Encodes the sparse bucket delta as the 2-element MessagePack array - /// `[ idx:[]i32, d_count:[]u64 ]` — parallel arrays of absolute bucket - /// index → Δcount — byte-identical to Rust - /// `rmp_serde::to_vec(&(Vec, Vec))` (compact mode), the layout - /// the `sketchlib-go` `asapmsgpack` DDSketch delta encoder mirrors. Only - /// the bucket cells cross the wire; the DataPoint-level metric scalars - /// (`d_count`/`d_sum`/`new_min`/`new_max`) are reconstructable and are - /// not carried, matching the proto delta. - fn to_msgpack(&self) -> Result, MsgPackError> { - let idx: Vec = self.buckets.iter().map(|(i, _)| *i).collect(); - let d_count: Vec = self.buckets.iter().map(|(_, c)| *c).collect(); - if self.negative_buckets.is_empty() && self.zero_count == 0 { - Ok(rmp_serde::to_vec(&(idx, d_count))?) - } else { - let ni: Vec<_> = self.negative_buckets.iter().map(|b| b.0).collect(); - let nc: Vec<_> = self.negative_buckets.iter().map(|b| b.1).collect(); - Ok(rmp_serde::to_vec(&(idx, d_count, ni, nc, self.zero_count))?) - } - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let DeltaWire(idx, d_count, ni, nc, zero_count) = rmp_serde::from_slice(bytes)?; - if idx.len() != d_count.len() || ni.len() != nc.len() { - return Err(rmp_serde::decode::Error::Syntax( - "DDSketch delta array lengths differ".into(), - ) - .into()); - } - Ok(DdSketchDelta { - buckets: idx.into_iter().zip(d_count).collect(), - negative_buckets: ni.into_iter().zip(nc).collect(), - zero_count, - ..DdSketchDelta::default() - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::message_pack_format::MessagePackCodec; - - #[test] - fn msgpack_delta_against_empty_round_trips() { - let mut w = DdSketch::new(0.01); - for i in 1..=200 { - w.update(i as f64); - } - let empty = DdSketch::new(0.01); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut recon = DdSketch::new(0.01); - recon.apply_delta_msgpack_bytes(&bytes).unwrap(); - assert_eq!(recon.total_count(), w.total_count()); - for q in [0.5, 0.9, 0.99] { - let got = recon.quantile(q).unwrap(); - let want = w.quantile(q).unwrap(); - assert!( - (got / want - 1.0).abs() <= 0.01, - "q={q}: got={got} want={want}" - ); - } - } - - #[test] - fn msgpack_delta_wire_layout_is_parallel_arrays() { - // Lock the cross-language wire shape: a 2-element array of parallel - // (idx, d_count) arrays, byte-identical to the tuple encoding the Go - // asapmsgpack DDSketch delta mirrors. - let mut w = DdSketch::new(0.01); - w.update(1.0); - w.update(1.0); - w.update(1000.0); - let empty = DdSketch::new(0.01); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut idx: Vec = Vec::new(); - let mut d_count: Vec = Vec::new(); - for (i, &c) in w.store_counts.iter().enumerate() { - if c != 0 { - idx.push(w.store_offset + i as i32); - d_count.push(c); - } - } - let expected = rmp_serde::to_vec(&(idx, d_count)).unwrap(); - assert_eq!(bytes, expected); - let d = DdSketchDelta::from_msgpack(&bytes).unwrap(); - assert_eq!(d.buckets.len(), 2, "two distinct buckets expected"); - } - - #[test] - fn test_new_empty() { - let d = DdSketch::new(0.01); - assert_eq!(d.total_count(), 0); - assert!(d.store_counts.is_empty()); - } - - #[test] - fn test_merge_aligned_same_offset() { - let mut a = DdSketch::from_raw(0.01, vec![1, 2, 3], -1); - let b = DdSketch::from_raw(0.01, vec![10, 20, 30], -1); - a.merge(&b).unwrap(); - assert_eq!(a.store_counts, vec![11, 22, 33]); - assert_eq!(a.store_offset, -1); - assert_eq!(a.total_count(), 66); - } - - #[test] - fn test_merge_overlapping_offsets() { - // a covers indices [-1, 0, 1]; b covers indices [0, 1, 2] - let mut a = DdSketch::from_raw(0.01, vec![1, 1, 1], -1); - let b = DdSketch::from_raw(0.01, vec![10, 10, 10], 0); - a.merge(&b).unwrap(); - // Merged window is [-1, 0, 1, 2] → [1, 11, 11, 10] - assert_eq!(a.store_counts, vec![1, 11, 11, 10]); - assert_eq!(a.store_offset, -1); - assert_eq!(a.total_count(), 33); - } - - #[test] - fn test_merge_disjoint_offsets() { - let mut a = DdSketch::from_raw(0.01, vec![1, 2], 0); - let b = DdSketch::from_raw(0.01, vec![3, 4], 5); - a.merge(&b).unwrap(); - // Window [0..7) → [1,2,0,0,0,3,4] - assert_eq!(a.store_counts, vec![1, 2, 0, 0, 0, 3, 4]); - assert_eq!(a.store_offset, 0); - } - - #[test] - fn test_apply_delta_additive_inside_store() { - let mut base = DdSketch::from_raw(0.01, vec![1, 2, 3], -1); - let delta = DdSketchDelta { - buckets: vec![(-1, 4), (0, 8), (1, 12)], - d_count: 24, - d_sum: 120.0, - min_changed: false, - new_min: 0.0, - max_changed: true, - new_max: 9.0, - ..Default::default() - }; - base.apply_delta(&delta).unwrap(); - assert_eq!(base.store_counts, vec![5, 10, 15]); - assert_eq!(base.total_count(), 30); - } - - #[test] - fn test_apply_delta_expands_store_on_new_bucket() { - // Base covers [0..2]; delta adds a bucket at absolute index 4. - let mut base = DdSketch::from_raw(0.01, vec![1, 2], 0); - let delta = DdSketchDelta { - buckets: vec![(4, 7)], - d_count: 7, - d_sum: 35.0, - min_changed: false, - new_min: 0.0, - max_changed: true, - new_max: 6.0, - ..Default::default() - }; - base.apply_delta(&delta).unwrap(); - assert_eq!(base.store_counts, vec![1, 2, 0, 0, 7]); - assert_eq!(base.store_offset, 0); - assert_eq!(base.total_count(), 10); - } - - #[test] - fn test_apply_delta_matches_full_merge() { - // Snapshot the sketch, add more samples via a merge, and confirm - // the delta+apply path lands at the same state. - let base = DdSketch::from_raw(0.01, vec![1, 2, 3], 0); - let addition = DdSketch::from_raw(0.01, vec![10, 0, 20], 0); - let mut via_merge = base.clone(); - via_merge.merge(&addition).unwrap(); - - let delta = DdSketchDelta { - buckets: vec![(0, 10), (2, 20)], - d_count: 30, - d_sum: 70.0, - min_changed: true, - new_min: 0.5, - max_changed: true, - new_max: 5.0, - ..Default::default() - }; - let mut via_delta = base; - via_delta.apply_delta(&delta).unwrap(); - - assert_eq!(via_delta.store_counts, via_merge.store_counts); - assert_eq!(via_delta.total_count(), via_merge.total_count()); - } - - #[test] - fn test_merge_alpha_mismatch() { - let mut a = DdSketch::new(0.01); - let b = DdSketch::new(0.02); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_msgpack_round_trip() { - let original = DdSketch::from_raw(0.01, vec![1, 2, 3], -2); - let bytes = original.to_msgpack().unwrap(); - let decoded = DdSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(decoded.alpha, original.alpha); - assert_eq!(decoded.store_counts, original.store_counts); - assert_eq!(decoded.store_offset, original.store_offset); - assert_eq!(decoded.total_count(), original.total_count()); - } - - /// The msgpack wire layout MUST be a 3-element array - /// `[alpha, store_counts, store_offset]` after dropping the - /// DataPoint-level METRIC scalars (count/sum/min/max). This pins the - /// element count so the bytes stay parity-aligned with the Go - /// reference implementation. - #[test] - fn test_msgpack_is_three_element_array() { - let sk = DdSketch::from_raw(0.01, vec![1, 2, 3], -2); - let bytes = sk.to_msgpack().unwrap(); - // rmp compact encoding leads with an array marker. A fixarray of - // length 3 is the single byte 0x93 (0b1001_0011). - assert_eq!( - bytes[0], 0x93, - "expected a 3-element msgpack fixarray (0x93), got {:#04x}", - bytes[0] - ); - } - - #[test] - fn test_insert_and_quantile_lognormal() { - // Ground-truth: insert a large i.i.d. log-normal sample into - // a full sketch and sanity-check P50 / P90 / P99. - let mut gt = DdSketch::new(0.01); - let mut rng = 0xdead_beefu64; - let mut next = || { - // xorshift64* - rng ^= rng << 13; - rng ^= rng >> 7; - rng ^= rng << 17; - rng - }; - // Box-Muller normal → log-normal(mu=3, sigma=0.7). - let lognormal = |u: u64, v: u64| -> f64 { - let r1 = (u as f64) / (u64::MAX as f64).max(1.0); - let r2 = (v as f64) / (u64::MAX as f64).max(1.0); - let z = (-2.0 * r1.max(1e-12).ln()).sqrt() * (2.0 * std::f64::consts::PI * r2).cos(); - (3.0 + 0.7 * z).exp() - }; - for _ in 0..100_000 { - gt.update(lognormal(next(), next())); - } - let p50 = gt.quantile(0.5).unwrap(); - let p99 = gt.quantile(0.99).unwrap(); - // Analytical P50 = exp(mu) = e^3 ≈ 20.09; - // P99 ≈ exp(mu + sigma * Φ⁻¹(0.99)) = e^(3 + 0.7×2.326) ≈ 102.4. - assert!( - (p50 / 20.09).ln().abs() < 0.05, - "P50 {p50} not close to 20.09" - ); - assert!( - (p99 / 102.4).ln().abs() < 0.05, - "P99 {p99} not close to 102.4" - ); - } - - /// Building a sketch via `base + apply_delta()` produces quantile - /// estimates within DDSketch's α bound of the ground-truth - /// full-sketch path. - #[test] - fn test_delta_chain_preserves_quantile_accuracy() { - let alpha = 0.01; - let mut rng = 0xcafe_babeu64; - let mut next = || { - rng ^= rng << 13; - rng ^= rng >> 7; - rng ^= rng << 17; - rng - }; - let lognormal = |u: u64, v: u64| -> f64 { - let r1 = (u as f64) / (u64::MAX as f64).max(1.0); - let r2 = (v as f64) / (u64::MAX as f64).max(1.0); - let z = (-2.0 * r1.max(1e-12).ln()).sqrt() * (2.0 * std::f64::consts::PI * r2).cos(); - (3.0 + 0.7 * z).exp() - }; - - // Path A (ground truth): one sketch, 50k samples inserted - // directly. - let mut full = DdSketch::new(alpha); - // Path B (delta chain): a base sketch from the first 10k - // samples, then 4 incremental "flushes" of 10k samples each, - // each transmitted as a delta computed against the previous - // snapshot. - let mut reconstituted = DdSketch::new(alpha); - let mut prev_snapshot = DdSketch::new(alpha); // what the "receiver" has cached - - let batch = 10_000; - let batches = 5; - for b in 0..batches { - let mut this_batch = prev_snapshot.clone(); - for _ in 0..batch { - let v = lognormal(next(), next()); - full.update(v); - this_batch.update(v); - } - // Compute a "delta" = diff of this_batch vs prev_snapshot - // in our in-memory struct shape. Matches what the Go - // reference implementation's `ComputeDelta` would put on - // the wire. - let delta = compute_dd_delta(&prev_snapshot, &this_batch); - if b == 0 { - // First batch seeds the reconstituted sketch. - reconstituted = this_batch.clone(); - } else { - reconstituted.apply_delta(&delta).unwrap(); - } - prev_snapshot = this_batch; - } - - // Both paths should see the same total count (exact), recovered - // by summing bucket counts now that the `count` scalar is gone. - assert_eq!( - reconstituted.total_count(), - full.total_count(), - "count diverged" - ); - - // And P50 / P90 / P99 should agree within α bound. - for q in [0.5, 0.9, 0.99] { - let got = reconstituted.quantile(q).unwrap(); - let want = full.quantile(q).unwrap(); - let rel_err = (got / want - 1.0).abs(); - assert!( - rel_err <= alpha, - "q={q} rel_err={rel_err:.4} exceeds α={alpha}: reconstituted={got}, full={want}", - ); - } - } - - /// Helper used only in the delta-chain test: computes a - /// `DdSketchDelta` from two snapshots. Mirrors the Go reference - /// implementation's `ComputeDelta` logic so the test exercises the - /// wire-format path end-to-end. - fn compute_dd_delta(snapshot: &DdSketch, current: &DdSketch) -> DdSketchDelta { - let mut cells = Vec::new(); - if !current.store_counts.is_empty() { - for (i, &c) in current.store_counts.iter().enumerate() { - if c == 0 { - continue; - } - let k = current.store_offset + i as i32; - let snap_count: u64 = if !snapshot.store_counts.is_empty() { - let idx = k as i64 - snapshot.store_offset as i64; - if idx >= 0 && (idx as usize) < snapshot.store_counts.len() { - snapshot.store_counts[idx as usize] - } else { - 0 - } - } else { - 0 - }; - let dc = c.saturating_sub(snap_count); - if dc > 0 { - cells.push((k, dc)); - } - } - } - let d_count = current.total_count() as i64 - snapshot.total_count() as i64; - DdSketchDelta { - buckets: cells, - d_count, - d_sum: 0.0, - min_changed: false, - new_min: 0.0, - max_changed: false, - new_max: 0.0, - ..Default::default() - } - } - - /// Delta-against-empty: a `compute_delta` against an EMPTY base - /// reconstructs the window's full state when applied (round-trip). - /// With `threshold = 1` every non-empty bucket survives, so applying - /// the delta to a fresh empty sketch yields the same bucket store as - /// the original window. - #[test] - fn test_compute_delta_against_empty_round_trips() { - let mut window = DdSketch::new(0.01); - for i in 1..=200 { - window.update(i as f64); - } - let empty = DdSketch::new(0.01); - - // Delta against empty = the window's full state in delta form. - let delta_bytes = window.compute_delta(&empty, 1); - - // Apply to a fresh empty base. - let mut reconstructed = DdSketch::new(0.01); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - - // Total count recovered exactly. - assert_eq!(reconstructed.total_count(), window.total_count()); - // Every non-empty bucket count matches (compare on absolute index). - for (i, &c) in window.store_counts.iter().enumerate() { - if c == 0 { - continue; - } - let k = window.store_offset + i as i32; - let idx = (k - reconstructed.store_offset) as usize; - assert_eq!(reconstructed.store_counts[idx], c, "bucket k={k}"); - } - // Quantiles agree within the α relative-accuracy bound. - for q in [0.5, 0.9, 0.99] { - let got = reconstructed.quantile(q).unwrap(); - let want = window.quantile(q).unwrap(); - assert!( - (got / want - 1.0).abs() <= 0.01, - "q={q}: reconstructed={got} window={want}" - ); - } - } - - /// Two consecutive windows each emit their OWN state — the second - /// window's delta-against-empty is NOT diffed against the first - /// window (no cross-window subtraction). - #[test] - fn test_consecutive_windows_emit_own_state() { - let empty = DdSketch::new(0.01); - - // Overlapping value ranges so a cross-window subtraction (win2 − - // win1) would genuinely differ from win2's own state — making the - // "no subtraction" assertion meaningful. - let mut win1 = DdSketch::new(0.01); - for i in 1..=50 { - win1.update(i as f64); - win1.update(i as f64); // win1 buckets carry count 2 - } - let mut win2 = DdSketch::new(0.01); - for i in 1..=50 { - win2.update(i as f64); // same buckets, count 1 - } - - // Each window diffs against EMPTY (delta-against-empty), not against - // the prior window. - let d2_against_empty = win2.compute_delta(&empty, 1); - let d2_against_win1 = win2.compute_delta(&win1, 1); - - // The delta-against-empty reconstructs win2 exactly. - let mut recon = DdSketch::new(0.01); - recon.apply_delta_bytes(&d2_against_empty).unwrap(); - assert_eq!(recon.total_count(), win2.total_count()); - - // Sanity: diffing against win1 would have produced a DIFFERENT - // (cross-window-subtracted) frame, proving the two are not the same - // and that diffing against empty avoids the cross-window subtraction. - assert_ne!( - d2_against_empty, d2_against_win1, - "delta-against-empty must differ from cross-window delta" - ); - } - - /// Cross-language byte-parity guard against the Go reference - /// implementation's `DDSketch.SerializePortable` output for the - /// deterministic input `(1..=50)` with `α = 0.01`. The hex blob - /// below was captured from a `proto.Marshal` of the Go envelope - /// (with `Producer` and `HashSpec` cleared). Any change to - /// [`DdSketch::update`]'s bucket-store growth that breaks parity - /// will surface here. - #[test] - fn test_update_then_envelope_matches_go_golden_bytes() { - use crate::proto::sketchlib::{ - DdSketchState, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let mut sk = DdSketch::new(0.01); - for i in 1..=50 { - sk.update(i as f64); - } - - let state = DdSketchState { - alpha: sk.wire_alpha(), - store_counts: sk.store_counts.clone(), - store_offset: sk.store_offset, - ..Default::default() - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: 0.0, - sketch_state: Some(SketchState::Ddsketch(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - // Byte string for the same `(1..=50)`, α=0.01 input AFTER dropping - // the DataPoint-level METRIC scalars (count/sum/min/max → proto - // tags 4-7 reserved). 403 bytes total: a `SketchEnvelope` proto - // wrapping a `DDSketchState` carrying only `alpha`, `store_counts` - // (Go chunk-128 padded layout, offset = -64, len = 384) and - // `store_offset` (sint32 zigzag 0x7f = -64). - // - // NOTE: this golden is currently the RUST-produced value. It MUST be - // reconciled against the Go reference implementation's regenerated - // golden before declaring cross-language byte parity. - const GOLDEN_HEX: &str = "0801728e03096214ae47e17a843f128003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000100000000000000000000000000000100000000000000000000010000000000000000010000000000000001000000000001000000000001000000000001000000010000000001000000010000010000000100000100000100000100000100010000010001000100010001000100010001000100010100010100010100010101000101010100010101010101010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000187f"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "DDSketch envelope bytes diverge from golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - fn decode_hex(s: &str) -> Vec { - let bytes: Vec = s - .as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect(); - bytes - } - - #[test] - fn interpolated_quantile_preserves_adjacent_ranks_and_legacy_convention() { - let mut sketch = DdSketch::new(0.01); - sketch.update(20.0); - sketch.update(40.0); - let lower = sketch.quantile(0.0).unwrap(); - let upper = sketch.quantile(1.0).unwrap(); - assert_eq!(sketch.quantile(0.9), Some(lower)); - for (q, expected) in [(0.0, 20.0), (0.5, 30.0), (0.9, 38.0), (1.0, 40.0)] { - let actual = sketch.quantile_interpolated(q).unwrap(); - assert!((actual - expected).abs() <= expected * 0.01); - assert_eq!(actual, lower * (1.0 - q) + upper * q); - } - for q in [-0.1, 1.1, f64::NAN, f64::INFINITY] { - assert!(sketch.quantile_interpolated(q).is_none()); - } - assert!(DdSketch::new(0.01).quantile_interpolated(0.5).is_none()); - let mut singleton = DdSketch::new(0.01); - singleton.update(20.0); - assert_eq!( - singleton.quantile_interpolated(0.9), - singleton.quantile(0.0) - ); - for values in [vec![20.0, 20.0], vec![10.0, 10.0, 20.0, 40.0]] { - let mut repeated = DdSketch::new(0.01); - for &value in &values { - repeated.update(value); - } - for q in [0.0, 0.5, 0.9, 1.0] { - let rank = q * (values.len() - 1) as f64; - let fraction = rank - rank.floor(); - let expected = values[rank.floor() as usize] * (1.0 - fraction) - + values[rank.ceil() as usize] * fraction; - assert!( - (repeated.quantile_interpolated(q).unwrap() - expected).abs() - <= 0.01 * expected - ); - } - } - for alpha in [0.0, -0.1, 1.0, f64::NAN] { - assert!( - DdSketch::from_raw(alpha, vec![1], 0) - .quantile_interpolated(0.5) - .is_none() - ); - } - assert!( - DdSketch::from_raw(0.01, vec![(1u64 << 53) + 1], 0) - .quantile_interpolated(0.5) - .is_none() - ); - // Negative and zero inputs retain their ranks during interpolation. - let mut unsupported = DdSketch::new(0.01); - unsupported.update(-20.0); - unsupported.update(0.0); - assert!((unsupported.quantile_interpolated(0.9).unwrap() + 2.0).abs() <= 0.02); - let overflow = DdSketch::from_raw(0.01, vec![u64::MAX, 1], 0); - assert!(overflow.quantile_interpolated(0.5).is_none()); - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } -} diff --git a/src/message_pack_format/portable/delta_set_aggregator.rs b/src/message_pack_format/portable/delta_set_aggregator.rs deleted file mode 100644 index c0ac76c9..00000000 --- a/src/message_pack_format/portable/delta_set_aggregator.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! Wire-format DTO for the delta set aggregator. -//! -//! Owns the [`SetAggregatorDelta`] wire DTO and its [`MessagePackCodec`] impl; its -//! ASAPv1 `to_asapv1` (§3.22) is beside `SetAggregator`'s. -//! The streaming/state-tracking logic lives in the downstream consumer -//! (ASAPQuery's accumulators) — only the over-the-wire shape lives here. - -use std::collections::HashSet; - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; - -/// Wire DTO for the delta set aggregator: a snapshot of added/removed -/// string keys between two consecutive observations. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SetAggregatorDelta { - pub added: HashSet, - pub removed: HashSet, -} - -impl MessagePackCodec for SetAggregatorDelta { - fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_msgpack_round_trip() { - let mut added = HashSet::new(); - added.insert("web".to_string()); - added.insert("api".to_string()); - - let mut removed = HashSet::new(); - removed.insert("db".to_string()); - - let bytes = SetAggregatorDelta { added, removed }.to_msgpack().unwrap(); - let result = SetAggregatorDelta::from_msgpack(&bytes).unwrap(); - - assert_eq!(result.added.len(), 2); - assert!(result.added.contains("web")); - assert!(result.added.contains("api")); - assert_eq!(result.removed.len(), 1); - assert!(result.removed.contains("db")); - } - - #[test] - fn test_empty_sets() { - let dr = SetAggregatorDelta { - added: HashSet::new(), - removed: HashSet::new(), - }; - let bytes = dr.to_msgpack().unwrap(); - let result = SetAggregatorDelta::from_msgpack(&bytes).unwrap(); - assert!(result.added.is_empty()); - assert!(result.removed.is_empty()); - } -} diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs deleted file mode 100644 index 0dde2e32..00000000 --- a/src/message_pack_format/portable/hll.rs +++ /dev/null @@ -1,1071 +0,0 @@ -//! Wire-format-aligned HyperLogLog types. -//! -//! Self-contained: maintains its own `Vec` register array. Parity -//! with Go's `HyperLogLog.SerializePortable` (and equivalence with -//! `sketches::HyperLogLogImpl<.., Ertl, P14>`) is locked in by the -//! `test_update_then_envelope_matches_sketchlib_go_bytes` test below and -//! by `tests/sketches_go_parity_probe.rs`. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::kind; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::{DEFAULT_CANONICAL_SEED_INDEX, DataInput, hash64_seeded}; - -// --------------------------------------------------------------------------- -// DEPRECATED — temporary compatibility shim, slated for removal. -// -// A few call sites still depend on this portable `HllSketch`. Its ASAPv1 codec -// now reuses the shared `envelope` framing plus the HLL metadata/payload types -// in `crate::sketches::hll`, so it stays byte-identical with the native -// `HyperLogLogImpl` / `HyperLogLogHipImpl` serialization. Do NOT add new callers. -// -// Prefer instead the native path: -// `sketches::hll::HyperLogLogImpl::to_asapv1` / -// `HyperLogLogHipImpl::to_asapv1` (+ `from_asapv1`). -// --------------------------------------------------------------------------- - -use crate::message_pack_format::envelope; -use crate::sketches::hll::{HllMetadata, HllPayloadHip, HllPayloadPlain, standard_hll_metadata}; - -/// Retained only for `test_msgpack_round_trip`'s magic-prefix assertion. -#[cfg(test)] -const HLL_WRAPPER_MAGIC: &[u8; 6] = b"ASAPv1"; - -/// Map a portable [`HllVariant`] to its ASAPv1 wire `kind_id`. `Unspecified` is -/// a placeholder with no wire form. -fn wire_kind_id(variant: HllVariant) -> Result<&'static [u8], MsgPackError> { - match variant { - HllVariant::Regular => Ok(kind::HLL_CLASSIC), - HllVariant::Datafusion => Ok(kind::HLL_ERTL), - HllVariant::Hip => Ok(kind::HLL_HIP), - HllVariant::Unspecified => Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized( - "ASAPv1 HLL: the Unspecified variant is not serializable".to_string(), - ), - )), - } -} - -/// HLL estimator variant. Mirrors `asap_sketchlib::proto::sketchlib::HllVariant`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum HllVariant { - Unspecified, - Regular, - Datafusion, - Hip, -} - -/// Sparse delta between two consecutive HLL snapshots — input shape for -/// [`HllSketch::apply_delta`]. Mirrors the `HLLDelta` proto in -/// `sketchlib-go/proto/hll/hll.proto`. -#[derive(Debug, Clone, Default)] -pub struct HllSketchDelta { - pub updates: Vec<(u32, u8)>, -} - -/// Minimal HLL state — registers + variant + precision. Register-wise -/// mergeable (max over aligned cells). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HllSketch { - pub variant: HllVariant, - pub precision: u32, - pub registers: Vec, - pub hip_kxq0: f64, - pub hip_kxq1: f64, - pub hip_est: f64, -} - -impl HllSketch { - pub fn new(variant: HllVariant, precision: u32) -> Self { - let n = 1usize << precision; - Self { - variant, - precision, - registers: vec![0u8; n], - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - } - } - - pub fn from_raw( - variant: HllVariant, - precision: u32, - registers: Vec, - hip_kxq0: f64, - hip_kxq1: f64, - hip_est: f64, - ) -> Self { - Self { - variant, - precision, - registers, - hip_kxq0, - hip_kxq1, - hip_est, - } - } - - pub fn merge( - &mut self, - other: &HllSketch, - ) -> Result<(), Box> { - if self.variant != other.variant { - return Err(format!( - "HllSketch variant mismatch: self={:?}, other={:?}", - self.variant, other.variant - ) - .into()); - } - if self.precision != other.precision { - return Err(format!( - "HllSketch precision mismatch: self={}, other={}", - self.precision, other.precision - ) - .into()); - } - if self.registers.len() != other.registers.len() { - return Err(format!( - "HllSketch register-length mismatch: self={}, other={}", - self.registers.len(), - other.registers.len() - ) - .into()); - } - for (s, o) in self.registers.iter_mut().zip(other.registers.iter()) { - if *o > *s { - *s = *o; - } - } - if self.variant == HllVariant::Hip { - self.hip_kxq0 += other.hip_kxq0; - self.hip_kxq1 += other.hip_kxq1; - self.hip_est += other.hip_est; - } - Ok(()) - } - - pub fn apply_delta( - &mut self, - delta: &HllSketchDelta, - ) -> Result<(), Box> { - let n = self.registers.len(); - for (idx, new_val) in &delta.updates { - let i = *idx as usize; - if i >= n { - return Err(format!( - "HllSketchDelta index {i} out of range (precision={} → {n} registers)", - self.precision - ) - .into()); - } - if *new_val > self.registers[i] { - self.registers[i] = *new_val; - } - } - Ok(()) - } - - /// Compute a sparse, proto-marshalled `HLLDelta` of `self` against a - /// `snapshot`. A register update is included when `self`'s register - /// value increased over the snapshot's (`self[i] > snapshot[i]`); its - /// carried value is the new (larger) register value. - /// - /// This is the Rust twin of the Go reference implementation's - /// `ComputeRegisterDelta` + `SerializeRegisterDelta`. HLL uses max - /// semantics, so only increases are meaningful and the delta is - /// **lossless** — every increased register is carried, regardless of - /// `threshold` (the parameter is accepted for a uniform delta API and - /// has no effect here; an HLL register update is never dropped). The - /// returned bytes are a `prost`-encoded - /// [`crate::proto::sketchlib::HllDelta`], byte-identical to the Go - /// `proto.Marshal(HLLDelta)` output for the same inputs (cross-language - /// byte parity). - /// - /// Delta-against-empty: when `snapshot` is the all-zero sketch, every - /// non-zero register of `self` is carried, so the result is this - /// window's full register state encoded as a delta (no cross-window - /// subtraction). HLL deltas carry only register max-updates — there are - /// no DataPoint-level metric scalars to drop. - pub fn compute_delta(&self, snapshot: &HllSketch, _threshold: u64) -> Vec { - use crate::proto::sketchlib::HllDelta as ProtoDelta; - use prost::Message; - - let cur = &self.registers; - let snap = &snapshot.registers; - let n = cur.len().min(snap.len()); - - // Varint-pack the increased registers as (index_delta, value) pairs in - // ascending index order — the same layout `encode_sparse_registers` - // uses for the full sparse state. Walking registers in index order - // yields a sorted sequence, so `prev` only ever advances. - let mut packed: Vec = Vec::new(); - let mut prev: usize = 0; - for i in 0..n { - if cur[i] > snap[i] { - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, cur[i] as u64); - prev = i; - } - } - // Guard: if `self` has more registers than the snapshot (should not - // happen at a fixed precision), carry all non-zero extras. Matches - // the Go reference's trailing-register guard. - for (i, &v) in cur.iter().enumerate().take(cur.len()).skip(n) { - if v > 0 { - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, v as u64); - prev = i; - } - } - - ProtoDelta { - packed_updates: packed, - } - .encode_to_vec() - } - - /// Apply a `prost`-encoded [`crate::proto::sketchlib::HllDelta`] to this - /// sketch in place (register max-merge). The Rust twin of the Go - /// reference implementation's `DeserializeRegisterDelta` + - /// `ApplyRegisterDelta`: each update sets - /// `register[index] = max(register[index], value)`. - /// - /// Returns `Err` if `bytes` is not a valid `HLLDelta` proto or a - /// register index is out of range for this sketch's precision. - pub fn apply_delta_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - use crate::proto::sketchlib::HllDelta as ProtoDelta; - use prost::Message; - - let proto = ProtoDelta::decode(bytes)?; - - // Unpack the (index_delta, value) blob — inverse of `compute_delta`, - // same layout as `decode_sparse_registers` but bounded by the blob's - // own contents rather than a fixed register count. - let packed = &proto.packed_updates; - let mut updates: Vec<(u32, u8)> = Vec::new(); - let mut off = 0usize; - let mut prev: u32 = 0; - let mut first = true; - while off < packed.len() { - let (delta, used) = - read_uvarint(&packed[off..]).ok_or("hll: delta index-delta varint corrupt")?; - off += used; - let (val, used) = - read_uvarint(&packed[off..]).ok_or("hll: delta value varint corrupt")?; - off += used; - - let idx = prev + delta as u32; - // First delta is an absolute index (prev=0); later deltas must be - // > 0 so indices stay strictly increasing and unique. - if !first && delta == 0 { - return Err(format!("hll: delta non-increasing index at idx {idx}").into()); - } - first = false; - if val > 0xff { - return Err(format!("hll: delta value {val} exceeds u8 at idx {idx}").into()); - } - updates.push((idx, val as u8)); - prev = idx; - } - - let delta = HllSketchDelta { updates }; - self.apply_delta(&delta) - } - - /// MessagePack twin of [`Self::compute_delta`]. Carries every register - /// that increased over `snapshot` (`self[i] > snapshot[i]`) as an - /// `(index, new_value)` pair, serialized as the parallel-array - /// MessagePack layout via [`HllSketchDelta`]'s [`MessagePackCodec`] - /// instead of proto. HLL uses max semantics, so the delta is lossless - /// regardless of `_threshold` (accepted for a uniform delta API). - /// Delta-against-empty carries every non-zero register — this window's - /// full register state encoded as a delta. - pub fn compute_delta_msgpack(&self, snapshot: &HllSketch, _threshold: u64) -> Vec { - let cur = &self.registers; - let snap = &snapshot.registers; - let n = cur.len().min(snap.len()); - let mut updates: Vec<(u32, u8)> = Vec::new(); - for i in 0..n { - if cur[i] > snap[i] { - updates.push((i as u32, cur[i])); - } - } - // Guard: registers present in `self` beyond the snapshot length - // (should not happen at a fixed precision) carry their non-zero - // values. Matches the proto path's trailing-register guard. - for (i, &v) in cur.iter().enumerate().skip(n) { - if v > 0 { - updates.push((i as u32, v)); - } - } - HllSketchDelta { updates } - .to_msgpack() - .expect("HllSketchDelta msgpack encode is infallible for owned update arrays") - } - - /// MessagePack twin of [`Self::apply_delta_bytes`]. Decodes the - /// parallel-array MessagePack [`HllSketchDelta`] and applies it in - /// place (register max-merge). - /// - /// Returns `Err` if `bytes` is not a valid MessagePack `HllSketchDelta` - /// or a register index is out of range for this sketch's precision. - pub fn apply_delta_msgpack_bytes( - &mut self, - bytes: &[u8], - ) -> Result<(), Box> { - let delta = HllSketchDelta::from_msgpack(bytes)?; - self.apply_delta(&delta) - } - - pub fn merge_refs( - inputs: &[&HllSketch], - ) -> Result> { - let first = inputs - .first() - .ok_or("HllSketch::merge_refs called with empty input")?; - let mut merged = HllSketch::new(first.variant, first.precision); - for hll in inputs { - merged.merge(hll)?; - } - Ok(merged) - } - - /// Insert a value. Math mirrors `sketches::HyperLogLogImpl::update_with_hash` - /// (DEFAULT_CANONICAL_SEED_INDEX, packed bucket/leading-zero); locked in by - /// `tests/sketches_go_parity_probe.rs`. - pub fn update(&mut self, value: &[u8]) { - let hashed_val = hash64_seeded(DEFAULT_CANONICAL_SEED_INDEX, &DataInput::Bytes(value)); - let p = self.precision as usize; - let register_bits = (u64::BITS as usize) - p; - let p_mask: u64 = (1u64 << p) - 1; - let bucket_num = ((hashed_val >> register_bits) & p_mask) as usize; - let leading_zero = ((hashed_val << p) + p_mask).leading_zeros() as u8 + 1; - if bucket_num < self.registers.len() && leading_zero > self.registers[bucket_num] { - self.registers[bucket_num] = leading_zero; - } - } - - /// Estimate cardinality (Classic HLL estimator with the small-range - /// correction). Ranks are drawn from a 64-bit hash, so the estimate needs - /// no large-range correction. - pub fn estimate(&self) -> f64 { - let m = self.registers.len() as f64; - if m == 0.0 { - return 0.0; - } - let mut z = 0.0_f64; - let mut zero_count = 0usize; - for ®_val in &self.registers { - if reg_val == 0 { - zero_count += 1; - } - z += 2f64.powi(-(reg_val as i32)); - } - let indicator = 1.0 / z; - - let alpha_m = 0.7213 / (1.0 + 1.079 / m); - let mut est = alpha_m * m * m * indicator; - - if est <= m * 5.0 / 2.0 && zero_count != 0 { - est = m * (m / zero_count as f64).ln(); - } - est - } - - /// Return the proto-enum value Go's `HyperLogLog.SerializePortable` - /// emits for this sketch's [`HllVariant`]. - #[inline] - pub fn wire_proto_variant(&self) -> i32 { - match self.variant { - HllVariant::Unspecified => 0, - HllVariant::Regular => 1, - HllVariant::Datafusion => 2, - HllVariant::Hip => 3, - } - } -} - -// --------------------------------------------------------------------------- -// Proto dual-read decode (DENSE tag 3 + SPARSE tag 7) -// --------------------------------------------------------------------------- - -use crate::proto::sketchlib::{HllSparseRegisters, HyperLogLogState}; - -/// Decode the full dense register array from a `HyperLogLogState`, accepting -/// BOTH wire encodings emitted by sketchlib-go: -/// - DENSE (`registers`, tag 3): raw 1-byte-per-register array, or -/// - SPARSE (`registers_sparse`, tag 7): HLL++ style varint-packed -/// (index_delta, value) pairs for non-zero registers only. -/// -/// The SPARSE field takes priority when present; otherwise the DENSE field is -/// used. Both reconstruct the identical `2^precision`-byte register array, so -/// downstream estimation / merge is encoding-agnostic. This is the Rust half of -/// the backend-decode-first rollout: it must ship before any producer emits -/// sparse. Mirrors the Go decoder in -/// `sketchlib-go/sketches/HLL/portable.go::registersFromState` + -/// `sparse.go::decodeSparseRegisters`. -pub fn registers_from_state( - state: &HyperLogLogState, -) -> Result, Box> { - if let Some(sp) = &state.registers_sparse { - return decode_sparse_registers(sp); - } - Ok(state.registers.clone()) -} - -/// Reconstruct the dense register array from a sparse message. Inverse of the -/// Go encoder `sparse.go::encodeSparseRegisters`. -pub fn decode_sparse_registers( - sp: &HllSparseRegisters, -) -> Result, Box> { - let n = sp.num_registers as usize; - let mut regs = vec![0u8; n]; - - let packed = &sp.packed; - let mut off = 0usize; - let mut prev: usize = 0; - let mut first = true; - while off < packed.len() { - let (delta, used) = - read_uvarint(&packed[off..]).ok_or("hll: sparse index-delta varint corrupt")?; - off += used; - let (val, used) = read_uvarint(&packed[off..]).ok_or("hll: sparse value varint corrupt")?; - off += used; - - let idx = prev + delta as usize; - // First delta is an absolute index (prev=0); later deltas must be > 0 - // so indices stay strictly increasing and unique. - if !first && delta == 0 { - return Err(format!("hll: sparse non-increasing index at idx {idx}").into()); - } - first = false; - if idx >= n { - return Err(format!("hll: sparse index {idx} out of range [0,{n})").into()); - } - if val > 0xff { - return Err(format!("hll: sparse value {val} exceeds u8 at idx {idx}").into()); - } - regs[idx] = val as u8; - prev = idx; - } - Ok(regs) -} - -/// Encode the non-zero registers of `regs` into an [`HllSparseRegisters`] -/// message, byte-identical to the Go encoder `sparse.go::encodeSparseRegisters`. -/// Provided so a Rust producer (or the reverse-direction golden test) can emit -/// the same sparse wire form Go does. -pub fn encode_sparse_registers(regs: &[u8]) -> HllSparseRegisters { - let mut packed: Vec = Vec::new(); - let mut prev: usize = 0; - for (i, &v) in regs.iter().enumerate() { - if v == 0 { - continue; - } - write_uvarint(&mut packed, (i - prev) as u64); - write_uvarint(&mut packed, v as u64); - prev = i; - } - HllSparseRegisters { - num_registers: regs.len() as u32, - packed, - } -} - -/// Append one unsigned LEB128 varint to `out`. Matches Go's -/// `encoding/binary.PutUvarint`. -fn write_uvarint(out: &mut Vec, mut v: u64) { - while v >= 0x80 { - out.push((v as u8) | 0x80); - v >>= 7; - } - out.push(v as u8); -} - -/// Read one unsigned LEB128 varint, returning (value, bytes_consumed). Matches -/// Go's `encoding/binary.Uvarint`. Returns None on truncation / overflow. -fn read_uvarint(buf: &[u8]) -> Option<(u64, usize)> { - let mut result: u64 = 0; - let mut shift = 0u32; - for (i, &b) in buf.iter().enumerate() { - if i >= 10 { - return None; // varint too long for u64 - } - if shift == 63 && b > 1 { - return None; // overflow - } - result |= ((b & 0x7f) as u64) << shift; - if b < 0x80 { - return Some((result, i + 1)); - } - shift += 7; - } - None -} - -impl MessagePackCodec for HllSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let kind_id = wire_kind_id(self.variant)?; - let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(self.precision))?; - let payload = if self.variant == HllVariant::Hip { - rmp_serde::to_vec(&HllPayloadHip { - registers: self.registers.clone(), - hip_kxq0: self.hip_kxq0, - hip_kxq1: self.hip_kxq1, - hip_est: self.hip_est, - })? - } else { - rmp_serde::to_vec(&HllPayloadPlain { - registers: self.registers.clone(), - })? - }; - Ok(envelope::encode(kind_id, &metadata, &payload)) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, metadata, payload) = envelope::split(bytes) - .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; - - // Validate the hash spec is the standard profile (precision is read from - // the metadata, so compare against a standard block built with it). - let meta: HllMetadata = envelope::decode_block(metadata)?; - if meta.admission_rate.is_some() { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized( - "ASAPv1 HLL envelope: HllSketch does not carry admission_rate".to_string(), - ), - )); - } - if meta != standard_hll_metadata(meta.precision) { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized( - "ASAPv1 HLL envelope: hash metadata mismatch".to_string(), - ), - )); - } - - let variant = if kind_id == kind::HLL_CLASSIC { - HllVariant::Regular - } else if kind_id == kind::HLL_ERTL { - HllVariant::Datafusion - } else if kind_id == kind::HLL_HIP { - HllVariant::Hip - } else { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: unsupported kind_id {kind_id:?}" - )), - )); - }; - - let (registers, hip_kxq0, hip_kxq1, hip_est) = if variant == HllVariant::Hip { - let p: HllPayloadHip = envelope::decode_block(payload)?; - (p.registers, p.hip_kxq0, p.hip_kxq1, p.hip_est) - } else { - let p: HllPayloadPlain = envelope::decode_block(payload)?; - (p.registers, 0.0, 0.0, 0.0) - }; - - // Fail closed if the register bin does not match `2^precision` - // (doc §2 validation rule 3). `checked_shl` avoids a shift-overflow - // panic on a crafted out-of-range `precision`. - let expected = 1usize.checked_shl(meta.precision).ok_or_else(|| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: precision {} out of range", - meta.precision - ))) - })?; - if registers.len() != expected { - return Err(MsgPackError::Decode( - rmp_serde::decode::Error::Uncategorized(format!( - "ASAPv1 HLL envelope: {} registers, expected 2^{} = {expected}", - registers.len(), - meta.precision - )), - )); - } - - Ok(HllSketch::from_raw( - variant, - meta.precision, - registers, - hip_kxq0, - hip_kxq1, - hip_est, - )) - } -} - -impl MessagePackCodec for HllSketchDelta { - /// Encodes the register delta as the 2-element MessagePack array - /// `[ reg_idx:[]u32, reg_val:[]u8 ]` — parallel arrays of register - /// index → new (max) value — byte-identical to Rust - /// `rmp_serde::to_vec(&(Vec, Vec))` (compact mode). The - /// `Vec` serializes as a msgpack array-of-int (not `bin`), matching - /// the Go `asapmsgpack` register-array convention. - fn to_msgpack(&self) -> Result, MsgPackError> { - let reg_idx: Vec = self.updates.iter().map(|(i, _)| *i).collect(); - let reg_val: Vec = self.updates.iter().map(|(_, v)| *v).collect(); - Ok(rmp_serde::to_vec(&(reg_idx, reg_val))?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let (reg_idx, reg_val): (Vec, Vec) = rmp_serde::from_slice(bytes)?; - Ok(HllSketchDelta { - updates: reg_idx.into_iter().zip(reg_val).collect(), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Ranks are drawn from a 64-bit hash, so a register state whose estimate - /// runs past the 32-bit range must still follow the HyperLogLog formula - /// rather than saturate or turn into a NaN. - #[test] - fn estimate_holds_past_the_32_bit_range() { - for rank in [12u8, 16, 18, 20] { - let mut sketch = HllSketch::new(HllVariant::Regular, 14); - sketch.registers.fill(rank); - let m = sketch.registers.len() as f64; - let alpha_m = 0.7213 / (1.0 + 1.079 / m); - let expected = alpha_m * m * 2f64.powi(rank as i32); - let estimate = sketch.estimate(); - let error = (estimate - expected).abs() / expected; - assert!( - error <= 1e-9, - "rank {rank} estimate {estimate} deviates from {expected} by {error:.6}" - ); - } - } - - #[test] - fn msgpack_delta_against_empty_round_trips() { - let mut w = HllSketch::new(HllVariant::Regular, 12); - for i in 0..2000u64 { - w.update(&i.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Regular, 12); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut recon = HllSketch::new(HllVariant::Regular, 12); - recon.apply_delta_msgpack_bytes(&bytes).unwrap(); - // Register delta over an empty base carries every non-zero register, - // so reconstructed registers are identical. - assert_eq!(recon.registers, w.registers); - let (got, want) = (recon.estimate(), w.estimate()); - assert!((got - want).abs() <= want * 0.001, "got={got} want={want}"); - } - - #[test] - fn msgpack_delta_wire_layout_is_parallel_arrays() { - use crate::message_pack_format::MessagePackCodec; - let mut w = HllSketch::new(HllVariant::Regular, 4); - w.update(b"a"); - w.update(b"b"); - let empty = HllSketch::new(HllVariant::Regular, 4); - let bytes = w.compute_delta_msgpack(&empty, 1); - let mut reg_idx: Vec = Vec::new(); - let mut reg_val: Vec = Vec::new(); - for (i, &v) in w.registers.iter().enumerate() { - if v > 0 { - reg_idx.push(i as u32); - reg_val.push(v); - } - } - let expected = rmp_serde::to_vec(&(reg_idx, reg_val)).unwrap(); - assert_eq!(bytes, expected); - let d = HllSketchDelta::from_msgpack(&bytes).unwrap(); - assert_eq!( - d.updates.len(), - w.registers.iter().filter(|&&v| v > 0).count() - ); - } - - #[test] - fn test_new_empty() { - let h = HllSketch::new(HllVariant::Regular, 4); - assert_eq!(h.registers.len(), 16); - assert!(h.registers.iter().all(|&r| r == 0)); - } - - #[test] - fn test_merge_register_wise_max() { - let mut a = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let b = HllSketch::from_raw(HllVariant::Regular, 2, vec![4, 2, 6, 0], 0.0, 0.0, 0.0); - a.merge(&b).unwrap(); - assert_eq!(a.registers, vec![4, 5, 6, 7]); - } - - #[test] - fn test_apply_delta_max_semantics() { - let mut h = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let delta = HllSketchDelta { - updates: vec![(0, 4), (1, 2), (2, 6), (3, 0)], - }; - h.apply_delta(&delta).unwrap(); - assert_eq!(h.registers, vec![4, 5, 6, 7]); - } - - #[test] - fn test_apply_delta_out_of_range() { - let mut h = HllSketch::new(HllVariant::Regular, 2); - let delta = HllSketchDelta { - updates: vec![(7, 3)], - }; - assert!(h.apply_delta(&delta).is_err()); - } - - #[test] - fn test_apply_delta_matches_full_merge() { - let base = HllSketch::from_raw(HllVariant::Regular, 2, vec![1, 5, 3, 7], 0.0, 0.0, 0.0); - let addition = HllSketch::from_raw(HllVariant::Regular, 2, vec![4, 0, 6, 0], 0.0, 0.0, 0.0); - let mut via_merge = base.clone(); - via_merge.merge(&addition).unwrap(); - - let delta = HllSketchDelta { - updates: vec![(0, 4), (2, 6)], - }; - let mut via_delta = base; - via_delta.apply_delta(&delta).unwrap(); - assert_eq!(via_delta.registers, via_merge.registers); - } - - #[test] - fn test_merge_variant_mismatch() { - let mut a = HllSketch::new(HllVariant::Regular, 4); - let b = HllSketch::new(HllVariant::Datafusion, 4); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_merge_precision_mismatch() { - let mut a = HllSketch::new(HllVariant::Regular, 4); - let b = HllSketch::new(HllVariant::Regular, 5); - assert!(a.merge(&b).is_err()); - } - - #[test] - fn test_merge_refs() { - let a = HllSketch::from_raw(HllVariant::Regular, 1, vec![1, 0], 0.0, 0.0, 0.0); - let b = HllSketch::from_raw(HllVariant::Regular, 1, vec![0, 3], 0.0, 0.0, 0.0); - let c = HllSketch::from_raw(HllVariant::Regular, 1, vec![2, 2], 0.0, 0.0, 0.0); - let merged = HllSketch::merge_refs(&[&a, &b, &c]).unwrap(); - assert_eq!(merged.registers, vec![2, 3]); - } - - #[test] - fn test_update_then_estimate_within_2pct() { - let n: usize = 50_000; - let mut h = HllSketch::new(HllVariant::Regular, 12); - for i in 0..n { - let key = format!("key-{i}"); - h.update(key.as_bytes()); - } - let est = h.estimate(); - let rel_err = (est - n as f64).abs() / n as f64; - assert!( - rel_err < 0.02, - "HLL estimate {est} not within 2% of {n} (rel_err {rel_err:.4})", - ); - } - - #[test] - fn test_estimate_empty_is_zero() { - let h = HllSketch::new(HllVariant::Regular, 4); - assert_eq!(h.estimate(), 0.0); - } - - #[test] - fn test_msgpack_round_trip() { - let original = HllSketch::from_raw( - HllVariant::Hip, - 3, - vec![0, 1, 2, 3, 4, 5, 6, 7], - 1.0, - 2.0, - 3.0, - ); - let bytes = original.to_msgpack().unwrap(); - assert!(bytes.starts_with(HLL_WRAPPER_MAGIC)); - let decoded = HllSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(decoded.registers, original.registers); - assert_eq!(decoded.precision, original.precision); - assert_eq!(decoded.hip_kxq0, 1.0); - } - - #[test] - fn from_msgpack_rejects_a_sampled_hll() { - let sampled = crate::HyperLogLog::::default() - .with_admission_rate(0.5) - .to_asapv1() - .unwrap(); - let err = HllSketch::from_msgpack(&sampled).unwrap_err().to_string(); - assert!(err.contains("does not carry admission_rate"), "{err}"); - } - - /// Cross-language byte-parity guard against - /// `sketchlib-go::HyperLogLog.SerializePortable` for the - /// deterministic input stream `(1..=50i32).map(|i| (i as f64) - /// .to_le_bytes())`. - #[test] - fn test_update_then_envelope_matches_sketchlib_go_bytes() { - use crate::proto::sketchlib::{ - HyperLogLogState, SketchEnvelope, sketch_envelope::SketchState, - }; - use prost::Message; - - let mut sk = HllSketch::new(HllVariant::Datafusion, 14); - for i in 1..=50i32 { - let v = i as f64; - sk.update(&v.to_le_bytes()); - } - - let state = HyperLogLogState { - variant: sk.wire_proto_variant(), - precision: sk.precision, - registers: sk.registers.clone(), - hip_kxq0: sk.hip_kxq0, - hip_kxq1: sk.hip_kxq1, - hip_est: sk.hip_est, - // This parity golden predates the sparse field and intentionally - // pins the DENSE wire form, so leave the sparse field unset. - registers_sparse: None, - }; - let envelope = SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - // Golden pins the pre-sampling wire form; 0.0 is the proto3 default - // (dual-read as 1.0) so the encoded bytes are unchanged. - sample_p: 0.0, - sketch_state: Some(SketchState::Hll(state)), - }; - let mut got = Vec::with_capacity(envelope.encoded_len()); - envelope.encode(&mut got).expect("prost encode"); - - const GOLDEN_HEX: &str = include_str!("../../sketches/testdata/hll_envelope_golden.hex"); - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got.len(), - want.len(), - "HLL envelope length differs: got {} bytes, want {} bytes", - got.len(), - want.len(), - ); - assert_eq!( - got, want, - "HLL envelope bytes diverge from sketchlib-go golden" - ); - } - - fn decode_hex(s: &str) -> Vec { - let s = s.trim(); - s.as_bytes() - .chunks(2) - .map(|pair| { - let high = hex_nibble(pair[0]); - let low = hex_nibble(pair[1]); - (high << 4) | low - }) - .collect() - } - - fn hex_nibble(c: u8) -> u8 { - match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - _ => panic!("non-hex byte {}", c as char), - } - } - - // ----- sparse full-state encoding ----- - - use crate::proto::sketchlib::HyperLogLogState; - - /// A handful of (index, value) pairs that exercise small and large index - /// deltas plus the maximum register value. - fn sample_regs(n: usize) -> Vec { - let mut regs = vec![0u8; n]; - regs[0] = 1; // first delta is absolute index 0 - regs[1] = 7; // adjacent index (delta 1) - regs[300] = 51; // max p=14 register value (Q+1) - regs[8191] = 12; // mid-range, multi-byte index delta - regs[n - 1] = 3; // last register - regs - } - - #[test] - fn sparse_round_trip_exact() { - let regs = sample_regs(16384); - let sp = encode_sparse_registers(®s); - assert_eq!(sp.num_registers, 16384); - let decoded = decode_sparse_registers(&sp).unwrap(); - assert_eq!(decoded, regs, "sparse round trip must be exact"); - } - - #[test] - fn registers_from_state_reads_both() { - let regs = sample_regs(16384); - - // SPARSE: registers_sparse set, registers empty. - let sparse_state = HyperLogLogState { - variant: 2, - precision: 14, - registers: Vec::new(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: Some(encode_sparse_registers(®s)), - }; - assert_eq!(registers_from_state(&sparse_state).unwrap(), regs); - - // DENSE: registers set, registers_sparse None. - let dense_state = HyperLogLogState { - variant: 2, - precision: 14, - registers: regs.clone(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: None, - }; - assert_eq!(registers_from_state(&dense_state).unwrap(), regs); - } - - #[test] - fn sparse_empty_is_all_zero() { - let regs = vec![0u8; 16384]; - let sp = encode_sparse_registers(®s); - assert!(sp.packed.is_empty()); - assert_eq!(decode_sparse_registers(&sp).unwrap(), regs); - } - - #[test] - fn sparse_rejects_out_of_range_index() { - // packed = uvarint(delta=5) , uvarint(value=3) but num_registers=4. - let sp = HllSparseRegisters { - num_registers: 4, - packed: vec![5, 3], - }; - assert!(decode_sparse_registers(&sp).is_err()); - } - - /// `compute_delta` against an EMPTY snapshot reconstructs the window's - /// full register state when its bytes are applied to a fresh empty - /// sketch (register max-merge round-trip). HLL deltas are lossless — - /// every increased register is carried. - #[test] - fn test_compute_delta_against_empty_round_trips() { - let mut window = HllSketch::new(HllVariant::Datafusion, 14); - for i in 0..5000u64 { - window.update(&i.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Datafusion, 14); - - let delta_bytes = window.compute_delta(&empty, 0); - - let mut reconstructed = HllSketch::new(HllVariant::Datafusion, 14); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - - assert_eq!(reconstructed.registers, window.registers); - } - - /// A delta computed between two non-empty snapshots reconstructs the - /// current sketch when applied to the base (max-merge of the registers - /// that increased). - #[test] - fn test_compute_delta_then_apply_matches_current() { - let mut base = HllSketch::new(HllVariant::Datafusion, 12); - for i in 0..2000u64 { - base.update(&i.to_le_bytes()); - } - let mut current = base.clone(); - for i in 2000..5000u64 { - current.update(&i.to_le_bytes()); - } - - let delta_bytes = current.compute_delta(&base, 0); - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&delta_bytes).unwrap(); - assert_eq!(reconstructed.registers, current.registers); - } - - /// Cross-language byte-parity guard: `compute_delta` against an empty - /// snapshot must emit bytes identical to the Go reference - /// implementation's `SerializeRegisterDelta(ComputeRegisterDelta(empty, - /// current))` for the same precision-14 sketch fed `(1..=50)` as f64 - /// little-endian byte values. A delta-against-empty carries every - /// non-zero register as an `(index, value)` update; the golden hex was - /// captured from a `proto.Marshal` of the Go reference's `HLLDelta`. - #[test] - fn test_compute_delta_matches_go_golden_bytes() { - let mut current = HllSketch::new(HllVariant::Datafusion, 14); - for i in 1..=50i32 { - let v = i as f64; - current.update(&v.to_le_bytes()); - } - let empty = HllSketch::new(HllVariant::Datafusion, 14); - let got = current.compute_delta(&empty, 0); - - // Captured from the Go reference implementation's - // SerializeRegisterDelta(ComputeRegisterDelta(empty, current)) for the - // same input, with the packed (index_delta, value) HLLDelta encoding. - const GOLDEN_HEX: &str = "0a81015801bb0402800c015001970401dd0301b10101b2080132016a01a901022f018402020902a10a029a03027501c7020428042b05ec0702a60102850205c2030258021101fe030128022002a60403cd0102e7060211060b0334020b069c0101ff06012a015e0581020170041201f80401a70404a60101bc03019e0103cc0201880203"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "HLL delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - } - - /// Cross-language byte-parity guard for a SUB-WINDOW delta (not against - /// empty): a precision-14 sketch is fed `(0..2000)` as f64 little-endian - /// bytes (the base/snapshot), then `(2000..5000)`; the delta carries only - /// the registers that grew between the two, packed as (index_delta, value) - /// pairs. Must be byte-identical to the Go reference implementation's - /// `SerializeRegisterDelta(ComputeRegisterDelta(base, current))`. - #[test] - fn test_compute_subwindow_delta_matches_go_golden_bytes() { - let mut base = HllSketch::new(HllVariant::Datafusion, 14); - for i in 0..2000i32 { - base.update(&(i as f64).to_le_bytes()); - } - let mut current = base.clone(); - for i in 2000..5000i32 { - current.update(&(i as f64).to_le_bytes()); - } - let got = current.compute_delta(&base, 0); - - // Captured from the Go reference implementation's - // SerializeRegisterDelta(ComputeRegisterDelta(base, current)) for the - // same input, with the packed (index_delta, value) HLLDelta encoding. - const GOLDEN_HEX: &str = "0ae627000402030401020302030c0109020502040201011108100202021304030101010b010e02040102030b0407020201050405021404060105010301010101020101030306020201040102011101010407010106080104030101010103020c0204021105040108010101180201030f010b01010103010e01120201030101010303030301030107030a0108010a0108012106060309010302070203010202030101010e0102020a020102060101020a01040101011702020203021003060103030e02110206010102040604010d010a050c011104010104050a0103020f0203010305050209010301020401020805010101010201020104010201060501040601020203020601020304010401020101010601010419010e010503050107020302010102010d010501110105050503020104050201020215050401030101020d041a010601190209020301020404070e01090202010b01030107040a0103010403010117050602040601010301040101030203030105010401020108030b0517020c01010506010402030103040704010102020d010a011201060304020c010c020e010b020702070101040101010104021504070109040804060502021802030104021605010107020603010101010201100104010605020205010e010406040102020f0203020a02040201030201010303010401030302010a030201060514020d02050104010201050103020802040105020602080213020d02020105020301140204010a0509060d03070206020902070101080a0202010a010e0102020503080201010e0101010403080104020d0102010501040403060401050405011203010101040e010202040303020c010203090201030d010a0306020201030208020e0103010701030113020202030501020c010b0101040502010105010203060105020a0315050b030c0101010f060405040204030b03010502020101080302020c010b02060403010404030102010f020f0107030601040109060301040301010703040303020801010104010b011101070102011101010102030201070106010601030201020301030302020e030501190108030201040205020503030106030a01050112020102010107030b02060209050201010304010502030106060601050108020602080201010905080205010204010203020303060205010102070201010308160304040f02010105020605070407030b0104010701010205020c02060103010b040c0102030202060102010e030102070108030301030205010101040301030303010203020802190105020601120101020701010101010302080303010f0104020202060402020403010103010503060203030701010105030903030508010101020103020305060605030e0203020603010102060f010201040202030504030103020f01130204010201020104010102010401010403050201030502040306060d0302010a010a02080103020101020208030301030102010403030305011003030206010f01010609030302020102030402020102010103020408010502130306010303060102010c01030202030401030101020101070202010901150104010f010401080107010601010206020b0201010d040303070202011403090210010302090403010403050201010b05010401010301090102010c010b04040206010d0102010b05040301020702020302010d0204030102110106010803020108030101070104020104040201030605050102040101010302010b01020103030e031101030101020103070112010403010101010b011001060313030301070203030803050101030601050207030a0104030302020202030c010101080103010c0102020404020102030c01110102030901140307010602020303020401070102010b020402030707010101010201070705130102011b0201060501010101010201020109010601020205010b0103040504070202010501080108020703030104030401010201030a010201030206010101020104020d02040102010102040301040605100201020801030306020e010402110106030602080105011101060106020d0216010702050301020101040409010403010201040102020205010401070207010d0205040704050202021201030510040303020602020703090204060102070102010301020101010401200101030202040103021301010107050a010702030311010303020103020b010505100302060102010104020a010402070210010704030304050a02010409010101010106030901030110031002020103010d0301020202040101020806080105010401060126020201080109020e0102030503120106010901010301010b01040201020402020115020d0108030f010c0303020c0101010602080504030802050202030f030202050404010603030208010a0204010d02020202030a0209030e030304030101060d0104020702010209020a0406010101070204020b0201030d021201020106031a0504050202040102010101020103030601080102020f021a020201120112020d02030405020403090304020102010108010203080101010704060104010902010401020203040107010e02070102060601080105010101060204010e010c020d010601050108010801020508010102070302030201030101010b020b02060107010c020b01020104020302060203040c02020105010301030202020902090103030503070606010103040216020401050205020404020208010b0207010502050108010502030102040f0206021203050105010a010a010b04090101030401040c03010102040104020c04090203020301040202020101060205020d010303040202020b010c010f0205030d010301050202030a02030309020203110103010102090106020402010202010303020206010a0104010601030208020803080110020702030205050201130102022102060204021504040102010a040c0401010503030207010701050103010d0104040601100201010201020302010a010501110102080504010106030101060601040a01050202080203020509010c010102090203010d0101020102120713021104070204010201010208020502060408010309020101020d0104010206030102010201010202050202040101010102050102030201060209020b01020105040202060401020f010103090504030101030111020e031001110204020602010201011101050204010a01040402030501010202010602070503011002020406010401030103020f0403050e0104020201010101040b0105030103010106020803080201010101010202050404010103011c0306030201010105010202060206010102020101010703010502010c010303020206030c021601130121020f03020307030e0109010602040102011f01090506010b01050402020e01020303020a040201020202010301030204030a010201080404030304110104010403070203060301020107020c050502020109030901020102010801030203090d0104021502010104040504090105030a010503110102020a010203050303040e030502030101030105020108010103020102030103010209030202010205010301010202020804010207010d020d020a010605080306010c0202021101090202030f010101040109020f0301020501010103020501040109021202080601070f0204020105070119030101110208040a0203020c010201070503030b0108030201040204020402010203010c022603040505010e04010103010702010109050b020801020405020e01140204020201050406020a020301060404010c010102050106020207010102010801050202020105040103010108010113050d0109020703070205010f0703050b0112010f02010103070b010201050101030201010203010705090605020f0103040b010801020210020501060204040c010104070105020a020104040201010403040109010c0101050a0305010301090108010b030b0201010601030202030e020203030101050202010106010d010d010601020101010c0304010901010103010401030201010902010101050b01010107010105020105020c01030405011404020104030a01030105010801050202020a0107010c04150105030301050401020603020203040b01100101010301030103070101010215020302010101010101080203020a010a021d031302030104020201030305030b01030203030b03010212020302030202050d010703040202040501040101030203060405031802010206020402020109020202070305021c01010105031c010503050305030d010e01090105030401070104010302010103010201100103020b0107050301090104010203030205030103010105010504080104020302020111050601050404022202070102010201090705080b01030109012903090206030104040107010102070201040f020201030103030b020804020207012d0105010301080103020201030202050501070105010202010304020303020401030802020103020102010501010c0106021201050317010a0304010101020207090102030205010b0204010b010801080312010f01150102020802040102011102080101040a0102010c020403020203010603050203010301040111031505040202030e01010102050607050101010502020301010c0210010d0407010903020202010e01050106020401070402010c010602090311060202100206020202020316020103010101010a030301060101010203150201050e04030204021c020c020c030102030101010502060102010d02010411040c0208010101010101040201010205050101060214010901010407020206160101050407050104010101010205050501060103010101060411020302070211010201090203021a0602010f0106030a04010106010301090101070501020107060102010404020b0102010202040110010201050106020901090309020d0105020601080101010501010205030403180401030d0105040802240209010a02020305020502040102010301090303020802040108010405020201040d0403010501020102020503040215010d030902030116010f010d0107030204070204040a0109011002090119031e020502090104031b050605020201010c0207020503010104010c0102010a01060502010c020d020202050101011102050107040f0102010201010106020305020101010402010302010c0306051505010104020802010102070c010803010506011101050207020302040103010301080103080d030c0103040d01010209040501030101010402020301010f01020102010301010401010202050408011c050904080106040401040105010402120102020103030201010a030103020402021303020305020403030201030901010504010902010305010301030203020a0108010702050107020a01160205020a060201030106030e02030103031001070102030d0119020202070505021202030108010101060808010202050108010f0102010701030404020802020214030c0211010b020301080303020105080105010401100106011309040203010103010208030b020102010301010e0107021501080203010402010101010606030204010d0108010e0102010401010105030903020302030c0103010104080101010405030201020c020101070201020102010401010301080102041f01070110030101060107010201060108010d010a030a0114041003020104010302020115011a02070802010301010106020b0101010201050109020703020205010a05050402030b0108010901070201020101020105040401110101020101040103010301010107010201180101020a030201030211011a02110103010102050103040e02160106010b02050a0e0202010203030101010402120305010301030305010805070304010402050102020402040106040105060105010b01110104010c0202020b01020104020e0201020704070206010c02030105010601030106011303070103010c0601010e021a01010704020801020201030c0506010a010203010202030e0311010201050105010b010504040203040a0102010302030302010102090301010403010104010e010501050a04070201060304030b0203010e01030201011b0102020c010701040303020d0604011001130202021304020505020201010105010a010f0106020801030113010802020104040b020104080104010e03090205020905020205010e040306020304010204100106010101150103050501050101010c0102010401050201030203010315010f0111020e0403010702030203030c0101020d02030205030a01040605050a02020101011201060502020201020101010a030f010706040105010101080502020301010608040b050101010104010d0103032001030201010f01040103011003010102060401070107010f01030116030301030205030902020611030102010101010101010201020202050403011b0208010b02030202010e0108010b010e01010508040701010103050604040515020202030104020703060203010601060106020f0101020302040102010c0302010101030105010a03020311020301030201010d03040107020502010109030b0302090801070304010301010402020501010201010501050202040302030305010101040201020802020101020f0107010b020302070313010b06050201030c010b04010106050302030102030401090208010901050307020d030b0309030e01090106010302020202010501040708010e0102010401010101020304010303010505050104020201070107020101040102010e0104011a0105050a030d030f010a02130103030b0107050e0203040403010208030d0701030101080206010a010e0203020501070303011102030203010101050117010e040d040701180201010e0201010402090204012c031c010e0204020502080113010201"; - let want = decode_hex(GOLDEN_HEX); - assert_eq!( - got, - want, - "HLL sub-window delta bytes diverge from the Go reference golden \ - ({} bytes got vs {} bytes want)", - got.len(), - want.len(), - ); - - // Sanity: applying the packed sub-window delta to the base - // reconstructs the current register state (register max-merge). - let mut reconstructed = base.clone(); - reconstructed.apply_delta_bytes(&got).unwrap(); - assert_eq!(reconstructed.registers, current.registers); - } -} diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs deleted file mode 100644 index a655a157..00000000 --- a/src/message_pack_format/portable/hydra_kll.rs +++ /dev/null @@ -1,292 +0,0 @@ -//! Wire-format-aligned HydraKLL sketch type. -//! -//! `HydraKllSketch` has no `sketches::*` counterpart — it's a -//! matrix-of-KLL composite owned entirely by this module. - -use std::cmp::Ordering; - -use serde::{Deserialize, Serialize}; -use xxhash_rust::xxh32::xxh32; - -use crate::message_pack_format::portable::kll::{KllSketch, KllSketchData}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; - -#[derive(Debug, Clone)] -pub struct HydraKllSketch { - pub sketch: Vec>, - pub rows: usize, - pub cols: usize, -} - -impl HydraKllSketch { - pub fn new(rows: usize, cols: usize, k: u16) -> Self { - let sketch = vec![vec![KllSketch::new(k); cols]; rows]; - Self { sketch, rows, cols } - } - - /// [`HydraKllSketch::new`] with an explicit compaction-coin seed shared by - /// every cell. - /// - /// `new` seeds from the wall clock, so two runs over the same input give - /// different per-key quantiles. Pass a seed when the result has to be - /// reproducible — replaying a stream, comparing two processes, or - /// asserting an accuracy bound in a test. - pub fn with_seed(rows: usize, cols: usize, k: u16, seed: u64) -> Self { - let sketch = vec![vec![KllSketch::with_seed(k, seed); cols]; rows]; - Self { sketch, rows, cols } - } - - /// Number of hash rows in the sketch matrix. - pub fn rows(&self) -> usize { - self.rows - } - - /// Number of columns in the sketch matrix. - pub fn cols(&self) -> usize { - self.cols - } - - pub fn update(&mut self, key: &str, value: f64) { - let key_bytes = key.as_bytes(); - for i in 0..self.rows { - let hash_value = xxh32(key_bytes, i as u32); - let col_index = (hash_value as usize) % self.cols; - self.sketch[i][col_index].update(value); - } - } - - /// Estimate the value at the given quantile `q` for `key` — the - /// median across rows of each row's KLL quantile estimate at the - /// hashed cell. - pub fn quantile(&self, key: &str, q: f64) -> f64 { - let key_bytes = key.as_bytes(); - let mut quantiles = Vec::with_capacity(self.rows); - - for i in 0..self.rows { - let hash_value = xxh32(key_bytes, i as u32); - let col_index = (hash_value as usize) % self.cols; - quantiles.push(self.sketch[i][col_index].quantile(q)); - } - - if quantiles.is_empty() { - return 0.0; - } - - quantiles.sort_by(|a, b| match a.partial_cmp(b) { - Some(ordering) => ordering, - None => Ordering::Equal, - }); - - let mid = quantiles.len() / 2; - if quantiles.len() % 2 == 0 { - (quantiles[mid - 1] + quantiles[mid]) / 2.0 - } else { - quantiles[mid] - } - } - - /// Merge another HydraKllSketch into self in place. Both operands - /// must have identical dimensions. - pub fn merge( - &mut self, - other: &HydraKllSketch, - ) -> Result<(), Box> { - if self.rows != other.rows || self.cols != other.cols { - return Err(format!( - "HydraKllSketch dimension mismatch: self={}x{}, other={}x{}", - self.rows, self.cols, other.rows, other.cols - ) - .into()); - } - for i in 0..self.rows { - for j in 0..self.cols { - self.sketch[i][j].merge(&other.sketch[i][j])?; - } - } - Ok(()) - } - - /// Merge from references, returning a new sketch — convenience for - /// batch reduction at API edges. - pub fn merge_refs( - inputs: &[&HydraKllSketch], - ) -> Result> { - let first = inputs - .first() - .ok_or("HydraKllSketch::merge_refs called with empty input")?; - let mut merged = (*first).clone(); - for h in inputs.iter().skip(1) { - merged.merge(h)?; - } - Ok(merged) - } - - /// One-shot aggregation: build a sketch from parallel keys/values. - pub fn aggregate_hydrakll( - rows: usize, - cols: usize, - k: u16, - keys: &[&str], - values: &[f64], - ) -> Option> { - if keys.is_empty() { - return None; - } - let mut sketch = Self::new(rows, cols, k); - for (key, &value) in keys.iter().zip(values.iter()) { - sketch.update(key, value); - } - sketch.to_msgpack().ok() - } -} - -// ----- Wire format ----- - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HydraKllSketchWire { - #[serde(rename = "row_num")] - pub rows: usize, - #[serde(rename = "col_num")] - pub cols: usize, - pub sketches: Vec>, -} - -impl MessagePackCodec for HydraKllSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let mut sketches = Vec::with_capacity(self.rows); - for row in &self.sketch { - let mut row_data = Vec::with_capacity(self.cols); - for cell in row { - row_data.push(KllSketchData { - k: cell.k, - sketch_bytes: cell.sketch_bytes(), - }); - } - sketches.push(row_data); - } - let wire = HydraKllSketchWire { - rows: self.rows, - cols: self.cols, - sketches, - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - use crate::sketches::kll::Kll; - use rmp_serde::decode::Error as RmpDecodeError; - - let wire: HydraKllSketchWire = rmp_serde::from_slice(bytes)?; - - if wire.sketches.len() != wire.rows { - return Err(MsgPackError::Decode(RmpDecodeError::Uncategorized( - format!( - "HydraKLL row count mismatch: expected {}, got {}", - wire.rows, - wire.sketches.len() - ), - ))); - } - - let mut sketch: Vec> = Vec::with_capacity(wire.rows); - for (row_idx, row) in wire.sketches.into_iter().enumerate() { - if row.len() != wire.cols { - return Err(MsgPackError::Decode(RmpDecodeError::Uncategorized( - format!( - "HydraKLL column count mismatch in row {}: expected {}, got {}", - row_idx, - wire.cols, - row.len() - ), - ))); - } - let mut accum_row: Vec = Vec::with_capacity(wire.cols); - for cell in row { - let backend = Kll::from_asapv1(&cell.sketch_bytes)?; - accum_row.push(KllSketch { k: cell.k, backend }); - } - sketch.push(accum_row); - } - - Ok(Self { - sketch, - rows: wire.rows, - cols: wire.cols, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_creation() { - let h = HydraKllSketch::new(2, 3, 200); - assert_eq!(h.rows, 2); - assert_eq!(h.cols, 3); - assert_eq!(h.sketch.len(), 2); - assert_eq!(h.sketch[0].len(), 3); - } - - #[test] - fn test_update_and_query() { - let mut h = HydraKllSketch::new(2, 10, 200); - h.update("key1", 5.0); - h.update("key1", 10.0); - let q = h.quantile("key1", 0.5); - assert!(q >= 0.0); - } - - #[test] - fn test_merge() { - let mut h1 = HydraKllSketch::new(2, 5, 200); - let mut h2 = HydraKllSketch::new(2, 5, 200); - - for i in 1..=5 { - h1.update("key1", i as f64); - } - for i in 6..=10 { - h2.update("key1", i as f64); - } - - h1.merge(&h2).unwrap(); - assert_eq!(h1.rows, 2); - assert_eq!(h1.cols, 5); - } - - #[test] - fn test_merge_dimension_mismatch() { - let mut h1 = HydraKllSketch::new(2, 5, 200); - let h2 = HydraKllSketch::new(3, 5, 200); - assert!(h1.merge(&h2).is_err()); - } - - #[test] - fn test_msgpack_round_trip() { - let mut h = HydraKllSketch::new(2, 3, 200); - h.update("key1", 5.0); - h.update("key2", 10.0); - - let bytes = h.to_msgpack().unwrap(); - let deserialized = HydraKllSketch::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.rows, 2); - assert_eq!(deserialized.cols, 3); - } - - #[test] - fn test_aggregate_hydrakll() { - let keys = ["a", "b", "a"]; - let values = [1.0, 2.0, 3.0]; - let bytes = HydraKllSketch::aggregate_hydrakll(2, 5, 200, &keys, &values).unwrap(); - let h = HydraKllSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(h.rows, 2); - assert_eq!(h.cols, 5); - } - - #[test] - fn test_aggregate_hydrakll_empty() { - assert!(HydraKllSketch::aggregate_hydrakll(2, 5, 200, &[], &[]).is_none()); - } -} diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs deleted file mode 100644 index ab686338..00000000 --- a/src/message_pack_format/portable/kll.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! Wire-format-aligned KLL sketch type for cross-language interop. -//! -//! `KllSketch` is a thin facade over the pure-Rust [`crate::sketches::kll::Kll`]. -//! The wire shape -//! also serves as a nested field for -//! [`crate::message_pack_format::portable::hydra_kll`]. - -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; -use crate::sketches::kll::Kll; - -/// Concrete KLL type backing the wire-format `KllSketch`. -pub type SketchlibKll = Kll; - -/// Creates a fresh sketchlib KLL sketch with the requested accuracy -/// parameter `k`. -pub fn new_sketchlib_kll(k: u16) -> SketchlibKll { - Kll::with_k(k as i32) -} - -/// [`new_sketchlib_kll`] with an explicit compaction-coin seed, so the sketch -/// replays identically across runs. -pub fn new_sketchlib_kll_with_seed(k: u16, seed: u64) -> SketchlibKll { - Kll::with_k_and_seed(k as i32, seed) -} - -/// Updates a sketchlib KLL with one numeric observation. -pub fn sketchlib_kll_update(inner: &mut SketchlibKll, value: f64) { - inner.update(&value); -} - -/// Queries a sketchlib KLL for the value at the requested quantile. -pub fn sketchlib_kll_quantile(inner: &SketchlibKll, q: f64) -> f64 { - inner.quantile(q) -} - -/// Merges `src` into `dst`. -pub fn sketchlib_kll_merge(dst: &mut SketchlibKll, src: &SketchlibKll) { - dst.merge(src); -} - -/// Serializes a sketchlib KLL into MessagePack bytes. -pub fn bytes_from_sketchlib_kll(inner: &SketchlibKll) -> Vec { - inner.to_asapv1().unwrap() -} - -/// Deserializes a sketchlib KLL from MessagePack bytes. -pub fn sketchlib_kll_from_bytes(bytes: &[u8]) -> Result> { - Ok(Kll::from_asapv1(bytes)?) -} - -/// Wire-format KLL sketch — a thin facade over [`SketchlibKll`] with a -/// fixed `k` parameter and msgpack codec via [`KllSketchData`]. -pub struct KllSketch { - pub k: u16, - pub(crate) backend: SketchlibKll, -} - -impl KllSketch { - pub fn new(k: u16) -> Self { - Self { - k, - backend: new_sketchlib_kll(k), - } - } - - /// [`KllSketch::new`] with an explicit compaction-coin seed. - /// - /// `new` seeds the coin from the wall clock, so two runs over the same - /// input produce different sketches. Pass a seed when the result has to be - /// reproducible — replaying a stream, comparing two processes, or - /// asserting an accuracy bound in a test. - pub fn with_seed(k: u16, seed: u64) -> Self { - Self { - k, - backend: new_sketchlib_kll_with_seed(k, seed), - } - } - - /// The configured `k` parameter (compactor capacity). - pub fn k(&self) -> u16 { - self.k - } - - /// Returns the raw sketch bytes (for JSON serialization, etc.). - pub fn sketch_bytes(&self) -> Vec { - bytes_from_sketchlib_kll(&self.backend) - } - - pub fn update(&mut self, value: f64) { - sketchlib_kll_update(&mut self.backend, value); - } - - pub fn count(&self) -> u64 { - self.backend.count() as u64 - } - - /// Estimate the value at the given quantile `q ∈ [0, 1]`. - pub fn quantile(&self, q: f64) -> f64 { - if self.count() == 0 { - return 0.0; - } - sketchlib_kll_quantile(&self.backend, q) - } - - /// Merge another KllSketch into self in place. Both operands must - /// have identical `k`. - pub fn merge( - &mut self, - other: &KllSketch, - ) -> Result<(), Box> { - if self.k != other.k { - return Err(format!("KllSketch k mismatch: self={}, other={}", self.k, other.k).into()); - } - sketchlib_kll_merge(&mut self.backend, &other.backend); - Ok(()) - } - - /// Merge from references without cloning. - pub fn merge_refs( - sketches: &[&Self], - ) -> Result> { - if sketches.is_empty() { - return Err("No sketches to merge".into()); - } - let k = sketches[0].k; - for s in sketches { - if s.k != k { - return Err("Cannot merge KllSketch with different k values".into()); - } - } - let mut merged = Self::new(k); - for s in sketches { - sketchlib_kll_merge(&mut merged.backend, &s.backend); - } - Ok(merged) - } - - /// One-shot aggregation: build a sketch from a slice of values. - pub fn aggregate_kll(k: u16, values: &[f64]) -> Option> { - if values.is_empty() { - return None; - } - let mut sketch = Self::new(k); - for &value in values { - sketch.update(value); - } - sketch.to_msgpack().ok() - } -} - -impl Clone for KllSketch { - fn clone(&self) -> Self { - // Structural clone: the backing `Kll` derives `Clone` (its fields - // are `Box<[..]>`/`Vec`/scalars), so copy it directly instead of a full - // msgpack serialize -> deserialize -> rebuild round-trip. Produces an - // identical sketch far more cheaply (no rmp encode/decode, far fewer - // allocations). - Self { - k: self.k, - backend: self.backend.clone(), - } - } -} - -impl KllSketch { - /// Reconstruct directly from portable wire state (k + level-ordered items + - /// level boundaries) without replaying items through `update()`. Bit-exact. - pub fn from_portable_state( - k: u16, - items: &[f64], - levels: &[usize], - num_levels: usize, - ) -> Result { - Ok(Self { - k, - backend: SketchlibKll::from_portable_state(k as usize, items, levels, num_levels)?, - }) - } -} - -impl std::fmt::Debug for KllSketch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("KllSketch") - .field("k", &self.k) - .field("sketch_n", &self.count()) - .finish() - } -} - -// Thread safety: sketchlib's KLL is used in a single-threaded context per -// accumulator instance and only shares read-only operations across threads. -unsafe impl Send for KllSketch {} -unsafe impl Sync for KllSketch {} - -/// Wire DTO for [`KllSketch`]. Also referenced as a nested field by -/// [`crate::message_pack_format::portable::hydra_kll::HydraKllSketchWire`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct KllSketchData { - pub k: u16, - pub sketch_bytes: Vec, -} - -impl MessagePackCodec for KllSketch { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wire = KllSketchData { - k: self.k, - sketch_bytes: self.sketch_bytes(), - }; - Ok(rmp_serde::to_vec(&wire)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wire: KllSketchData = rmp_serde::from_slice(bytes)?; - let backend = Kll::from_asapv1(&wire.sketch_bytes)?; - Ok(Self { k: wire.k, backend }) - } -} - -impl MessagePackCodec for KllSketchData { - /// Encodes the `(k, sketch_bytes)` DTO directly, byte-identical to - /// [`KllSketch::to_msgpack`] (which serializes this same struct). Lets - /// producers that hold a raw sketchlib KLL backend (via - /// `to_asapv1`) emit the portable KLL msgpack form without - /// constructing a [`KllSketch`] facade. - fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn kll_data_msgpack_matches_kll_sketch() { - let mut sk = KllSketch::new(200); - for i in 0..500 { - sk.update(i as f64); - } - let via_sketch = sk.to_msgpack().unwrap(); - let data = KllSketchData { - k: sk.k(), - sketch_bytes: sk.sketch_bytes(), - }; - let via_data = data.to_msgpack().unwrap(); - assert_eq!( - via_sketch, via_data, - "KllSketchData codec must byte-match KllSketch" - ); - let rt = KllSketchData::from_msgpack(&via_data).unwrap(); - assert_eq!(rt.k, data.k); - assert_eq!(rt.sketch_bytes, data.sketch_bytes); - } - - #[test] - fn test_kll_creation() { - let kll = KllSketch::new(200); - assert_eq!(kll.count(), 0); - assert_eq!(kll.k, 200); - } - - #[test] - fn test_kll_update() { - let mut kll = KllSketch::new(200); - kll.update(10.0); - kll.update(20.0); - kll.update(15.0); - assert_eq!(kll.count(), 3); - } - - #[test] - fn test_kll_quantile() { - let mut kll = KllSketch::new(200); - for i in 1..=10 { - kll.update(i as f64); - } - assert_eq!(kll.quantile(0.0), 1.0); - assert_eq!(kll.quantile(1.0), 10.0); - let median = kll.quantile(0.5); - assert!( - (5.0..=6.0).contains(&median), - "median should be between 5 and 6; got {median}" - ); - } - - #[test] - fn test_kll_merge() { - let mut kll1 = KllSketch::new(200); - let mut kll2 = KllSketch::new(200); - - for i in 1..=5 { - kll1.update(i as f64); - } - for i in 6..=10 { - kll2.update(i as f64); - } - - kll1.merge(&kll2).unwrap(); - assert_eq!(kll1.count(), 10); - assert_eq!(kll1.quantile(0.0), 1.0); - assert_eq!(kll1.quantile(1.0), 10.0); - } - - #[test] - fn test_msgpack_round_trip() { - let mut kll = KllSketch::new(200); - for i in 1..=5 { - kll.update(i as f64); - } - - let bytes = kll.to_msgpack().unwrap(); - let deserialized = KllSketch::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.k, 200); - assert_eq!(deserialized.count(), 5); - assert_eq!(deserialized.quantile(0.0), 1.0); - assert_eq!(deserialized.quantile(1.0), 5.0); - } - - #[test] - fn test_aggregate_kll() { - let values = [1.0, 2.0, 3.0, 4.0, 5.0]; - let bytes = KllSketch::aggregate_kll(200, &values).unwrap(); - let kll = KllSketch::from_msgpack(&bytes).unwrap(); - assert_eq!(kll.count(), 5); - assert_eq!(kll.quantile(0.0), 1.0); - assert_eq!(kll.quantile(1.0), 5.0); - } - - #[test] - fn test_aggregate_kll_empty() { - assert!(KllSketch::aggregate_kll(200, &[]).is_none()); - } -} diff --git a/src/message_pack_format/portable/mod.rs b/src/message_pack_format/portable/mod.rs deleted file mode 100644 index bd7189f9..00000000 --- a/src/message_pack_format/portable/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Older per-sketch MessagePack wire types, being retired. -//! -//! Each per-algorithm submodule holds one sketch's pre-envelope wire type. -//! These carry a cross-language contract of their own, on goldens apart -//! from ASAPv1's `asapv1_golden/`: HLL, Count-Min, Count Sketch and KLL pin -//! byte parity against Go's serializers, KLL in both directions; DDSketch's -//! golden is a Rust self-pin, not yet reconciled. New code uses ASAPv1: -//! prefer a sketch's own `to_asapv1`; do not add callers here. - -pub mod countminsketch; -pub mod countminsketch_topk; -pub mod countsketch; -pub mod countsketch_topk; -pub mod ddsketch; -pub mod delta_set_aggregator; -pub mod hll; -pub mod hydra_kll; -pub mod kll; -pub mod sampling; -pub mod set_aggregator; diff --git a/src/message_pack_format/portable/sampling.rs b/src/message_pack_format/portable/sampling.rs deleted file mode 100644 index 9f62b452..00000000 --- a/src/message_pack_format/portable/sampling.rs +++ /dev/null @@ -1,133 +0,0 @@ -//! Query-time rescaling for NitroSketch / geometric-sampled sketches. -//! -//! Sampling lives on the wire as a single `sample_p` field on the -//! [`SketchEnvelope`] (NOT inside any -//! per-sketch state struct — so downstream code that builds state structs by -//! literal is unaffected). The producer admits each stream update with -//! probability `p` and stores the RAW SAMPLED state. The backend reads `p` here -//! and applies the `× 1/p` rescale at QUERY time: -//! -//! - **count-like** estimators (HLL cardinality, CountMin / CountSketch -//! frequency, SUM / COUNT) are inverse-probability unbiased: multiply by -//! `1/p`. See [`rescale_count`]. -//! - **quantile** estimators (KLL, DDSketch) are scale-invariant under uniform -//! sampling — every retained item carries the same `1/p` weight, which -//! cancels in the rank computation — so the quantile is read unchanged. The -//! carried `p` is informational (it widens the error budget, §7). See -//! [`is_quantile_scale_invariant`]. -//! -//! Dual-read default: an absent / `0.0` `sample_p` (every pre-sampling -//! producer, and every `p = 1.0` exact producer, which leaves the field at the -//! proto3 default) is interpreted as `1.0` — i.e. no rescale, a true no-op. - -use crate::proto::sketchlib::SketchEnvelope; - -/// Read the effective sampling probability from an envelope, applying the -/// dual-read default: a `0.0` (unset / pre-sampling) `sample_p` means `1.0` -/// (exact, no sampling). Values are clamped to `(0, 1]`; a malformed negative or -/// `>1` value falls back to `1.0` rather than corrupting the rescale. -#[inline] -pub fn effective_sample_p(env: &SketchEnvelope) -> f64 { - sample_p_or_default(env.sample_p) -} - -/// Apply the dual-read default to a raw `sample_p` scalar (e.g. read directly -/// off a decoded envelope field). `0.0` → `1.0`; out-of-range → `1.0`. -#[inline] -pub fn sample_p_or_default(p: f64) -> f64 { - if p > 0.0 && p <= 1.0 { - p - } else { - // 0.0 (unset / exact), NaN, negative, or >1 all mean "no sampling". - 1.0 - } -} - -/// Rescale a count-like estimate (cardinality / frequency / sum / count) read -/// from a sampled sketch back to its unbiased full-stream estimate: `raw / p`. -/// With `p == 1.0` this returns `raw` unchanged. -#[inline] -pub fn rescale_count(raw: f64, p: f64) -> f64 { - let p = sample_p_or_default(p); - raw / p -} - -/// Rescale a count-like estimate using the `sample_p` carried by `env`. -#[inline] -pub fn rescale_count_with_env(raw: f64, env: &SketchEnvelope) -> f64 { - rescale_count(raw, effective_sample_p(env)) -} - -/// Whether a quantile estimate needs NO rescale under uniform sampling. -/// Always `true`: KLL and DDSketch quantiles are scale-invariant because every -/// retained item carries the identical `1/p` weight, which cancels in the rank. -/// Provided as a named, self-documenting guard for call sites that branch on -/// family. -#[inline] -pub const fn is_quantile_scale_invariant() -> bool { - true -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::proto::sketchlib::{HyperLogLogState, SketchEnvelope, sketch_envelope::SketchState}; - - fn env_with_p(p: f64) -> SketchEnvelope { - SketchEnvelope { - format_version: 1, - producer: None, - hash_spec: None, - sample_p: p, - sketch_state: Some(SketchState::Hll(HyperLogLogState { - variant: 2, - precision: 14, - registers: Vec::new(), - hip_kxq0: 0.0, - hip_kxq1: 0.0, - hip_est: 0.0, - registers_sparse: None, - })), - } - } - - #[test] - fn dual_read_zero_means_one() { - // A pre-sampling / exact envelope leaves sample_p at the proto3 default. - assert_eq!(effective_sample_p(&env_with_p(0.0)), 1.0); - assert_eq!(sample_p_or_default(0.0), 1.0); - } - - #[test] - fn out_of_range_falls_back_to_one() { - assert_eq!(sample_p_or_default(-0.5), 1.0); - assert_eq!(sample_p_or_default(1.5), 1.0); - assert_eq!(sample_p_or_default(f64::NAN), 1.0); - } - - #[test] - fn valid_p_passes_through() { - assert_eq!(effective_sample_p(&env_with_p(0.1)), 0.1); - assert_eq!(sample_p_or_default(1.0), 1.0); - } - - #[test] - fn rescale_count_inverts_probability() { - // Sampled raw count ≈ p·n; raw/p recovers n. - assert!((rescale_count(10_000.0, 0.1) - 100_000.0).abs() < 1e-6); - assert_eq!(rescale_count(123.0, 1.0), 123.0); - // Dual-read: a 0.0 p must not divide-by-zero; it means no sampling. - assert_eq!(rescale_count(123.0, 0.0), 123.0); - } - - #[test] - fn rescale_with_env() { - assert!((rescale_count_with_env(5_000.0, &env_with_p(0.05)) - 100_000.0).abs() < 1e-6); - assert_eq!(rescale_count_with_env(42.0, &env_with_p(0.0)), 42.0); - } - - #[test] - fn quantiles_are_scale_invariant() { - assert!(is_quantile_scale_invariant()); - } -} diff --git a/src/sketch_framework/eh/wire.rs b/src/sketch_framework/eh/wire.rs index bf76e49c..24f5ba65 100644 --- a/src/sketch_framework/eh/wire.rs +++ b/src/sketch_framework/eh/wire.rs @@ -30,9 +30,9 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::EXPONENTIAL_HISTOGRAM; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::EXPONENTIAL_HISTOGRAM; use crate::sketch_framework::eh_bucket_sketch::wire::{SketchState, rebuild_sketch, sketch_state}; use super::{EhBucket, ExponentialHistogram, compute_l2_mass, infer_merge_norm}; diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index d3a26d3a..c1330f2a 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -1,6 +1,6 @@ //! Sketch enums used by the exponential histogram framework. -use crate::message_pack_format::admission_rate; +use crate::asapv1::admission_rate; use crate::{DataInput, Vector2D}; use serde::{Deserialize, Serialize}; diff --git a/src/sketch_framework/eh_bucket_sketch/wire.rs b/src/sketch_framework/eh_bucket_sketch/wire.rs index eeafda5d..5dfcaca8 100644 --- a/src/sketch_framework/eh_bucket_sketch/wire.rs +++ b/src/sketch_framework/eh_bucket_sketch/wire.rs @@ -30,10 +30,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::EH_BUCKET_SKETCH; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::EH_BUCKET_SKETCH; use super::EhBucketSketch; diff --git a/src/sketch_framework/hydra/wire.rs b/src/sketch_framework/hydra/wire.rs index 28b74607..373689dc 100644 --- a/src/sketch_framework/hydra/wire.rs +++ b/src/sketch_framework/hydra/wire.rs @@ -46,13 +46,13 @@ use serde::{Deserialize, Serialize}; use crate::common::hash::check_matrix_rows; use crate::input::HydraCounter; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::{ +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::{ HYDRA_COUNT_MIN, HYDRA_COUNT_SKETCH, HYDRA_HLL, HYDRA_KLL, HYDRA_UNIVMON, }; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::wire_key::exact_vec; use crate::sketch_framework::univmon::wire::{PyramidPayload, UnivMonMetadata, univmon_metadata}; use crate::sketches::countminsketch_heap::heap_wire::{EMPTY_KEY_TYPE, check_empty_key_type}; use crate::{ @@ -1114,8 +1114,8 @@ fn pack_univmon_cells(key_type: &str, payloads: &[&[u8]]) -> Result, Rmp "usize" => pack!(usize), "f32" => pack!(f32), "f64" => pack!(f64), - "string" => pack!(crate::message_pack_format::wire_key::WireString), - "bytes" => pack!(crate::message_pack_format::wire_key::WireBytes), + "string" => pack!(crate::asapv1::wire_key::WireString), + "bytes" => pack!(crate::asapv1::wire_key::WireBytes), other => Err(encode_error(format!( "ASAPv1 Hydra: key_type {other:?} is not a wire key type" ))), @@ -1149,8 +1149,8 @@ fn unpack_univmon_cells(key_type: &str, payload: &[u8]) -> Result>, "usize" => unpack!(usize), "f32" => unpack!(f32), "f64" => unpack!(f64), - "string" => unpack!(crate::message_pack_format::wire_key::WireString), - "bytes" => unpack!(crate::message_pack_format::wire_key::WireBytes), + "string" => unpack!(crate::asapv1::wire_key::WireString), + "bytes" => unpack!(crate::asapv1::wire_key::WireBytes), other => Err(decode_error(format!( "ASAPv1 Hydra: key_type {other:?} is not a wire key type" ))), @@ -1347,7 +1347,7 @@ mod tests { /// UTF-8 `bin` under `cell_key_type` `"string"` are rejected. #[test] fn hydra_univmon_reads_string_keys_from_str_only() { - use crate::message_pack_format::wire_key::WireBytes; + use crate::asapv1::wire_key::WireBytes; let mut hydra = grid(HydraCounter::UnivMon(UnivMon::new(4, 2, 16, 3))); let keys = ["alpha", "beta", "gamma", "delta"]; feed(&mut hydra, (0..40).map(|i| DataInput::Str(keys[i % 4]))); diff --git a/src/sketch_framework/univmon/wire.rs b/src/sketch_framework/univmon/wire.rs index c2ff2bfb..d0e4484b 100644 --- a/src/sketch_framework/univmon/wire.rs +++ b/src/sketch_framework/univmon/wire.rs @@ -34,10 +34,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::UNIVMON; -use crate::message_pack_format::wire_key::{WireBytes, WireString, exact_vec}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::UNIVMON; +use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; use crate::sketches::countminsketch_heap::heap_wire::{ EMPTY_KEY_TYPE, check_distinct_keys, check_empty_key_type, heap_entries, key_type_of, rebuild_heap, diff --git a/src/sketch_framework/univmon_q/wire.rs b/src/sketch_framework/univmon_q/wire.rs index 2422c80f..b2749cb4 100644 --- a/src/sketch_framework/univmon_q/wire.rs +++ b/src/sketch_framework/univmon_q/wire.rs @@ -34,9 +34,9 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::UNIVMON_Q; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::UNIVMON_Q; use crate::{HashProfile, SketchHasher}; use super::{ diff --git a/src/sketch_framework/univmon_two_tier/wire.rs b/src/sketch_framework/univmon_two_tier/wire.rs index bb777fb8..4c978a48 100644 --- a/src/sketch_framework/univmon_two_tier/wire.rs +++ b/src/sketch_framework/univmon_two_tier/wire.rs @@ -21,9 +21,9 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::UNIVMON_TWO_TIER; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::UNIVMON_TWO_TIER; use crate::sketch_framework::univmon::wire::{ check_layers, decode_pyramid, encode_pyramid, pyramid_key_type, pyramid_state, rebuild_layers, update_mode_of, update_mode_tag, diff --git a/src/sketches/bloom/wire.rs b/src/sketches/bloom/wire.rs index 16800fd6..c1512f33 100644 --- a/src/sketches/bloom/wire.rs +++ b/src/sketches/bloom/wire.rs @@ -25,9 +25,9 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::BLOOM; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::BLOOM; use crate::{BitMatrix, HashProfile, SketchHasher}; use super::{BLOOM_MAX_BITS, BLOOM_MAX_SLICES, Bloom, BloomMode}; diff --git a/src/sketches/coco/wire.rs b/src/sketches/coco/wire.rs index 0de335f9..34658ab3 100644 --- a/src/sketches/coco/wire.rs +++ b/src/sketches/coco/wire.rs @@ -40,9 +40,9 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COCO; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COCO; use crate::{DataInput, HashProfile, SketchHasher, Vector2D}; use super::{CocoBucket, CocoRng, CocoSketch}; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 8038640e..3e58b4ca 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::structure_utils::AdmittedRows; -use crate::message_pack_format::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; use crate::octo_delta::{ CountMinSketchCellDelta, KeyedCountMinSketchCellDelta, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_MAX_PROMOTION_THRESHOLD, diff --git a/src/sketches/countminsketch/delta.rs b/src/sketches/countminsketch/delta.rs index 3f572afe..47784bb4 100644 --- a/src/sketches/countminsketch/delta.rs +++ b/src/sketches/countminsketch/delta.rs @@ -7,12 +7,12 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::{add_cells, check_cells}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_MIN_DELTA; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::{add_cells, check_cells}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_MIN_DELTA; +use crate::asapv1::wire_key::exact_vec; use crate::{DefaultXxHasher, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::CountMinSketch; @@ -131,7 +131,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::envelope::hex; + use crate::asapv1::envelope::hex; type Delta = CountMinSketchDelta; diff --git a/src/sketches/countminsketch/wire.rs b/src/sketches/countminsketch/wire.rs index bd7bbded..60b02da2 100644 --- a/src/sketches/countminsketch/wire.rs +++ b/src/sketches/countminsketch/wire.rs @@ -21,12 +21,12 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::DeltaCounter; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_MIN; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::DeltaCounter; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_MIN; +use crate::asapv1::wire_key::exact_vec; use crate::{FastPath, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::CountMinSketch; diff --git a/src/sketches/countminsketch_heap/delta.rs b/src/sketches/countminsketch_heap/delta.rs index 5b7728cd..dc4c900c 100644 --- a/src/sketches/countminsketch_heap/delta.rs +++ b/src/sketches/countminsketch_heap/delta.rs @@ -8,10 +8,10 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::delta::{add_cells, check_cells}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_MIN_WITH_HEAP_DELTA; +use crate::asapv1::delta::{add_cells, check_cells}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_MIN_WITH_HEAP_DELTA; use crate::sketches::countminsketch::{CmsWireCounter, CmsWireMode}; use crate::{ DefaultXxHasher, HashProfile, HeapCounter, RegularPath, SketchHasher, TopKHeap, Vector2D, @@ -156,7 +156,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::envelope::hex; + use crate::asapv1::envelope::hex; use crate::{DataInput, OwnedDataInput}; type Delta = CountMinSketchWithHeapDelta; diff --git a/src/sketches/countminsketch_heap/heap_wire.rs b/src/sketches/countminsketch_heap/heap_wire.rs index c632278a..f851cd5f 100644 --- a/src/sketches/countminsketch_heap/heap_wire.rs +++ b/src/sketches/countminsketch_heap/heap_wire.rs @@ -50,8 +50,8 @@ use std::cmp::Ordering; use std::collections::HashSet; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::envelope; -use crate::message_pack_format::wire_key::{WireBytes, WireString, exact_vec}; +use crate::asapv1::envelope; +use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; use crate::{HashProfile, HeapCount, OwnedDataInput, TopKHeap}; /// Metadata `key_type` of a heap that holds nothing. A heap with no entries has @@ -606,7 +606,7 @@ pub(crate) fn rebuild_heap( #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::envelope; + use crate::asapv1::envelope; use crate::{DataInput, DefaultXxHasher, RegularPath, Vector2D}; use rmp_serde::from_slice; diff --git a/src/sketches/countminsketch_heap/wire.rs b/src/sketches/countminsketch_heap/wire.rs index c97e3132..7e3198f9 100644 --- a/src/sketches/countminsketch_heap/wire.rs +++ b/src/sketches/countminsketch_heap/wire.rs @@ -19,10 +19,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_MIN_WITH_HEAP; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_MIN_WITH_HEAP; use crate::sketches::countminsketch::{CmsWireCounter, CmsWireMode}; use crate::{CountMinSketch, HashProfile, HeapCounter, SketchHasher, Vector2D}; diff --git a/src/sketches/countminsketch_space_saving/summary_wire.rs b/src/sketches/countminsketch_space_saving/summary_wire.rs index ab15c51e..64afc5cc 100644 --- a/src/sketches/countminsketch_space_saving/summary_wire.rs +++ b/src/sketches/countminsketch_space_saving/summary_wire.rs @@ -6,8 +6,8 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::wire_key::{WireBytes, WireString, exact_vec}; +use crate::asapv1::envelope; +use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; use crate::sketches::countminsketch_heap::heap_wire::{ EMPTY_KEY_TYPE, check_distinct_keys, check_empty_key_type, check_geometry, key_type_of, }; @@ -347,7 +347,7 @@ pub(crate) fn decode( #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::kind; + use crate::asapv1::kind; use crate::{ CountMinSketch, CountMinSketchWithHeap, CountMinSketchWithSpaceSaving, CountSketchWithSpaceSaving, DataInput, DefaultXxHasher, FastPath, MATRIX_MAX_ROWS, diff --git a/src/sketches/countminsketch_space_saving/wire.rs b/src/sketches/countminsketch_space_saving/wire.rs index deec30d0..b5f931d0 100644 --- a/src/sketches/countminsketch_space_saving/wire.rs +++ b/src/sketches/countminsketch_space_saving/wire.rs @@ -5,8 +5,8 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::kind; +use crate::asapv1::admission_rate; +use crate::asapv1::kind; use crate::sketches::countminsketch::{CmsWireCounter, CmsWireMode}; use crate::{CountMinSketch, HashProfile, SketchHasher, Vector2D}; diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 036ff942..8feb53b5 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -8,7 +8,7 @@ //! ICALP 2002. use crate::common::structure_utils::AdmittedRows; -use crate::message_pack_format::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; use crate::{ DataInput, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, DefaultXxHasher, FastPath, FastPathHasher, FixedMatrix, MatrixFastHash, MatrixStorage, NitroTarget, QuickMatrixI64, diff --git a/src/sketches/countsketch/delta.rs b/src/sketches/countsketch/delta.rs index ac41abb9..b9383c62 100644 --- a/src/sketches/countsketch/delta.rs +++ b/src/sketches/countsketch/delta.rs @@ -7,12 +7,12 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::{add_cells, check_cells}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_SKETCH_DELTA; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::{add_cells, check_cells}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_SKETCH_DELTA; +use crate::asapv1::wire_key::exact_vec; use crate::{DefaultXxHasher, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::wire::{CsWireCounter, CsWireMode, check_geometry, decode_metadata, encode_metadata}; @@ -135,7 +135,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::envelope::hex; + use crate::asapv1::envelope::hex; type Delta = CountSketchDelta; diff --git a/src/sketches/countsketch/wire.rs b/src/sketches/countsketch/wire.rs index 0451ee81..5152075a 100644 --- a/src/sketches/countsketch/wire.rs +++ b/src/sketches/countsketch/wire.rs @@ -26,12 +26,12 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::DeltaCounter; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_SKETCH; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::DeltaCounter; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_SKETCH; +use crate::asapv1::wire_key::exact_vec; use crate::{FastPath, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::{CountSketch, CountSketchCounter}; diff --git a/src/sketches/countsketch_heap/delta.rs b/src/sketches/countsketch_heap/delta.rs index 95f8db9e..c35b012a 100644 --- a/src/sketches/countsketch_heap/delta.rs +++ b/src/sketches/countsketch_heap/delta.rs @@ -8,10 +8,10 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::delta::{add_cells, check_cells}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_SKETCH_WITH_HEAP_DELTA; +use crate::asapv1::delta::{add_cells, check_cells}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_SKETCH_WITH_HEAP_DELTA; use crate::sketches::countminsketch_heap::heap_wire::{ DeltaCells, check_geometry, decode_metadata, decode_payload, encode_metadata, encode_payload, heap_entries, rebuild_heap, wire_key_type, @@ -148,7 +148,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::message_pack_format::envelope::hex; + use crate::asapv1::envelope::hex; use crate::{DataInput, OwnedDataInput}; type Delta = CountSketchWithHeapDelta; diff --git a/src/sketches/countsketch_heap/wire.rs b/src/sketches/countsketch_heap/wire.rs index e7e48dc7..c7a568d8 100644 --- a/src/sketches/countsketch_heap/wire.rs +++ b/src/sketches/countsketch_heap/wire.rs @@ -22,10 +22,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_SKETCH_WITH_HEAP; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_SKETCH_WITH_HEAP; use crate::sketches::countminsketch_heap::heap_wire::{ decode_metadata, decode_payload, encode_metadata, encode_payload, heap_entries, rebuild_heap, wire_key_type, diff --git a/src/sketches/countsketch_l2_norm/wire.rs b/src/sketches/countsketch_l2_norm/wire.rs index 919f2c65..cf8aaade 100644 --- a/src/sketches/countsketch_l2_norm/wire.rs +++ b/src/sketches/countsketch_l2_norm/wire.rs @@ -32,9 +32,9 @@ use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::COUNT_SKETCH_WITH_L2_NORM; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::COUNT_SKETCH_WITH_L2_NORM; use crate::{HashProfile, SketchHasher, Vector1D, Vector2D}; use super::CountSketchWithL2Norm; diff --git a/src/sketches/countsketch_space_saving/wire.rs b/src/sketches/countsketch_space_saving/wire.rs index 57441278..907bf3fd 100644 --- a/src/sketches/countsketch_space_saving/wire.rs +++ b/src/sketches/countsketch_space_saving/wire.rs @@ -5,8 +5,8 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::kind; +use crate::asapv1::admission_rate; +use crate::asapv1::kind; use crate::sketches::countminsketch_space_saving::summary_wire::{ self, DecodedSummaryTopK, SummaryKind, }; diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 8a0b44dd..9466e414 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -19,7 +19,7 @@ use crate::DataInput; use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; use crate::common::structures::Vector1D; -use crate::message_pack_format::admission_rate::{self, admit_all}; +use crate::asapv1::admission_rate::{self, admit_all}; use crate::octo_delta::{DdSign, DdSketchBucketDelta}; use serde::{Deserialize, Serialize}; @@ -205,8 +205,7 @@ impl TryFrom for DdSketch { /// handled without growing the dense store: smaller magnitudes enter the /// zero bucket, while larger magnitudes are rejected. /// -/// Single source of truth shared by core `DdSketch`, the portable wire twin, -/// and tests, so the two implementations cannot drift algebraically again. +/// Single source of truth shared by `DdSketch` and tests. pub fn ddsketch_indexable_bounds(alpha: f64) -> (f64, f64) { let gamma = (1.0 + alpha) / (1.0 - alpha); let inv_log_gamma = 1.0 / gamma.ln(); @@ -526,8 +525,7 @@ impl DdSketch { /// This is a REAL runtime check, not a `debug_assert!`: a `debug_assert!` /// compiles out in release builds, leaving a mismatched merge to corrupt /// results with no signal at all. DataDog's `MergeWith` and sketchlib-go's - /// Go `Merge` both return an error here; the portable `DdSketch::merge` in - /// this same crate does too. + /// Go `Merge` both return an error here. pub fn merge(&mut self, other: &DdSketch) -> Result<(), String> { if (self.alpha - other.alpha).abs() >= 1e-12 || (self.gamma - other.gamma).abs() >= 1e-12 { return Err(format!( diff --git a/src/sketches/ddsketch/delta.rs b/src/sketches/ddsketch/delta.rs index ac1a98fb..d34308d4 100644 --- a/src/sketches/ddsketch/delta.rs +++ b/src/sketches/ddsketch/delta.rs @@ -7,11 +7,11 @@ use std::ops::RangeInclusive; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::check_cells; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::DDSKETCH_DELTA; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::check_cells; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::DDSKETCH_DELTA; use super::wire::{check_alpha, decode_metadata, encode_metadata}; use super::{Buckets, DdSketch, GROW_CHUNK, ddsketch_indexable_bounds}; diff --git a/src/sketches/ddsketch/wire.rs b/src/sketches/ddsketch/wire.rs index 1a584d14..113549a7 100644 --- a/src/sketches/ddsketch/wire.rs +++ b/src/sketches/ddsketch/wire.rs @@ -30,11 +30,11 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use crate::common::structures::Vector1D; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::DDSKETCH; -use crate::message_pack_format::wire_key::exact; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::DDSKETCH; +use crate::asapv1::wire_key::exact; use super::{Buckets, DdSketch, DdSketchState}; diff --git a/src/sketches/elastic/wire.rs b/src/sketches/elastic/wire.rs index bc3ee11d..c6a004c5 100644 --- a/src/sketches/elastic/wire.rs +++ b/src/sketches/elastic/wire.rs @@ -37,10 +37,10 @@ use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::hash::check_matrix_rows; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::ELASTIC; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::ELASTIC; use crate::{CountMinSketch, HashProfile, SketchHasher, Vector2D}; use super::{ElasticSketch, HeavyBucket}; diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 5f25b46a..4a037028 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -33,7 +33,7 @@ // - Refactored into a generic `HyperLogLog` structure. // ---------------------------------------------------------------- -use crate::message_pack_format::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; use crate::structures::fixed_structure::{ HllRegisterStorage, HllRegistersP12, HllRegistersP14, HllRegistersP16, }; diff --git a/src/sketches/hll/delta.rs b/src/sketches/hll/delta.rs index 500f087f..85a984ab 100644 --- a/src/sketches/hll/delta.rs +++ b/src/sketches/hll/delta.rs @@ -7,9 +7,9 @@ use std::marker::PhantomData; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::delta::check_cells; -use crate::message_pack_format::envelope; +use crate::asapv1::admission_rate; +use crate::asapv1::delta::check_cells; +use crate::asapv1::envelope; use crate::structures::fixed_structure::HllRegisterStorage; use crate::{DefaultXxHasher, HashProfile, SketchHasher}; diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index 112a99d7..f2cf0bfd 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -13,10 +13,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use crate::message_pack_format::admission_rate; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::wire_key::exact; +use crate::asapv1::admission_rate; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::wire_key::exact; use crate::structures::fixed_structure::HllRegisterStorage; use crate::{DefaultXxHasher, HashProfile, SketchHasher}; @@ -109,7 +109,6 @@ pub(crate) fn hll_metadata( } /// The standard ProjectASAP profile metadata (the [`DefaultXxHasher`] profile). -/// Used by the portable path, which only represents standard-profile sketches. pub(crate) fn standard_hll_metadata(precision: u32) -> HllMetadata { hll_metadata::(precision, None) } @@ -440,6 +439,86 @@ mod tests { assert_serialization_round_trip::("HllDsP12"); } + /// Known register and HIP state serializes to the `asapv1_golden/` bytes, + /// and decodes back to that state. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + + fn plain( + regs: &[u8], + ) -> HyperLogLogImpl { + HyperLogLogImpl { + registers: registers_from_bytes(regs).expect("registers"), + _marker: PhantomData, + _hasher: PhantomData, + admission_rate: 1.0, + } + } + fn hip(regs: &[u8], kxq0: f64, kxq1: f64, est: f64) -> HyperLogLogHipImpl { + HyperLogLogHipImpl { + registers: registers_from_bytes(regs).expect("registers"), + kxq0, + kxq1, + est, + admission_rate: 1.0, + } + } + + let mut p12 = vec![0u8; 4096]; + for (index, value) in [(0, 1), (1, 7), (100, 42), (4095, 3)] { + p12[index] = value; + } + let mut p14 = vec![0u8; 16384]; + for (index, value) in [(0, 1), (1, 7), (8192, 42), (16383, 51)] { + p14[index] = value; + } + + let classic = hex(include_str!("../../../asapv1_golden/hll_classic_p12.hex")); + let sketch = plain::(&p12); + assert_eq!(sketch.to_asapv1().unwrap(), classic); + let decoded = HyperLogLogImpl::::from_asapv1(&classic).unwrap(); + assert_eq!(decoded.registers.as_slice(), p12.as_slice()); + + let ertl = hex(include_str!("../../../asapv1_golden/hll_ertl_mle_p12.hex")); + let sketch = plain::(&p12); + assert_eq!(sketch.to_asapv1().unwrap(), ertl); + let decoded = HyperLogLogImpl::::from_asapv1(&ertl).unwrap(); + assert_eq!(decoded.registers.as_slice(), p12.as_slice()); + + for (golden, regs, state) in [ + ( + hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")), + &p12, + (1.5, 2.5, 3.0), + ), + ( + hex(include_str!("../../../asapv1_golden/hll_hip_p14.hex")), + &p14, + (16380.5, 0.25, 4.125), + ), + ] { + let (kxq0, kxq1, est) = state; + let encoded = if regs.len() == 4096 { + hip::(regs, kxq0, kxq1, est).to_asapv1().unwrap() + } else { + hip::(regs, kxq0, kxq1, est).to_asapv1().unwrap() + }; + assert_eq!(encoded, golden); + } + let hip_p14 = hex(include_str!("../../../asapv1_golden/hll_hip_p14.hex")); + let decoded = HyperLogLogHipImpl::::from_asapv1(&hip_p14).unwrap(); + assert_eq!(decoded.registers.as_slice(), p14.as_slice()); + assert_eq!( + (decoded.kxq0, decoded.kxq1, decoded.est), + (16380.5, 0.25, 4.125) + ); + let hip_p12 = hex(include_str!("../../../asapv1_golden/hll_hip_p12.hex")); + let decoded = HyperLogLogHipImpl::::from_asapv1(&hip_p12).unwrap(); + assert_eq!(decoded.registers.as_slice(), p12.as_slice()); + assert_eq!((decoded.kxq0, decoded.kxq1, decoded.est), (1.5, 2.5, 3.0)); + } + #[test] fn hll_envelope_structure_and_kind_id_guard() { // ASAPv1 envelope header for an Ertl-MLE sketch: magic, version 0x01, @@ -479,52 +558,6 @@ mod tests { assert_eq!(decoded.est, sketch.est); } - /// Drift guard: the native path and the (deprecated) portable path must emit - /// byte-identical ASAPv1 envelopes. Keep until golden byte-vectors exist. - #[test] - fn native_and_portable_hll_bytes_match() { - use crate::message_pack_format::MessagePackCodec; - use crate::message_pack_format::portable::hll::{HllSketch, HllVariant}; - - // Ertl-MLE / Datafusion: registers-only payload. - let mut native = HyperLogLog::::default(); - for v in 0..1000 { - native.update(&DataInput::U64(v)); - } - let native_bytes = native.to_asapv1().expect("native serialize"); - let portable = HllSketch::from_raw( - HllVariant::Datafusion, - HllRegisters::PRECISION as u32, - native.registers_as_slice().to_vec(), - 0.0, - 0.0, - 0.0, - ); - assert_eq!( - native_bytes, - portable.to_msgpack().expect("portable serialize") - ); - - // HIP: registers + three running scalars. - let mut hip = HyperLogLogHip::default(); - for v in 0..1000 { - hip.update(&DataInput::U64(v)); - } - let hip_bytes = hip.to_asapv1().expect("native serialize"); - let portable_hip = HllSketch::from_raw( - HllVariant::Hip, - HllRegisters::PRECISION as u32, - hip.registers.as_slice().to_vec(), - hip.kxq0, - hip.kxq1, - hip.est, - ); - assert_eq!( - hip_bytes, - portable_hip.to_msgpack().expect("portable serialize") - ); - } - // A test-only custom hasher: it hashes exactly like `DefaultXxHasher` // (delegation) but declares a DIFFERENT `HashProfile` (distinct profile id // and seed list). Serialization is derived from the profile, so an diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index e72de1b8..e2e1c205 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -331,9 +331,8 @@ impl Kll { s } - /// Reconstruct a KLL directly from portable wire state — the retained - /// `items` in level order plus the `levels` boundary array (proto contract: - /// `levels[0] == 0`, `levels[num_levels] == items.len()`). Avoids replaying + /// Reconstruct a KLL directly from the retained `items` in level order + /// plus the `levels` boundary array (`levels[0] == 0`, `levels[num_levels] == items.len()`). Avoids replaying /// every item through `update()`: it places the items straight into the /// internal buffer and fixes up the level boundaries, so the result is /// bit-identical to the source sketch (identical quantiles) at a fraction @@ -781,12 +780,8 @@ impl Kll { self.num_levels as u32 } - /// Returns the compaction-coin state in `sketchlib-go::CoinState` - /// shape: `(state, bit_cache, remaining_bits)`. Wire-format wrappers - /// pack this directly into the `KLLState.coin` proto field; without - /// it the coin state would have to be poked from private internals - /// or reconstructed by a serde round-trip, neither of which is - /// stable across `asap_sketchlib` releases. + /// Returns the compaction-coin state as + /// `(state, bit_cache, remaining_bits)`. #[inline] pub fn coin_state(&self) -> (u64, u64, u32) { ( @@ -800,10 +795,8 @@ impl Kll { /// length `num_levels + 1`, starting at 0 and ending at the total /// number of retained items. The chunk /// `items_top_down()[level_bounds_top_down()[i] .. level_bounds_top_down()[i+1]]` is the - /// **top-most-first** run for the proto's `KLLState.levels`/`items` - /// fields, matching `sketchlib-go::KLLSketch.SerializePortable`. - /// See the `KLLState` docstring in `proto/kll/kll.proto`: index `i` - /// in `levels` maps to compactor level `num_levels - 1 - i` in the + /// **top-most-first** run of the ASAPv1 KLL payload's `levels`/`items`: + /// index `i` in `levels` maps to compactor level `num_levels - 1 - i` in the /// in-memory representation. pub fn level_bounds_top_down(&self) -> Vec { // Walk from top compactor-level downward, accumulating sizes. @@ -1180,7 +1173,7 @@ mod tests { ); } - // Direct reconstruction from portable wire state must be BIT-EXACT: + // Direct reconstruction from level state must be BIT-EXACT: // identical quantiles to the source sketch (unlike a lossy // replay-through-`update()` reconstruction). #[test] @@ -1190,7 +1183,7 @@ mod tests { for i in 0..200_000u32 { src.update(&((i as f64) * 0.0007 + 3.0)); } - // Extract the compacted portable form (proto contract: levels[0]==0, + // Extract the compacted form (levels[0]==0, // levels[num_levels]==items.len()). let off = src.levels[0]; let items: Vec = src.items[off..src.max_capacity].to_vec(); diff --git a/src/sketches/kll/wire.rs b/src/sketches/kll/wire.rs index 7cbe63c9..abf4cc1f 100644 --- a/src/sketches/kll/wire.rs +++ b/src/sketches/kll/wire.rs @@ -30,9 +30,9 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use crate::common::numerical::NumericalValue; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::wire_key::exact_vec; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::wire_key::exact_vec; use super::{ CAPACITY_CACHE_LEN, Coin, Kll, MAX_K, MAX_LEVELS, checked_weighted_count, compute_max_capacity, diff --git a/src/sketches/kll_dynamic/wire.rs b/src/sketches/kll_dynamic/wire.rs index 179e0969..82517765 100644 --- a/src/sketches/kll_dynamic/wire.rs +++ b/src/sketches/kll_dynamic/wire.rs @@ -15,8 +15,8 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use crate::Vector1D; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; +use crate::asapv1::envelope; +use crate::asapv1::kind; use crate::sketches::kll::{ Coin, KllCoinWire, KllPayload, KllWireItem, kll_metadata, split_and_validate_meta, validate_kll_payload, @@ -96,8 +96,8 @@ where #[cfg(test)] mod tests { - use crate::message_pack_format::envelope; - use crate::message_pack_format::kind; + use crate::asapv1::envelope; + use crate::asapv1::kind; use crate::sketches::kll_dynamic::KllDynamic; fn build_dynamic(k: i32, n: u64) -> KllDynamic { diff --git a/src/sketches/kmv/wire.rs b/src/sketches/kmv/wire.rs index ff205c35..b318d769 100644 --- a/src/sketches/kmv/wire.rs +++ b/src/sketches/kmv/wire.rs @@ -28,9 +28,9 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::KMV; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::KMV; use crate::{CommonHeap, HashProfile, RetainSmallest, SketchHasher}; use super::Kmv; diff --git a/src/sketches/mod.rs b/src/sketches/mod.rs index ddb6a47b..8539d027 100644 --- a/src/sketches/mod.rs +++ b/src/sketches/mod.rs @@ -78,6 +78,10 @@ pub use countminsketch::{CountMinSketch, CountMinSketchDelta, QUICKSTART_COLS, Q pub mod elastic; pub use elastic::ElasticSketch; + +/// An exact set of string keys and its delta. +pub mod set_aggregator; +pub use set_aggregator::{SetAggregator, SetAggregatorDelta}; pub use elastic::HeavyBucket; /// HyperLogLog implementations and aliases. diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/sketches/set_aggregator.rs similarity index 51% rename from src/message_pack_format/portable/set_aggregator.rs rename to src/sketches/set_aggregator.rs index ba93676b..e8e23518 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/sketches/set_aggregator.rs @@ -1,17 +1,15 @@ -//! Set aggregator: wire-format type for tracking a set of unique string keys. +//! Set aggregator: an exact set of string keys. //! -//! `SetAggregator` is not a sketch — it's a plain `HashSet` wrapped -//! for cross-language interop. `to_asapv1` writes ASAPv1 (§3.21); -//! [`MessagePackCodec`] writes `StringSet { values: HashSet }`. +//! `SetAggregator` is not a sketch — it is a plain `HashSet` with an +//! ASAPv1 form (§3.21); [`SetAggregatorDelta`] is its delta (§3.22). use std::collections::HashSet; -use serde::{Deserialize, Serialize}; - -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; - +mod delta; mod wire; +pub use delta::SetAggregatorDelta; + /// Set aggregator for tracking a set of unique string keys. #[derive(Debug, Clone)] pub struct SetAggregator { @@ -62,41 +60,9 @@ impl Default for SetAggregator { } } -// ----- Wire format ----- - -/// Borrowed serialize-side wire DTO. Avoids cloning the underlying set -/// on the encode path. -#[derive(Serialize)] -pub(crate) struct StringSetRef<'a> { - pub values: &'a HashSet, -} - -/// Owned deserialize-side wire DTO. -#[derive(Deserialize)] -pub(crate) struct StringSetOwned { - pub values: HashSet, -} - -impl MessagePackCodec for SetAggregator { - fn to_msgpack(&self) -> Result, MsgPackError> { - let wrapper = StringSetRef { - values: &self.values, - }; - Ok(rmp_serde::to_vec(&wrapper)?) - } - - fn from_msgpack(bytes: &[u8]) -> Result { - let wrapper: StringSetOwned = rmp_serde::from_slice(bytes)?; - Ok(Self { - values: wrapper.values, - }) - } -} - #[cfg(test)] mod tests { use super::*; - use serde::Deserialize; #[test] fn test_creation() { @@ -131,32 +97,4 @@ mod tests { assert!(sa1.values.contains("api")); assert!(sa1.values.contains("db")); } - - #[test] - fn test_msgpack_round_trip() { - let mut sa = SetAggregator::new(); - sa.update("web"); - sa.update("api"); - - let bytes = sa.to_msgpack().unwrap(); - let deserialized = SetAggregator::from_msgpack(&bytes).unwrap(); - - assert_eq!(deserialized.values.len(), 2); - assert!(deserialized.values.contains("web")); - assert!(deserialized.values.contains("api")); - } - - #[test] - fn test_msgpack_matches_wire_format() { - #[derive(Deserialize)] - struct StringSet { - values: HashSet, - } - let mut sa = SetAggregator::new(); - sa.update("a"); - let bytes = sa.to_msgpack().unwrap(); - let decoded: StringSet = - rmp_serde::from_slice(&bytes).expect("should decode as StringSet { values: ... }"); - assert!(decoded.values.contains("a")); - } } diff --git a/src/sketches/set_aggregator/delta.rs b/src/sketches/set_aggregator/delta.rs new file mode 100644 index 00000000..94d4d958 --- /dev/null +++ b/src/sketches/set_aggregator/delta.rs @@ -0,0 +1,11 @@ +//! The delta of a [`super::SetAggregator`] between two snapshots. + +use std::collections::HashSet; + +/// The string keys added to and removed from a set between two consecutive +/// observations (ASAPv1 kind `08 80`, §3.22). +#[derive(Debug, Clone)] +pub struct SetAggregatorDelta { + pub added: HashSet, + pub removed: HashSet, +} diff --git a/src/message_pack_format/portable/set_aggregator/wire.rs b/src/sketches/set_aggregator/wire.rs similarity index 97% rename from src/message_pack_format/portable/set_aggregator/wire.rs rename to src/sketches/set_aggregator/wire.rs index 152ac16b..046a034c 100644 --- a/src/message_pack_format/portable/set_aggregator/wire.rs +++ b/src/sketches/set_aggregator/wire.rs @@ -10,12 +10,11 @@ use std::collections::HashSet; use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::portable::delta_set_aggregator::SetAggregatorDelta; -use crate::message_pack_format::wire_key::WireString; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::wire_key::WireString; -use super::SetAggregator; +use super::{SetAggregator, SetAggregatorDelta}; /// The metadata map both kinds carry, a msgpack **map** (`to_vec_named`). #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/sketches/space_saving/wire.rs b/src/sketches/space_saving/wire.rs index b0e94d8b..2bc2e449 100644 --- a/src/sketches/space_saving/wire.rs +++ b/src/sketches/space_saving/wire.rs @@ -46,10 +46,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::SPACE_SAVING; -use crate::message_pack_format::wire_key::{WireBytes, WireString, exact_vec}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::SPACE_SAVING; +use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; use crate::{HashProfile, OwnedDataInput, SketchHasher}; use std::collections::HashSet; diff --git a/src/sketches/uniform/wire.rs b/src/sketches/uniform/wire.rs index ef242f8f..43bc7b9a 100644 --- a/src/sketches/uniform/wire.rs +++ b/src/sketches/uniform/wire.rs @@ -25,10 +25,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::envelope; -use crate::message_pack_format::kind; -use crate::message_pack_format::kind::UNIFORM_SAMPLING; -use crate::message_pack_format::wire_key::{exact, exact_vec}; +use crate::asapv1::envelope; +use crate::asapv1::kind; +use crate::asapv1::kind::UNIFORM_SAMPLING; +use crate::asapv1::wire_key::{exact, exact_vec}; use super::{SampleEntry, UniformSampling}; diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index 352bfd1d..8d6f34a5 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -56,7 +56,7 @@ Compared against: a `(row, col)` scan of `table[i][j].full_key` / `.val`, and th - wire - ASAPv1 keeps `estimate` for 64 keys; `edge_dimension` x `edge_rows` -## count_min.rs — 19 +## count_min.rs — 11 Compared against: an exact `HashMap` count, columns via the public `hash_for_matrix`. Path laws compare two real implementations. @@ -68,22 +68,14 @@ Compared against: an exact `HashMap` count, columns via the public `hash_for_mat - estimator - `estimate(k)` never drops on a non-negative insert; all 32 keys probed after every insert - `truth(k) <= estimate(k) <= truth(k) + min over rows of colliding mass`; 1-8 cols x 32 keys, every row collides - - portable `CountMinSketch`: `estimate(k) >= truth(k)` -- merge - - commutative, associative, empty is identity - - `merge(S(xs), S(ys)) == S(xs ++ ys)` - - `merge` never lowers `estimate(k)`; `k` from either stream or from neither - `merge_max` - disjoint keys: `max(a, b) <= cell <= a + b`; an empty peer cell moves nothing; `estimate(k)` still inside the band above - `merge_max(s, s) == s` - shared keys: `max(a, b) <= cell <= a + b`, and the same for every estimate -- model across a rebuilt backend - - after any sequence of insert, merge, `apply_delta`, msgpack round trip, mismatched merge: geometry unchanged; every cell `>= 0`; every row sums to total mass; `truth(k) <= estimate(k) <= band`. `apply_delta == merge`; a mismatched merge is refused and changes nothing - wire - - msgpack keeps geometry, cells, estimates - ASAPv1 keeps 64 estimates; `edge_rows` x `edge_dimension` -## count_sketch.rs — 12 +## count_sketch.rs — 8 Compared against: a key's `(col, sign)` per row, read off a one-key sketch; the stream's exact signed sum. Regime: 512 keys; sign law 1-5 rows x 1-7 cols, median law 1-7 rows x 1-5 cols. @@ -97,33 +89,25 @@ Compared against: a key's `(col, sign)` per row, read off a one-key sketch; the - linearity - insert `w` then `-w` for every `w`: cells back to start; signed stream over a background - `update_weighted(k, -1)` undoes `update(k)`, cell for cell - - `cells(S(xs)) + cells(S(ys)) == cells(S(xs ++ ys))`; integral weights, power-of-two widths - - negate every weight: every cell negated -- merge - - portable `CountSketch`: `merge(S(xs), S(ys)) == S(xs ++ ys)` to `REL_TOL` - wire - - msgpack keeps cells and estimates - ASAPv1 keeps 64 estimates; `edge_rows` x `edge_dimension` -## ddsketch.rs — 14 +## ddsketch.rs — 10 -Compared against: the sorted stream. `DdSketch::get_value_at_quantile` targets the `ceil(q*n)`-th value, portable `DdSketch::quantile` the `floor(q*(n-1))`-th. Streams carry negatives, zeros and positives inside the indexable band. +Compared against: the sorted stream. `DdSketch::get_value_at_quantile` targets the `ceil(q*n)`-th value. Streams carry negatives, zeros and positives inside the indexable band. - placement - negative store, zero counter, positive store hold exactly the three class counts; `get_count() == arrivals` - - portable `DdSketch`: the same three counts - `sum() == stream sum`, bit for bit - - first dense allocation: one 128-bucket chunk, seed value at its center; both impls, both signed stores + - first dense allocation: one 128-bucket chunk, seed value at its center; both signed stores - relative error `<= alpha` - - `|quantile(q) - exact| <= alpha * |exact|`, across the sign boundary, both impls + - `|quantile(q) - exact| <= alpha * |exact|`, across the sign boundary - `quantile(q)` non-decreasing in `q`, across the sign boundary - single value `v`: `quantile(q) == v` for every `q`; `min`, `max`, `q=0`, `q=1` equal the tracked extremes - merge - - counts add; both stores, both offsets, zero counter identical either way round - merged `quantile(q)` within `alpha` of the combined stream's exact value - - different `alpha`: refused, receiver count unchanged, both impls; same `alpha`: merges + - different `alpha`: refused, receiver count unchanged; same `alpha`: merges - wire - - msgpack keeps stores, offsets, zero count, quantiles - ASAPv1 keeps count, `alpha`, `sum`, quantiles; stream of NaN, ±inf, ±0, f64 extremes ## elastic.rs — 12 @@ -228,7 +212,7 @@ Compared against: the offers sorted and truncated. `TopKHeap` is path dependent - one offer per key: the `k` largest counts resident - `update` returns true iff the key was resident or there was room, i.e. nothing was displaced -## hll.rs — 101 +## hll.rs — 97 Models use only what the paper fixes; register indexing and rank mapping stay free. For `Classic` and `Ertl`, `estimate` is a function of the registers, so laws relate estimates rather than restate the formula. Registers `<= 40`, so `sum(2^-M[j])` is exact in f64. @@ -247,17 +231,15 @@ Models use only what the paper fixes; register indexing and rank mapping stay fr - `merge == elementwise max` - commutative, idempotent, empty is identity - `merge(S(xs), S(ys)) == S(xs ++ ys)` -- HIP, per precision (20); registers read back through the wire format +- HIP, per precision (20); registers read back from the ASAPv1 payload - empty: `estimate() == 0` - `estimate` non-decreasing; a repeated key adds 0 - order matters: a descending run into one register estimates exactly 1, an ascending run `> 1` - an arrival that raises a register adds `m / sum(2^-M[j])` over the prior registers; one that raises nothing adds 0; both sides exact in f64, compared after truncation to `usize` -- portable `HllSketch` (5) - - merge commutative, idempotent, empty identity, `== S(xs ++ ys)`; precisions 4-11 - - msgpack keeps variant, precision, registers, estimate; `Regular`, `Datafusion`, `Hip`; precisions 4-13 +- wire - ASAPv1 on `HyperLogLog` keeps registers and estimate; empty included -## hydra.rs — 7 +## hydra.rs — 5 Compared against: the map rebuilt from the subkey encoding (`label ":" value`, joined by `";"`, escaped) and the public matrix hash at `HYDRA_SEED_INDEX`. All records share one payload value, so each cell's Count-Min is exact. Labels and values include the escaped characters. Cols in {1, 2, 3, 8, 64, 251, 256}. @@ -268,13 +250,10 @@ Compared against: the map rebuilt from the subkey encoding (`label ":" value`, j - an answer is a median of the subkey's cells: at least half the rows `<=` it and at least half `>=` it. Probes: every subset of every record, every single-column equality, one absent value - merge - `merge(S(xs), S(ys)) == S(xs ++ ys)`, cell for cell -- `HydraKllSketch`, no schema, no fan-out - - one key hits one cell per row at `xxh32(key, row) % cols`, read back as retained count - - query == median of its row cells, as an order statistic; an absent key reads 0 - wire - ASAPv1 keeps every subpopulation answer and the schema -## kll.rs — 14 +## kll.rs — 13 Compaction is randomized, so no law names retained items. Compared against: the sorted stream. Rank band `16 * n / k`; worst drift seen about 3.3. The accuracy law draws `k` from {200, 400} and asserts band `< n/10`. @@ -296,7 +275,6 @@ Compaction is randomized, so no law names retained items. Compared against: the - same seed, same stream: same items - `clear` re-seeds from the stored seed: replay == fresh, items and levels - wire - - msgpack keeps `k`, count, quantiles - ASAPv1 keeps count and quantiles under NaN, ±inf, ±0, f64 extremes ## kll_dynamic.rs — 14 @@ -393,7 +371,7 @@ Exact. Compared against: a `HashSet` of the stream. Keys from `[a-c]{0,2}`, so s - empty is a two-sided identity - `merge_refs` unions all inputs; refuses an empty list - wire - - msgpack keeps the key set, and again on a second round trip; keys include `""`, non-ASCII, embedded NUL, 300 chars, arbitrary `String` + - ASAPv1 keeps the key set, and again on a second round trip; keys include `""`, non-ASCII, embedded NUL, 300 chars, arbitrary `String` ## space_saving.rs — 19 diff --git a/tests/README.md b/tests/README.md index f030accc..15a5f025 100644 --- a/tests/README.md +++ b/tests/README.md @@ -23,7 +23,7 @@ its theorem. | `e2e/matrix_instances.rs` | Every built-in `(storage, hashing path)` instance of CountMinSketch / Count / CountMinSketchWithHeap / CountSketchWithHeap, plus counter-width edges | | `e2e/numeric_types.rs` | Every `NumericalValue` type through `Kll`, `KllDynamic` and `DdSketch::add` | | `e2e/windows.rs` | Every `EhBucketSketch` variant in an `ExponentialHistogram`, every `TumblingWindow` payload, and `MicroCm`'s in-cell sliding window | -| `e2e/composition.rs` | `HashSketchEnsemble`, `UnivMonQ`'s config surface, and the portable sketch+heap facade | +| `e2e/composition.rs` | `HashSketchEnsemble` and `UnivMonQ`'s config surface | | `e2e/nitro.rs` | Every Nitro ingestion path — row-level `CountMinSketch`/`CountSketch`, `NitroBatch::update_batch` / `update_batch_cached_step`, and the bare `Vector2D` target — under one sampling model, plus merge, seeding, saturation and serde/context continuation. **All Nitro E2E behaviour lives here**; the sampler's own structural tests are unit tests in `src/common/structure_utils.rs` and `src/sketch_framework/nitro.rs`, where the cursor and skip counter are visible without a public accessor | | `e2e/frameworks.rs` | Hydra's subpopulation lattice and UnivMon composition | | `e2e/heavy_hitters.rs` | Space-Saving's error sandwich, `min_count` ceiling and Stream-Summary lists under sustained eviction, plus CocoSketch's exact mass partitions and multi-seed unbiasedness, and Elastic through the batteries and the heavy-hitter properties no battery models | diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 3c46c343..67c38d1e 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -11,9 +11,6 @@ use std::collections::HashSet; use asap_sketchlib::common::input::HydraCounter; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::hll::HllSketch; -use asap_sketchlib::message_pack_format::portable::hll::HllVariant; use asap_sketchlib::sketches::CountSketchDelta; use asap_sketchlib::sketches::hll::HllWireVariant; use asap_sketchlib::{ @@ -154,96 +151,38 @@ const CS_VALS: [[i64; 4]; 2] = [[0, 127, 128, 65536], [-1, -33, -32768, -2147483 const CS_F64_VALS: [[f64; 4]; 2] = [[0.0, -0.0, 1.5, -2.25], [127.0, -32768.5, 0.125, -1e300]]; // --------------------------------------------------------------------------- -// HLL: build known state -> serialize == golden, and golden round-trips. +// HLL: golden decodes to the known state and round-trips. Encoding the known +// state is checked in `src/sketches/hll/wire.rs`, which can build it. // --------------------------------------------------------------------------- #[test] fn hll_classic_p12_matches_golden() { let want = decode_hex(GOLDEN_CLASSIC); - let regs = p12_registers(); - - // Build known state and serialize. - let got = HllSketch::from_raw(HllVariant::Regular, 12, regs.clone(), 0.0, 0.0, 0.0) - .to_msgpack() - .expect("serialize"); - assert_eq!(got, want, "Classic P12 bytes diverge from golden"); - - // Native wire path (src/sketches/hll.rs) round-trips the golden identically. let native = HyperLogLogP12::::from_asapv1(&want).expect("native decode"); - assert_eq!(native.registers_as_slice(), regs.as_slice()); + assert_eq!(native.registers_as_slice(), p12_registers().as_slice()); assert_eq!(native.to_asapv1().expect("re-serialize"), want); - - // Portable decode round-trips to the same known state. - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.precision, 12); - assert_eq!(decoded.variant, HllVariant::Regular); } #[test] fn hll_ertl_mle_p12_matches_golden() { let want = decode_hex(GOLDEN_ERTL); - let regs = p12_registers(); - - let got = HllSketch::from_raw(HllVariant::Datafusion, 12, regs.clone(), 0.0, 0.0, 0.0) - .to_msgpack() - .expect("serialize"); - assert_eq!(got, want, "Ertl-MLE P12 bytes diverge from golden"); - let native = HyperLogLogP12::::from_asapv1(&want).expect("native decode"); - assert_eq!(native.registers_as_slice(), regs.as_slice()); + assert_eq!(native.registers_as_slice(), p12_registers().as_slice()); assert_eq!(native.to_asapv1().expect("re-serialize"), want); - - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.variant, HllVariant::Datafusion); } #[test] fn hll_hip_p12_matches_golden() { let want = decode_hex(GOLDEN_HIP); - let regs = p12_registers(); - - let got = HllSketch::from_raw(HllVariant::Hip, 12, regs.clone(), 1.5, 2.5, 3.0) - .to_msgpack() - .expect("serialize"); - assert_eq!(got, want, "HIP P12 bytes diverge from golden"); - - // Native HIP wire path round-trips the golden identically. let native = HyperLogLogHipP12::from_asapv1(&want).expect("native decode"); assert_eq!(native.to_asapv1().expect("re-serialize"), want); - - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.variant, HllVariant::Hip); - assert_eq!(decoded.hip_kxq0, 1.5); - assert_eq!(decoded.hip_kxq1, 2.5); - assert_eq!(decoded.hip_est, 3.0); } #[test] fn hll_hip_p14_matches_golden() { let want = decode_hex(GOLDEN_HIP_P14); - let mut regs = vec![0u8; 16384]; - for (index, value) in [(0, 1), (1, 7), (8192, 42), (16383, 51)] { - regs[index] = value; - } - - let got = HllSketch::from_raw(HllVariant::Hip, 14, regs.clone(), 16380.5, 0.25, 4.125) - .to_msgpack() - .expect("serialize"); - assert_eq!(got, want, "HIP P14 bytes diverge from golden"); - let native = HyperLogLogHipP14::from_asapv1(&want).expect("native decode"); assert_eq!(native.to_asapv1().expect("re-serialize"), want); - - let decoded = HllSketch::from_msgpack(&want).expect("decode"); - assert_eq!(decoded.registers, regs); - assert_eq!(decoded.precision, 14); - assert_eq!(decoded.variant, HllVariant::Hip); - assert_eq!(decoded.hip_kxq0, 16380.5); - assert_eq!(decoded.hip_kxq1, 0.25); - assert_eq!(decoded.hip_est, 4.125); } // --------------------------------------------------------------------------- diff --git a/tests/bug_verification.rs b/tests/bug_verification.rs index fa54cb58..742b7982 100644 --- a/tests/bug_verification.rs +++ b/tests/bug_verification.rs @@ -10,51 +10,7 @@ //! in `tests/e2e/nitro.rs`, which names both failure modes on the same run //! that already exercises every ingestion path. -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::{CountSketchWithL2Norm, DataInput, DdSketch, DefaultXxHasher}; - -// --------------------------------------------------------------------------- -// Bug 3: portable DdSketch reports gamma^(k+0.5) (bucket log-midpoint) as the -// representative (portable/ddsketch.rs:426). For values sitting at the lower -// edge of bucket k the relative error is sqrt(gamma)-1 ~= alpha + alpha^2/2, -// violating the advertised relative-accuracy guarantee. Core DDSketch already -// uses gamma^k * (1 + alpha) (ddsketch.rs:383-392), so the two disagree on -// identical data. -// -// Synthetic data: value gamma^k * (1 + 1e-6) repeated 10_000x, alpha = 0.05. -// Truth for every quantile: the value itself. Current behavior: 5.13% error. -// --------------------------------------------------------------------------- -#[test] -fn portable_ddsketch_respects_alpha_at_bucket_edges() { - let alpha = 0.05; - let gamma = (1.0f64 + alpha) / (1.0 - alpha); - let v = gamma.powi(20) * (1.0 + 1e-6); - - let mut port = PortableDds::new(alpha); - let mut core = DdSketch::new(alpha); - for _ in 0..10_000 { - port.update(v); - core.add(&v); - } - - let q_port = port.quantile(0.5).unwrap(); - let rel_port = ((q_port - v) / v).abs(); - assert!( - rel_port <= alpha * (1.0 + 1e-6), - "portable DdSketch median off by {rel_port:.5} (> alpha={alpha}) at a \ - bucket-edge value" - ); - - // Both implementations must sit within alpha of the truth on identical - // data. (Exact equality is impossible: core clamps representatives to its - // observed min/max, which the portable wire format does not carry.) - let q_core = core.get_value_at_quantile(0.5).unwrap(); - let rel_core = ((q_core - v) / v).abs(); - assert!( - rel_core <= alpha * (1.0 + 1e-6), - "core DDSketch median off by {rel_core:.5} (> alpha={alpha})" - ); -} +use asap_sketchlib::{CountSketchWithL2Norm, DataInput, DefaultXxHasher}; // --------------------------------------------------------------------------- // Bug 4: CountSketchWithL2Norm's hot-path L2 accumulation ran in plain i64 diff --git a/tests/common/conformance.rs b/tests/common/conformance.rs index 595dbf10..b2b046af 100644 --- a/tests/common/conformance.rs +++ b/tests/common/conformance.rs @@ -430,9 +430,7 @@ where /// relative-value band says nothing about KLL's. /// /// The order statistic compared against comes from the spec's own -/// `DdRankConvention`, so a core `DdSketch` (`ceil(q*n)`) and the portable -/// `DdSketch` (`floor(q*(n-1))`) are each scored on the question they actually -/// answer. +/// `DdRankConvention`. pub fn relative_quantile_battery( sketch: &str, new_sketch: F, diff --git a/tests/common/specs.rs b/tests/common/specs.rs index df96b27a..975b669c 100644 --- a/tests/common/specs.rs +++ b/tests/common/specs.rs @@ -996,21 +996,14 @@ pub fn breakpoint_rank_interval_distance(estimated: &[(f64, f64)], sorted_truth: // DDSketch: relative value error // --------------------------------------------------------------------------- -/// Which order statistic a DDSketch implementation answers a quantile query -/// with. The two shipped implementations do not agree, and the truth a test -/// compares against has to follow the implementation it is testing. +/// Which order statistic `DdSketch` answers a quantile query with; the truth a +/// test compares against follows it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DdRankConvention { /// `DdSketch::get_value_at_quantile`: `rank = ceil(q * n)`, 1-based, so the /// answer is `sorted[ceil(q*n) - 1]`. `q = 0` and `q = 1` short-circuit to /// the exact stored minimum and maximum. CeilNearestRank, - /// Portable `DdSketch::quantile`: `target = floor(q * (n - 1))`, 0-based, - /// so the answer is `sorted[floor(q*(n-1))]` — the lower-quantile - /// convention of the DDSketch paper and of DataDog's reference - /// implementation. No exact minimum or maximum is retained, so the - /// endpoints are bucket representatives like any other rank. - LowerFloor, } impl DdRankConvention { @@ -1024,7 +1017,6 @@ impl DdRankConvention { let q = q.clamp(0.0, 1.0); match self { DdRankConvention::CeilNearestRank => (((q * n as f64).ceil() as usize).clamp(1, n)) - 1, - DdRankConvention::LowerFloor => ((q * (n - 1) as f64).floor() as usize).min(n - 1), } } @@ -1036,7 +1028,6 @@ impl DdRankConvention { pub fn name(self) -> &'static str { match self { DdRankConvention::CeilNearestRank => "ceil(q*n) nearest-rank", - DdRankConvention::LowerFloor => "floor(q*(n-1)) lower-quantile", } } } @@ -1045,8 +1036,7 @@ impl DdRankConvention { /// /// DDSketch's guarantee is on the *value*, not the rank: the returned estimate /// for the `q`-quantile must be within `alpha` relative error of the exact -/// order statistic **at the same `q` under that implementation's own rank -/// convention**: +/// order statistic **at the same `q` under the sketch's rank convention**: /// /// ```text /// |est - true| / |true| <= alpha + numerical_slack(true) @@ -1081,14 +1071,6 @@ impl RelativeQuantileSpec { } } - /// A spec for the portable `DdSketch`. - pub fn portable(alpha: f64) -> Self { - Self { - alpha, - convention: DdRankConvention::LowerFloor, - } - } - /// A few ULP of headroom, scaled by the magnitude of the logarithm because /// that is what `powf`/`ln` round-off actually tracks. /// diff --git a/tests/conformance_kit.rs b/tests/conformance_kit.rs index b82dd286..cfb5842d 100644 --- a/tests/conformance_kit.rs +++ b/tests/conformance_kit.rs @@ -10,8 +10,6 @@ use common::conformance::{ }; use common::{FreqTruth, zipf_u64}; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::{ Bloom, CountMinSketch, CountMinSketchFold, CountMinSketchWithHeap, CountSketchFold, CountSketchWithHeap, CountSketchWithL2Norm, DataInput, DdSketch, FastPath, HyperLogLog, @@ -132,17 +130,6 @@ impl CardinalityOps for HllHipAdapter { } } -struct PortableHllAdapter(HllSketch); - -impl CardinalityOps for PortableHllAdapter { - fn ingest(&mut self, key: &[u8]) { - self.0.update(key); - } - fn estimate(&self) -> f64 { - self.0.estimate() - } -} - struct KllAdapter(Kll); impl QuantileOps for KllAdapter { @@ -176,17 +163,6 @@ impl QuantileOps for KllDynamicAdapter { } } -struct PortableDdsAdapter(PortableDds); - -impl QuantileOps for PortableDdsAdapter { - fn update(&mut self, value: f64) { - self.0.update(value); - } - fn quantile(&self, q: f64) -> f64 { - self.0.quantile(q).expect("non-empty sketch") - } -} - struct UnivMonQAdapter(UnivMonQ); impl QuantileOps for UnivMonQAdapter { @@ -313,15 +289,6 @@ fn hll_variants_pass_cardinality_conformance() { spec, ) .assert_ok(); - - conformance::cardinality_battery( - "portable HllSketch", - || PortableHllAdapter(HllSketch::new(HllVariant::Regular, 14)), - &unique, - 100_000, - spec, - ) - .assert_ok(); } /// Compaction-coin seed for the KLL adapters. The unseeded `with_k` @@ -365,26 +332,6 @@ fn kll_family_passes_quantile_conformance() { .assert_ok(); } -/// DDSketch through the **relative-value-error** battery, against exact -/// nearest-rank order statistics — the guarantee it actually makes. -#[test] -fn ddsketch_passes_relative_quantile_conformance() { - const ALPHA: f64 = 0.01; - let values: Vec = common::normal_f64(40_000, 500.0, 80.0, 7001) - .into_iter() - .filter(|v| *v > 0.0) - .collect(); - - conformance::relative_quantile_battery( - "PortableDds", - || PortableDdsAdapter(PortableDds::new(ALPHA)), - &values, - common::specs::RelativeQuantileSpec::portable(ALPHA), - &conformance::DEFAULT_QUANTILE_QS, - ) - .assert_ok(); -} - #[test] fn univmonq_passes_quantile_conformance() { let values: Vec = common::uniform_u64(30_000, 50_000, 7002) diff --git a/tests/ddsketch_signed.rs b/tests/ddsketch_signed.rs index a14c8d25..9413fe0a 100644 --- a/tests/ddsketch_signed.rs +++ b/tests/ddsketch_signed.rs @@ -1,40 +1,26 @@ use asap_sketchlib::DdSketch; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDdSketch; -// Both public implementations must count negative and zero observations. +// Negative and zero observations are counted. #[test] fn signed_samples_and_zero_are_retained() { - let mut portable = PortableDdSketch::new(0.01); let mut core = DdSketch::new(0.01); for v in [-20.0, -2.0, 0.0, 0.0, 2.0, 20.0] { - portable.update(v); core.add(&v); } - assert_eq!(portable.total_count(), 6); assert_eq!(core.get_count(), 6); - assert_eq!(portable.quantile(0.5), Some(0.0)); assert_eq!(core.get_value_at_quantile(0.5), Some(0.0)); - assert!(portable.quantile(0.0).unwrap() < -19.0); - assert!(portable.quantile(1.0).unwrap() > 19.0); } -// Signed snapshots must survive both codecs and merge with positive-only state. +// Signed snapshots survive ASAPv1 and merge with positive-only state. #[test] fn signed_merge_and_serialization() { - let mut p = PortableDdSketch::new(0.01); let mut c = DdSketch::new(0.01); for v in [-4.0, 0.0, 4.0] { - p.update(v); c.add(&v); } - let mut p2 = PortableDdSketch::from_msgpack(&p.to_msgpack().unwrap()).unwrap(); let mut c2 = DdSketch::from_asapv1(&c.to_asapv1().unwrap()).unwrap(); - p2.merge(&p).unwrap(); c2.merge(&c).unwrap(); - assert_eq!(p2.total_count(), 6); assert_eq!(c2.get_count(), 6); - assert_eq!(p2.quantile_interpolated(0.5), Some(0.0)); assert_eq!(c2.get_value_at_quantile(0.5), Some(0.0)); } @@ -50,44 +36,32 @@ fn signed_order_statistics_obey_relative_bound() { }) .collect(); values.sort_by(f64::total_cmp); - let mut portable = PortableDdSketch::new(0.01); let mut core = DdSketch::new(0.01); for v in &values { - portable.try_update(*v).unwrap(); core.try_add(v).unwrap(); } for i in 0..=1000 { let q = i as f64 / 1000.; - let pi = (q * (values.len() - 1) as f64).floor() as usize; let ci = ((q * values.len() as f64).ceil() as usize).max(1) - 1; - for (got, want) in [ - (portable.quantile(q).unwrap(), values[pi]), - (core.get_value_at_quantile(q).unwrap(), values[ci]), - ] { - assert!( - (got - want).abs() <= 0.010000001 * want.abs(), - "q={q}, got={got}, want={want}" - ); - } + let (got, want) = (core.get_value_at_quantile(q).unwrap(), values[ci]); + assert!( + (got - want).abs() <= 0.010000001 * want.abs(), + "q={q}, got={got}, want={want}" + ); } } #[test] fn zero_only_special_values_and_octo_flush() { use asap_sketchlib::sketch_framework::octo::DdSketchWorker; - let mut portable = PortableDdSketch::new(0.01); let mut core = DdSketch::new(0.01); for v in [0., -0., 0.] { - portable.try_update(v).unwrap(); core.try_add(&v).unwrap(); } for v in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { - assert!(portable.try_update(v).is_err()); assert!(core.try_add(&v).is_err()); } - assert_eq!(portable.total_count(), 3); assert_eq!(core.get_count(), 3); - assert_eq!(portable.quantile_interpolated(0.9), Some(0.)); let mut worker = DdSketchWorker::new(0.01); let mut replay = DdSketch::new(0.01); for v in [-10., -10., -10., -0.1, 0., 0., 0.1, 10.] { @@ -104,20 +78,12 @@ fn zero_only_special_values_and_octo_flush() { #[test] fn tiny_values_are_counted_in_zero_bucket_and_signed_serde_survives() { let mut core = DdSketch::new(0.01); - let mut portable = PortableDdSketch::new(0.01); let tiny = f64::MIN_POSITIVE / 2.; for v in [-tiny, tiny, 0.] { core.try_add(&v).unwrap(); - portable.try_update(v).unwrap(); } assert_eq!(core.get_count(), 3); assert_eq!(core.zero_count(), 3); - assert_eq!(portable.total_count(), 3); - assert_eq!(portable.quantile_interpolated(0.5), Some(0.)); - let named: PortableDdSketch = - rmp_serde::from_slice(&rmp_serde::to_vec_named(&portable).unwrap()).unwrap(); - assert_eq!(named.total_count(), 3); - assert_eq!(named.quantile(0.5), Some(0.)); core.try_add(&-3.).unwrap(); let decoded: DdSketch = rmp_serde::from_slice(&rmp_serde::to_vec(&core).unwrap()).unwrap(); assert_eq!(decoded.get_count(), 4); @@ -125,15 +91,3 @@ fn tiny_values_are_counted_in_zero_bucket_and_signed_serde_survives() { assert_eq!(decoded.min(), core.min()); assert_eq!(decoded.max(), core.max()); } - -#[test] -fn cross_zero_interpolation_is_not_a_relative_error_certificate() { - let mut sketch = PortableDdSketch::new(0.01); - sketch.update(-1.); - sketch.update(1.001); - let exact = (-1. + 1.001) / 2.; - let estimated = sketch.quantile_interpolated(0.5).unwrap(); - // Both magnitudes land in the same bucket, so their representatives cancel. - assert_eq!(estimated, 0.); - assert!((estimated - exact).abs() > 0.01 * exact.abs()); -} diff --git a/tests/e2e/cardinality.rs b/tests/e2e/cardinality.rs index 8ba1610e..73fd0656 100644 --- a/tests/e2e/cardinality.rs +++ b/tests/e2e/cardinality.rs @@ -26,7 +26,6 @@ use common::{ }; use common::{uniform_u64, zipf_u64}; -use asap_sketchlib::message_pack_format::portable::hll::{HllSketch, HllVariant}; use asap_sketchlib::sketches::hll::{HyperLogLogHipImpl, HyperLogLogImpl}; use asap_sketchlib::{ Classic, DataInput, Ertl, HyperLogLogHipP12, HyperLogLogHipP14, HyperLogLogHipP16, @@ -248,77 +247,6 @@ hll_battery!( not_mergeable ); -/// The portable wire type across every variant tag and precision. -/// -/// `HllVariant` selects the *wire tag*, not the estimator: `HllSketch::estimate` -/// runs the same register-based Classic formula for `Regular`, `Datafusion` and -/// `Hip` alike. So all three are held to the register model — checking the -/// `Hip` tag against HIP's tighter constant would be asserting a property this -/// type does not implement. -#[test] -fn portable_hll_variants_and_precisions_satisfy_the_register_error_model() { - const N: usize = 200_000; - - for (v, variant) in [HllVariant::Regular, HllVariant::Datafusion, HllVariant::Hip] - .into_iter() - .enumerate() - { - for (p, precision) in [12u32, 14, 16].into_iter().enumerate() { - let spec = CardinalityConfidenceSpec::hll(precision, Z); - // A distinct stream per (variant, precision). Reusing one stream - // across the three variants would feed three estimators the *same* - // registers, whose errors are then near-perfectly correlated — nine - // readings of at most three independent experiments. - let seed = 2001 + (v * 3 + p) as u64; - let stream = uniform_u64(N, u64::MAX / 4, seed); - let truth: std::collections::HashSet = stream.iter().copied().collect(); - - let mut hll = HllSketch::new(variant, precision); - for k in &stream { - hll.update(k.to_be_bytes().as_slice()); - } - if let Err(detail) = spec.check(hll.estimate(), truth.len()) { - panic!("{variant:?}/p{precision} stream_seed={seed}: {detail}"); - } - - // Merging a second sketch built over the SAME identities is a - // register-wise max with itself: the estimate must not move at all. - let mut other = HllSketch::new(variant, precision); - for k in &truth { - other.update((*k).to_be_bytes().as_slice()); - } - let before = hll.estimate(); - hll.merge(&other).expect("merge"); - assert_eq!( - hll.estimate(), - before, - "{variant:?} p{precision}: merging identical identities moved the estimate" - ); - - // Disjoint shards over the same stream merge by register-wise max, - // so the result is the *same* sketch as the single pass and its - // estimate is the *same number*. That is an equality, not a second - // confidence-band reading. - let mut left = HllSketch::new(variant, precision); - let mut right = HllSketch::new(variant, precision); - for (i, k) in stream.iter().enumerate() { - if i % 2 == 0 { - left.update(k.to_be_bytes().as_slice()); - } else { - right.update(k.to_be_bytes().as_slice()); - } - } - left.merge(&right).expect("shard merge"); - assert_eq!( - left.estimate(), - before, - "{variant:?} p{precision}: an even/odd shard merge must reproduce the \ - single pass exactly (registers combine by maximum)" - ); - } - } -} - /// A larger precision must actually buy accuracy. A fixed percentage band /// cannot see this — it passes identically at p12 and p16, so it would not /// notice a precision parameter that never reached the register array. @@ -768,7 +696,6 @@ fn a_custom_precision_merge_reproduces_the_single_pass_registers_for_every_estim #[test] fn a_set_aggregator_delta_describes_the_change_and_survives_the_wire() { use asap_sketchlib::SetAggregatorDelta; - use asap_sketchlib::message_pack_format::MessagePackCodec; use std::collections::HashSet; let mut before = SetAggregator::new(); @@ -798,8 +725,8 @@ fn a_set_aggregator_delta_describes_the_change_and_survives_the_wire() { "both departing keys are removed" ); - let bytes = delta.to_msgpack().expect("encode"); - let decoded = SetAggregatorDelta::from_msgpack(&bytes).expect("decode"); + let bytes = delta.to_asapv1().expect("encode"); + let decoded = SetAggregatorDelta::from_asapv1(&bytes).expect("decode"); assert_eq!(decoded.added, added, "added set survived the wire"); assert_eq!(decoded.removed, removed, "removed set survived the wire"); @@ -820,7 +747,7 @@ fn a_set_aggregator_delta_describes_the_change_and_survives_the_wire() { removed: HashSet::new(), }; let round_tripped = - SetAggregatorDelta::from_msgpack(&empty.to_msgpack().expect("encode empty")) + SetAggregatorDelta::from_asapv1(&empty.to_asapv1().expect("encode empty")) .expect("decode empty"); assert!( round_tripped.added.is_empty() && round_tripped.removed.is_empty(), diff --git a/tests/e2e/composition.rs b/tests/e2e/composition.rs index 84fd6990..f1b05cae 100644 --- a/tests/e2e/composition.rs +++ b/tests/e2e/composition.rs @@ -1,5 +1,5 @@ -//! Composition layers: `HashSketchEnsemble`, `UnivMonQ`'s configuration -//! surface, and the portable facade types that pair a sketch with a heap. +//! Composition layers: `HashSketchEnsemble` and `UnivMonQ`'s configuration +//! surface. //! //! Nitro is a composition layer too, but all of its behaviour lives in //! `tests/e2e/nitro.rs`. @@ -15,10 +15,6 @@ use crate::common; use common::specs::{CardinalityConfidenceSpec, CountMinSpec, CountSketchSpec, KllRankSpec, Tally}; use common::{FreqTruth, NumericTruth, uniform_u64, zipf_u64}; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::countminsketch_topk::CountMinSketchWithHeap; -use asap_sketchlib::message_pack_format::portable::countsketch_topk::CountSketchWithHeap; -use asap_sketchlib::message_pack_format::portable::kll::KllSketch; use asap_sketchlib::{ Classic, CountMinSketch, CountSketch, DataInput, EnsembleSketch, Ertl, FastPath, HashSketchEnsemble, HyperLogLog, HyperLogLogHip, UnivMonQ, UnivMonQConfig, Vector2D, @@ -564,235 +560,6 @@ fn univmonq_multi_shard_merge_with_distinct_source_ids_covers_the_union() { assert_eq!(merged.max(), Some(truth.max()), "merged max must be exact"); } -// -------------------------------------------------------- Portable facade - -/// The portable sketch-plus-heap types, on a real stream against exact truth: -/// the point estimate under the right family's bound, the heap consistent with -/// it, and both surviving a MessagePack round trip and a merge. -#[test] -fn portable_count_min_with_heap_satisfies_the_count_min_bound_through_merge_and_wire() { - const HEAP: usize = 32; - let stream = zipf_u64(N, DOMAIN, 1.1, STREAM_SEED); - let mut truth = FreqTruth::default(); - let mut single = CountMinSketchWithHeap::new(ROWS, COLS, HEAP); - let mut left = CountMinSketchWithHeap::new(ROWS, COLS, HEAP); - let mut right = CountMinSketchWithHeap::new(ROWS, COLS, HEAP); - for (i, k) in stream.iter().enumerate() { - truth.observe(*k as i64); - let key = format!("k{k}"); - single.update(&key, 1.0); - if i % 2 == 0 { - left.update(&key, 1.0); - } else { - right.update(&key, 1.0); - } - } - - let context = format!( - "rows={ROWS} cols={COLS} heap={HEAP} zipf(1.1) domain={DOMAIN} n={N} seed={STREAM_SEED:#x}" - ); - let spec = CountMinSpec::new(ROWS, COLS); - spec.assert_contract( - "portable CountMinSketchWithHeap", - &truth, - |k| single.estimate(&format!("k{k}")), - &context, - ); - - // The heap must agree with the sketch it sits beside. - let mut heap_tally = Tally::default(); - for item in single.topk_heap_items() { - let est = single.estimate(&item.key); - heap_tally.record(item.value == est, || { - format!( - "key {}: heap holds {} but the sketch estimates {est}", - item.key, item.value - ) - }); - } - heap_tally.assert_none("portable CountMinSketchWithHeap heap consistency", &context); - - // Merge, then the same contract. - let merged = CountMinSketchWithHeap::merge_refs(&[&left, &right]).expect("merge"); - spec.assert_contract( - "portable CountMinSketchWithHeap after merge", - &truth, - |k| merged.estimate(&format!("k{k}")), - &context, - ); - - // Wire round trip, then the same contract again: serialization must not - // change a single answer. - let bytes = single.to_msgpack().expect("encode"); - let decoded = CountMinSketchWithHeap::from_msgpack(&bytes).expect("decode"); - let mut wire_tally = Tally::default(); - for (k, _) in truth.pairs() { - let key = format!("k{k}"); - let a = single.estimate(&key); - let b = decoded.estimate(&key); - wire_tally.record(a == b, || format!("key {key}: before {a} after {b}")); - } - wire_tally.assert_none("portable CountMinSketchWithHeap wire round trip", &context); - spec.assert_contract( - "portable CountMinSketchWithHeap after a wire round trip", - &truth, - |k| decoded.estimate(&format!("k{k}")), - &context, - ); -} - -#[test] -fn portable_count_sketch_with_heap_satisfies_the_l2_bound_through_merge_and_wire() { - const HEAP: usize = 32; - const CS_ROWS: usize = 5; - let stream = zipf_u64(N, DOMAIN, 1.1, STREAM_SEED); - let mut truth = FreqTruth::default(); - let mut single = CountSketchWithHeap::new(CS_ROWS, COLS, HEAP); - let mut left = CountSketchWithHeap::new(CS_ROWS, COLS, HEAP); - let mut right = CountSketchWithHeap::new(CS_ROWS, COLS, HEAP); - for (i, k) in stream.iter().enumerate() { - truth.observe(*k as i64); - let key = format!("k{k}"); - single.update(&key, 1.0); - if i % 2 == 0 { - left.update(&key, 1.0); - } else { - right.update(&key, 1.0); - } - } - - let context = format!( - "rows={CS_ROWS} cols={COLS} heap={HEAP} zipf(1.1) domain={DOMAIN} n={N} seed={STREAM_SEED:#x}" - ); - let spec = CountSketchSpec::new(CS_ROWS, COLS); - spec.assert_contract( - "portable CountSketchWithHeap", - &truth, - |k| single.estimate(&format!("k{k}")), - &context, - ); - - let mut heap_tally = Tally::default(); - for item in single.topk_heap_items() { - let est = single.estimate(&item.key); - heap_tally.record(item.value == est, || { - format!( - "key {}: heap holds {} but the sketch estimates {est}", - item.key, item.value - ) - }); - } - heap_tally.assert_none("portable CountSketchWithHeap heap consistency", &context); - - let merged = CountSketchWithHeap::merge_refs(&[&left, &right]).expect("merge"); - spec.assert_contract( - "portable CountSketchWithHeap after merge", - &truth, - |k| merged.estimate(&format!("k{k}")), - &context, - ); - - let bytes = single.to_msgpack().expect("encode"); - let decoded = CountSketchWithHeap::from_msgpack(&bytes).expect("decode"); - spec.assert_contract( - "portable CountSketchWithHeap after a wire round trip", - &truth, - |k| decoded.estimate(&format!("k{k}")), - &context, - ); -} - -/// The portable KLL facade under the DataSketches maximum-rank-error -/// characterization, seeded so a failure reproduces. `KllSketch::new` seeds -/// from the wall clock; `with_seed` is what an accuracy test must use. -/// -/// Trial unit is one sketch, scored on its worst rank error over the `q` grid, -/// with an independent compaction seed per trial. The post-wire sketch is *not* -/// a separate trial — a round trip that preserves every answer bit for bit, as -/// asserted below, gives literally the same numbers. -#[test] -fn portable_kll_sketch_satisfies_the_rank_error_characterization_through_merge_and_wire() { - const K: u16 = 200; - const TRIALS: u64 = 12; - let values: Vec = uniform_u64(N, 1_000_000, STREAM_SEED) - .into_iter() - .map(|v| v as f64) - .collect(); - let truth = NumericTruth::new(values.clone()); - let qs = [0.1f64, 0.25, 0.5, 0.75, 0.9]; - let spec = KllRankSpec::datasketches(K as usize); - let context = format!( - "k={K} uniform n={N} stream_seed={STREAM_SEED:#x}, {TRIALS} independent \ - compaction seeds from 0x5EED_0400" - ); - - let mut tally = Tally::default(); - for t in 0..TRIALS { - let seed = 0x5EED_0400u64.wrapping_add(t.wrapping_mul(0x9E37_79B9_7F4A_7C15)); - let mut single = KllSketch::with_seed(K, seed); - let mut left = KllSketch::with_seed(K, seed ^ 0xAAAA); - let mut right = KllSketch::with_seed(K, seed ^ 0x5555); - for (i, v) in values.iter().enumerate() { - single.update(*v); - if i % 2 == 0 { - left.update(*v); - } else { - right.update(*v); - } - } - left.merge(&right).expect("same-k merge"); - - spec.record_trial( - &mut tally, - &format!("portable KllSketch single pass seed={seed:#x}"), - truth.sorted(), - &qs, - |q| single.quantile(q), - ); - spec.record_trial( - &mut tally, - &format!("portable KllSketch two-shard merge seed={seed:#x}"), - truth.sorted(), - &qs, - |q| left.quantile(q), - ); - - // The wire round trip must preserve every answer bit for bit, which is - // an equality rather than a band — and is why the decoded sketch does - // not enter the rank battery as a second trial. - let bytes = single.to_msgpack().expect("encode"); - let decoded = KllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(decoded.k(), K, "k must survive the wire"); - assert_eq!( - decoded.count(), - single.count(), - "retained mass must survive the wire" - ); - let mut wire_tally = Tally::default(); - for &q in &qs { - let (a, b) = (single.quantile(q), decoded.quantile(q)); - wire_tally.record(a == b, || format!("q={q}: before {a} after {b}")); - } - wire_tally.assert_none( - &format!("portable KllSketch wire round trip (seed={seed:#x})"), - &context, - ); - - // A mismatched `k` must be refused rather than silently merged. - let other_k = KllSketch::with_seed(K * 2, seed); - assert!( - single.merge(&other_k).is_err(), - "merging sketches with different k must fail" - ); - } - - tally.assert_independent_binomial( - "portable KllSketch / maximum normalized rank error per compaction seed", - spec.trial_failure_probability, - &format!("{context}; single pass and two-shard merge, q grid {qs:?}"), - ); -} - // --------------------------------------------------------------------------- // The documented input matrix // --------------------------------------------------------------------------- diff --git a/tests/e2e/frequency.rs b/tests/e2e/frequency.rs index a7391978..be87757d 100644 --- a/tests/e2e/frequency.rs +++ b/tests/e2e/frequency.rs @@ -24,8 +24,6 @@ use common::{ use common::{FreqTruth, uniform_u64, zipf_u64}; use std::collections::HashMap; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketch as PortableCountMinSketch; -use asap_sketchlib::message_pack_format::portable::countsketch::CountSketch as PortableCountSketch; use asap_sketchlib::{ CountMinSketch, CountMinSketchFold, CountSketch, CountSketchFold, CountSketchWithL2Norm, DataInput, DefaultXxHasher, FastPath, RegularPath, Vector2D, @@ -704,47 +702,6 @@ fn folded_sketches_keep_their_own_bounds_through_a_sixteen_way_merge() { ); } -// ------------------------------------------------------ Portable wire twins - -/// The portable wire twins answer over string keys and must satisfy the same -/// two theorems as their core counterparts — Count-Min additive, Count Sketch -/// L2 — with no extra slack for being on the wire side. -#[test] -fn portable_cms_and_cs_string_keys_satisfy_their_own_bounds() { - const CM_ROWS: usize = 3; - const CS_ROWS: usize = 5; - const COLS: usize = 4096; - const STREAM_SEED: u64 = 1006; - - let stream = zipf_u64(50_000, 2048, 1.1, STREAM_SEED); - // The portable types key on strings; `FreqTruth` keys on i64. The stream's - // u64 values are the identity behind both, so `k -> "k{k}"` is injective - // and the two truths agree key for key. - let mut truth = FreqTruth::default(); - let mut pcs = PortableCountMinSketch::new(CM_ROWS, COLS); - let mut pcss = PortableCountSketch::new(CS_ROWS, COLS); - for k in &stream { - truth.observe(*k as i64); - let key = format!("k{k}"); - pcs.update(&key, 1.0); - pcss.update(&key, 1.0); - } - let context = format!("zipf(1.1) domain=2048 n=50000 string keys, stream_seed={STREAM_SEED}"); - - CountMinSpec::new(CM_ROWS, COLS).assert_contract( - "portable CountMinSketch", - &truth, - |k| pcs.estimate(&format!("k{k}")), - &context, - ); - CountSketchSpec::new(CS_ROWS, COLS).assert_contract( - "portable CountSketch", - &truth, - |k| pcss.estimate(&format!("k{k}")), - &context, - ); -} - // --------------------------------------------------------------------------- // The documented input matrix // --------------------------------------------------------------------------- diff --git a/tests/e2e/quantiles.rs b/tests/e2e/quantiles.rs index ea411c4d..787b3c33 100644 --- a/tests/e2e/quantiles.rs +++ b/tests/e2e/quantiles.rs @@ -31,9 +31,8 @@ //! //! DDSketch's guarantee is deterministic — bucket width alone, no hash and no //! sampling — so its batteries tolerate zero violations and no statistical -//! model applies. The two shipped implementations answer a quantile query with -//! **different order statistics**, so each is compared against the truth for -//! its own convention; see `DdRankConvention`. +//! model applies. `DdSketch` is compared against the truth for its own rank +//! convention; see `DdRankConvention`. use crate::common; @@ -43,8 +42,6 @@ use common::{ monotonic_f64, normal_f64, outside_in_ordering, uniform_u64, zipf_f64, }; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDds; -use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; use asap_sketchlib::{ DataInput, DdSketch, Kll, KllConfig, KllDynamic, TumblingWindow, UnivMonQ, UnivMonQConfig, }; @@ -587,37 +584,28 @@ fn dds_streams(alpha: f64, n: usize, seed: u64) -> Vec<(&'static str, Vec)> ] } -/// DDSketch's relative-value-error guarantee, for the core sketch and the -/// portable wire twin, at every supported alpha. +/// DDSketch's relative-value-error guarantee at every supported alpha. /// -/// Each implementation is compared against the exact order statistic **of its -/// own rank convention**: `DdSketch::get_value_at_quantile` answers -/// `sorted[ceil(q*n) - 1]`, while the portable `DdSketch::quantile` answers -/// `sorted[floor(q*(n-1))]`. They are different questions (see -/// `ddsketch_core_and_portable_answer_different_order_statistics`), so a single -/// truth helper would score one of them against the other one's question and -/// quietly absorb the difference into `alpha`. +/// `DdSketch::get_value_at_quantile` answers `sorted[ceil(q*n) - 1]` (see +/// `ddsketch_answers_the_ceil_nearest_rank_order_statistic`), and is compared +/// against that exact order statistic. /// /// The tolerance is `alpha + numerical_slack`, where the slack is a few ULP of /// the logarithmic mapping — never a percentage of alpha, which would license /// breaking the advertised guarantee by that percentage. #[test] -fn ddsketch_core_and_portable_satisfy_the_relative_value_error_contract() { +fn ddsketch_satisfies_the_relative_value_error_contract() { const SAMPLE_SIZES: [usize; 3] = [1_000, 20_000, 100_000]; for &alpha in &DDS_ALPHAS { let core_spec = RelativeQuantileSpec::core(alpha); - let port_spec = RelativeQuantileSpec::portable(alpha); let mut core_tally = Tally::default(); - let mut port_tally = Tally::default(); for (i, &n) in SAMPLE_SIZES.iter().enumerate() { let seed = 3_005_000u64 + i as u64 * 101 + (alpha * 1e6) as u64; for (label, values) in dds_streams(alpha, n, seed) { let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); for v in &values { core.add(v); - port.update(*v); } let truth = NumericTruth::new(values.clone()); assert_eq!( @@ -625,17 +613,9 @@ fn ddsketch_core_and_portable_satisfy_the_relative_value_error_contract() { truth.len(), "{label} alpha={alpha} n={n} seed={seed}: core dropped samples" ); - assert_eq!( - port.total_count() as usize, - truth.len(), - "{label} alpha={alpha} n={n} seed={seed}: portable dropped samples" - ); core_spec.tally_into(&mut core_tally, truth.sorted(), &DDS_QS, |q| { core.get_value_at_quantile(q) }); - port_spec.tally_into(&mut port_tally, truth.sorted(), &DDS_QS, |q| { - port.quantile(q) - }); } } // The guarantee is deterministic, not probabilistic: DDSketch's error @@ -649,44 +629,24 @@ fn ddsketch_core_and_portable_satisfy_the_relative_value_error_contract() { .collect::>() ); core_tally.assert_none(&format!("core DDSketch alpha={alpha}"), &context); - port_tally.assert_none(&format!("portable DdSketch alpha={alpha}"), &context); } } -/// The two implementations answer a quantile query with **different order -/// statistics**, and this pins the divergence rather than letting `alpha` -/// absorb it. -/// -/// - `DdSketch::get_value_at_quantile` uses `rank = ceil(q * n)`, 1-based. -/// - Portable `DdSketch::quantile` uses `target = floor(q * (n - 1))`, 0-based -/// — the lower-quantile convention of the DDSketch paper and of DataDog's -/// reference implementation, which is also what the wire format's Go twin -/// answers. -/// -/// **The decision taken here is to keep both.** The portable type exists to be -/// byte- and answer-compatible with `sketchlib-go`, so its convention is fixed -/// by an external contract; the core type's `ceil` convention is what its own -/// callers have been reading for the life of the API, and it is what lets `q=0` -/// and `q=1` return the exactly retained minimum and maximum. Changing either -/// silently moves numbers under existing callers, and the divergence is only -/// observable at small `n` or ragged `q` — precisely the cases pinned below. -/// What is *not* acceptable is leaving it undocumented, or scoring both against -/// one truth helper, which is what the previous revision did. -/// -/// The probes are chosen so the two formulas disagree: at `n = 3, q = 0.4` the -/// core answers `sorted[1]` and the portable `sorted[0]`. +/// `DdSketch::get_value_at_quantile` answers `rank = ceil(q * n)`, 1-based. +/// The probes sit on values three buckets apart at the coarsest alpha, so a +/// one-rank slip is a different bucket and therefore outside `alpha`. #[test] -fn ddsketch_core_and_portable_answer_different_order_statistics() { - // (n, q, expected core 0-based index, expected portable 0-based index) - const PROBES: [(usize, f64, usize, usize); 8] = [ - (3, 0.4, 1, 0), - (4, 0.34, 1, 1), - (4, 0.3, 1, 0), - (7, 0.2, 1, 1), - (7, 0.6, 4, 3), - (5, 0.5, 2, 2), - (10, 0.25, 2, 2), - (10, 0.15, 1, 1), +fn ddsketch_answers_the_ceil_nearest_rank_order_statistic() { + // (n, q, expected 0-based index) + const PROBES: [(usize, f64, usize); 8] = [ + (3, 0.4, 1), + (4, 0.34, 1), + (4, 0.3, 1), + (7, 0.2, 1), + (7, 0.6, 4), + (5, 0.5, 2), + (10, 0.25, 2), + (10, 0.15, 1), ]; // Values one bucket apart at the coarsest alpha, so a one-rank difference @@ -695,8 +655,7 @@ fn ddsketch_core_and_portable_answer_different_order_statistics() { const ALPHA: f64 = 0.01; let gamma = (1.0 + ALPHA) / (1.0 - ALPHA); - let mut disagreements = 0usize; - for &(n, q, core_idx, port_idx) in &PROBES { + for &(n, q, core_idx) in &PROBES { let values: Vec = (0..n).map(|i| 100.0 * gamma.powi(3 * i as i32)).collect(); let sorted = values.clone(); @@ -705,70 +664,31 @@ fn ddsketch_core_and_portable_answer_different_order_statistics() { core_idx, "core convention ceil(q*n)-1 at n={n} q={q}" ); - assert_eq!( - DdRankConvention::LowerFloor.index(n, q), - port_idx, - "portable convention floor(q*(n-1)) at n={n} q={q}" - ); let mut core = DdSketch::new(ALPHA); - let mut port = PortableDds::new(ALPHA); for v in &values { core.add(v); - port.update(*v); } let core_spec = RelativeQuantileSpec::core(ALPHA); - let port_spec = RelativeQuantileSpec::portable(ALPHA); let core_est = core.get_value_at_quantile(q).expect("non-empty"); - let port_est = port.quantile(q).expect("non-empty"); - if let Err(detail) = core_spec.check(q, core_est, sorted[core_idx]) { panic!("core DDSketch n={n} q={q}: {detail}"); } - if let Err(detail) = port_spec.check(q, port_est, sorted[port_idx]) { - panic!("portable DdSketch n={n} q={q}: {detail}"); - } - - if core_idx != port_idx { - disagreements += 1; - // Each implementation must be answering *its own* order statistic, - // so the two answers must differ here: if they agreed, one of them - // would have silently changed convention. - assert!( - core_est != port_est, - "n={n} q={q}: the conventions pick different order statistics \ - ({core_idx} vs {port_idx}) but both returned {core_est}" - ); - } } - assert!( - disagreements >= 3, - "the probe set must contain cases where the two conventions genuinely \ - disagree; only {disagreements} did" - ); } -/// Endpoint behaviour, which is **not** the same on the two implementations. -/// -/// - The core sketch tracks the exact minimum and maximum beside the bucket -/// store, and `get_value_at_quantile` short-circuits `q <= 0` and `q >= 1` to -/// them. Its endpoints are therefore *exact*, with zero error. -/// - The portable sketch carries no min/max scalars at all — they were removed -/// from the wire — so its endpoints are ordinary bucket representatives and -/// are only guaranteed within `alpha`. The previous revision of this test -/// claimed it "clamps its bucket representative into [min, max], so its -/// endpoints are exact too", which is not what the code does. +/// The sketch tracks the exact minimum and maximum beside the bucket store, and +/// `get_value_at_quantile` short-circuits `q <= 0` and `q >= 1` to them, so its +/// endpoints are exact. #[test] -fn ddsketch_core_endpoints_are_exact_and_portable_endpoints_are_alpha_relative() { +fn ddsketch_endpoints_are_exact() { for &alpha in &DDS_ALPHAS { let values = log_uniform_f64(5_000, (1.0 + alpha) / (1.0 - alpha), 3..30, 4_242); let truth = NumericTruth::new(values.clone()); let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); for v in &values { core.add(v); - port.update(*v); } assert_eq!( core.get_value_at_quantile(0.0), @@ -790,17 +710,6 @@ fn ddsketch_core_endpoints_are_exact_and_portable_endpoints_are_alpha_relative() Some(truth.max()), "core alpha={alpha}: max() must be exact" ); - - // The portable twin holds only buckets, so its endpoints get the same - // relative-value guarantee as any other quantile and nothing more. - let spec = RelativeQuantileSpec::portable(alpha); - let (p0, p1) = (port.quantile(0.0).unwrap(), port.quantile(1.0).unwrap()); - if let Err(detail) = spec.check(0.0, p0, truth.min()) { - panic!("portable alpha={alpha} at q=0: {detail}"); - } - if let Err(detail) = spec.check(1.0, p1, truth.max()) { - panic!("portable alpha={alpha} at q=1: {detail}"); - } } } @@ -813,11 +722,7 @@ fn ddsketch_core_endpoints_are_exact_and_portable_endpoints_are_alpha_relative() fn ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries() { for &alpha in &DDS_ALPHAS { let gamma = (1.0 + alpha) / (1.0 - alpha); - // A one-sample sketch answers every q with that one sample, so the two - // rank conventions coincide here by construction (both index 0) and the - // probe isolates the mapping from any rank effect. let core_spec = RelativeQuantileSpec::core(alpha); - let port_spec = RelativeQuantileSpec::portable(alpha); for k in [-40i32, -7, 0, 1, 13, 60, 200] { let edge = gamma.powi(k); if edge <= 0.0 || !edge.is_finite() { @@ -832,9 +737,7 @@ fn ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries() { ]; for probe in probes { let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); core.add(&probe); - port.update(probe); if core.get_count() == 0 { continue; // outside the indexable range at this alpha } @@ -844,10 +747,6 @@ fn ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries() { if let Err(detail) = core_spec.check(0.5, est, probe) { panic!("core DDSketch alpha={alpha} gamma^{k} boundary probe: {detail}"); } - let pest = port.quantile(0.5).unwrap(); - if let Err(detail) = port_spec.check(0.5, pest, probe) { - panic!("portable DdSketch alpha={alpha} gamma^{k} boundary probe: {detail}"); - } } } } @@ -858,13 +757,11 @@ fn ddsketch_satisfies_the_relative_error_contract_at_bucket_boundaries() { /// guarantee is stated over. #[test] fn ddsketch_merge_and_delta_replay_preserve_the_relative_error_contract() { - use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketchDelta; use asap_sketchlib::octo_delta::DdSketchBucketDelta; const N: usize = 40_000; for &alpha in &DDS_ALPHAS { let spec = RelativeQuantileSpec::core(alpha); - let port_spec = RelativeQuantileSpec::portable(alpha); let values = zipf_f64(N, 8_192, 1.1, 1e3, 1e7, 7_654_321 + (alpha * 1e5) as u64); let truth = NumericTruth::new(values.clone()); @@ -929,49 +826,6 @@ fn ddsketch_merge_and_delta_replay_preserve_the_relative_error_contract() { &DDS_QS[1..6] ), ); - - // Portable side: merge, then a delta carrying the same buckets. - let mut pa = PortableDds::new(alpha); - let mut pb = PortableDds::new(alpha); - for (i, v) in values.iter().enumerate() { - if i % 2 == 0 { - pa.update(*v); - } else { - pb.update(*v); - } - } - pa.merge(&pb).expect("same-alpha portable merge"); - let mut port_tally = Tally::default(); - port_spec.tally_into(&mut port_tally, truth.sorted(), &DDS_QS, |q| pa.quantile(q)); - port_tally.assert_none( - &format!("portable DdSketch alpha={alpha} after merge"), - &format!("zipf n={N} over [1e3, 1e7]"), - ); - - let mut pr = PortableDds::new(alpha); - let delta = DdSketchDelta { - buckets: pa - .store_counts - .iter() - .enumerate() - .filter(|(_, c)| **c > 0) - .map(|(i, c)| (pa.store_offset + i as i32, *c)) - .collect(), - d_count: pa.total_count() as i64, - ..DdSketchDelta::default() - }; - pr.apply_delta(&delta).expect("benign delta"); - let mut pr_tally = Tally::default(); - port_spec.tally_into(&mut pr_tally, truth.sorted(), &DDS_QS[1..6], |q| { - pr.quantile(q) - }); - pr_tally.assert_none( - &format!("portable DdSketch alpha={alpha} rebuilt from a delta"), - &format!( - "zipf n={N} over [1e3, 1e7], interior q grid {:?}", - &DDS_QS[1..6] - ), - ); } } @@ -1563,82 +1417,18 @@ fn tumbling_kll_windows_are_exact_and_answers_satisfy_the_rank_contract() { ); } -// --------------------------------------------- Portable HydraKll per-key - -/// Hydra routes each key to its own KLL cell, so each cell answers under the -/// same rank characterization as a standalone KLL over that key's observations. -/// -/// Every cell of a `HydraKllSketch` is cloned from one seeded prototype, so the -/// two keys of a single grid share a compaction seed and are **not** independent -/// trials. Each grid is therefore reduced to its worst rank error across both -/// keys and the whole q grid, and the binomial runs over twelve independent -/// prototype seeds. -#[test] -fn portable_hydra_kll_per_key_medians_satisfy_the_rank_characterization() { - const K: usize = 200; - const TRIALS: usize = 12; - const QS: [f64; 5] = [0.1, 0.25, 0.5, 0.75, 0.9]; - - let spec = KllRankSpec::datasketches(K); - let mut truths: Vec<(&str, NumericTruth)> = Vec::new(); - for (name, base, seed) in [("svc-a", 100.0f64, 3010u64), ("svc-b", 900.0, 3011)] { - let vals: Vec = normal_f64(4000, base, base * 0.05, seed) - .into_iter() - .map(f64::abs) // HydraKll cells are KLL: positive domain - .collect(); - truths.push((name, NumericTruth::new(vals))); - } - - let mut tally = Tally::default(); - for t in 0..TRIALS { - let seed = kll_trial_seed(0x5EED_0500 + t as u64); - let mut hk = HydraKllSketch::with_seed(3, 256, K as u16, seed); - for (name, truth) in &truths { - for v in truth.sorted() { - hk.update(name, *v); - } - } - let mut worst = 0.0f64; - let mut detail = String::new(); - for (name, truth) in &truths { - let (e, d) = spec.max_rank_error(truth.sorted(), &QS, |q| hk.quantile(name, q)); - if e >= worst { - worst = e; - detail = format!("key {name}: {d}"); - } - } - let eps = spec.epsilon(); - tally.record(worst <= eps, || { - format!("seed={seed:#x}: max rank error {worst:.6} > eps(k={K}) = {eps:.6}; {detail}") - }); - } - tally.assert_independent_binomial( - "portable HydraKll per-key quantiles / maximum rank error per prototype seed", - spec.trial_failure_probability, - &format!( - "k={K}, 3x256 grid, normal(100, 5) and normal(900, 45), stream seeds 3010/3011, \ - {TRIALS} independent prototype seeds from kll_trial_seed(0x5EED_0500..), \ - q grid {QS:?}" - ), - ); -} - // ------------------------------------------- DDSketch input rejection /// Structural guards, not accuracy: values the mapping cannot index and -/// deltas spanning an implausible bucket range must be rejected without -/// corrupting state. No error bound applies to any of it. +/// mismatched mappings must be rejected without corrupting state. No error +/// bound applies to any of it. #[test] fn ddsketch_rejects_untrackable_values_and_mapping_mismatches() { let alpha = 0.01; let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); - // Non-finite / above-maximum-magnitude values must be - // dropped by BOTH implementations: silently, without corrupting bucket 0 - // (NaN floor-casts to 0) and without letting one sample force a distant- - // bucket allocation (unguarded, f64::MAX maps ~35k buckets away at - // alpha=0.01 — ~277 KiB of amplification per sample, scaling with 1/lnγ). + // Non-finite / above-maximum-magnitude values must be dropped silently, + // without corrupting bucket 0 (NaN floor-casts to 0). for v in [ f64::NAN, f64::NEG_INFINITY, @@ -1647,78 +1437,42 @@ fn ddsketch_rejects_untrackable_values_and_mapping_mismatches() { 1e308, ] { core.add(&v); - port.update(v); } assert_eq!(core.get_count(), 0, "core must drop untrackable extremes"); - assert_eq!( - port.total_count(), - 0, - "portable must drop untrackable extremes" - ); - assert!( - port.store_counts.len() < 10_000, - "portable store grew to {} buckets from untrackable input", - port.store_counts.len() - ); - assert_eq!(port.quantile(0.5), None, "nothing trackable was added"); // Normal samples still work after the rejected inputs. let good = [1.0f64, 2.0, 4.0, 8.0, 16.0]; for v in good { core.add(&v); - port.update(v); } assert_eq!(core.get_count(), 5); - assert_eq!(port.total_count(), 5); - assert_between( - port.store_counts.len() as f64, - 1.0, - 2.0 * 256.0, // initial GROW_CHUNK seed + at most one more - "store stays compact", - ); // Boundary acceptance: values just INSIDE the indexable range must still // be tracked, so the guards reject only genuinely unmappable extremes. - // Bounds come from the shared production helper — core and portable MUST - // agree because they compute from the same function. let (min_idx, max_idx) = asap_sketchlib::sketches::ddsketch::ddsketch_indexable_bounds(alpha); - let mut port_boundary = PortableDds::new(alpha); let mut core_boundary = DdSketch::new(alpha); let just_inside_min = min_idx * (1.0 + 1e-9); // a hair above the floor let just_inside_max = max_idx * (1.0 - 1e-9); // a hair below the ceiling - port_boundary.update(just_inside_min); - port_boundary.update(just_inside_max); core_boundary.add(&just_inside_min); core_boundary.add(&just_inside_max); assert_eq!( - port_boundary.total_count(), + core_boundary.get_count(), 2, "in-range extremes near boundaries must be kept" ); - assert_eq!(core_boundary.get_count(), 2, "core boundary agreement"); - assert_eq!(port_boundary.total_count(), core_boundary.get_count()); // Below the minimum counts as zero; above the maximum is rejected. - port_boundary.update(min_idx * 0.5); - port_boundary.update(max_idx * (1.0 + 1e-6)); core_boundary.add(&(min_idx * 0.5)); - assert_eq!( - port_boundary.total_count(), - 3, - "sub-indexable neighbor enters zero bucket" - ); + core_boundary.add(&(max_idx * (1.0 + 1e-6))); assert_eq!( core_boundary.get_count(), 3, "core retains sub-indexable neighbor as zero" ); - - assert_eq!(port_boundary.zero_count, 1); assert_eq!(core_boundary.zero_count(), 1); - // Item 2 contract: mismatched mappings are a runtime error in BOTH types, - // not a debug-only assertion. + // Mismatched mappings are a runtime error, not a debug-only assertion. let other_alpha = 0.05; let mut core_other = DdSketch::new(other_alpha); core_other.add(&1.0); @@ -1726,80 +1480,16 @@ fn ddsketch_rejects_untrackable_values_and_mapping_mismatches() { core.merge(&core_other).is_err(), "core merge must reject alpha mismatch" ); - let mut port_other = PortableDds::new(other_alpha); - port_other.update(1.0); - assert!( - port.merge(&port_other).is_err(), - "portable merge must reject alpha mismatch" - ); - // Tiny-alpha regression: at alpha=1e-9 ln(gamma) ~ 2e-9, and naive - // reciprocal-multiplied guard formulas diverge from core's — admitting - // v=1e-300 whose bucket index saturates i32 and overflows ensure_bucket. - // Both implementations count it as zero without allocating a bucket. - let mut tiny = PortableDds::new(1e-9); + // Tiny-alpha regression: at alpha=1e-9 ln(gamma) ~ 2e-9, and v=1e-300's + // bucket index would saturate i32. It is counted as zero. let mut tiny_core = DdSketch::new(1e-9); - tiny.update(1e-300); tiny_core.add(&1e-300); - assert_eq!( - tiny.total_count(), - 1, - "portable counts sub-indexable value at tiny alpha" - ); assert_eq!( tiny_core.get_count(), 1, "core counts sub-indexable value at tiny alpha" ); - assert_eq!( - tiny.store_counts.len(), - 0, - "no allocation may occur for zero-mapped values" - ); -} - -#[test] -fn portable_ddsketch_rejects_hostile_delta_spans() { - use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketchDelta; - - let alpha = 0.01; - let mut base = PortableDds::new(alpha); - base.update(1.0); - let (len_before, offset_before) = (base.store_counts.len(), base.store_offset); - - // A corrupt/hostile delta pointing near i32::MAX must be rejected with an - // error BEFORE any allocation: the naive pad would be ~2e9 buckets. - let hostile = DdSketchDelta { - buckets: vec![(i32::MAX - 1, 1), (7, 3)], - ..DdSketchDelta::default() - }; - assert!( - base.apply_delta(&hostile).is_err(), - "hostile far-span delta must be rejected" - ); - assert_eq!( - base.store_counts.len(), - len_before, - "state untouched on rejection" - ); - assert_eq!( - base.store_offset, offset_before, - "offset untouched on rejection" - ); - - // Benign deltas still apply: bucket index 6 carries count 5 afterward, - // and its representative gamma^6*(1+alpha) becomes visible in queries. - let benign = DdSketchDelta { - buckets: vec![(6, 5)], - ..DdSketchDelta::default() - }; - base.apply_delta(&benign).expect("benign delta"); - let gamma = (1.0 + alpha) / (1.0 - alpha); - assert_eq!( - base.quantile(0.9), - Some(gamma.powf(6.0) * (1.0 + alpha)), - "applied delta count visible through queries" - ); } const KLL_M_AXIS: [usize; 5] = [2, 4, 8, 16, 64]; @@ -2024,9 +1714,7 @@ fn ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_para for (i, &alpha) in DDS_EXTREME_ALPHAS.iter().enumerate() { let core_spec = RelativeQuantileSpec::core(alpha); - let port_spec = RelativeQuantileSpec::portable(alpha); let mut core_tally = Tally::default(); - let mut port_tally = Tally::default(); let seed = 0x0DDA_0000u64 + i as u64 * 7919; let (min_indexable, max_indexable) = asap_sketchlib::sketches::ddsketch::ddsketch_indexable_bounds(alpha); @@ -2039,10 +1727,8 @@ fn ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_para continue; } let mut core = DdSketch::new(alpha); - let mut port = PortableDds::new(alpha); for v in &values { core.add(v); - port.update(*v); } let truth = NumericTruth::new(values.clone()); assert_eq!( @@ -2050,24 +1736,15 @@ fn ddsketch_satisfies_the_relative_value_error_contract_at_extreme_accuracy_para truth.len(), "{label} alpha={alpha}: core dropped an indexable sample" ); - assert_eq!( - port.total_count() as usize, - truth.len(), - "{label} alpha={alpha}: portable dropped an indexable sample" - ); core_spec.tally_into(&mut core_tally, truth.sorted(), &DDS_QS, |q| { core.get_value_at_quantile(q) }); - port_spec.tally_into(&mut port_tally, truth.sorted(), &DDS_QS, |q| { - port.quantile(q) - }); } let context = format!( "alpha={alpha} n={N} indexable=[{min_indexable:.3e}, {max_indexable:.3e}] \ q grid {DDS_QS:?}" ); core_tally.assert_none(&format!("core DDSketch alpha={alpha}"), &context); - port_tally.assert_none(&format!("portable DdSketch alpha={alpha}"), &context); } } diff --git a/tests/msgpack_compat.rs b/tests/msgpack_compat.rs deleted file mode 100644 index 45f646ce..00000000 --- a/tests/msgpack_compat.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! Cross-language MessagePack envelope compatibility tests. -//! -//! These tests pin down the wire format described in -//! [`asap_sketchlib::message_pack_format`] and shared with `sketchlib-go`. -//! -//! # Coverage today (round-trip only) -//! -//! For every wire-format-aligned type, encode a populated instance, -//! decode the bytes back, and assert structural equality. This catches -//! Rust-side encoder/decoder regressions but does NOT verify byte-level -//! parity with the Go implementation. -//! -//! # Future: golden-bytes fixtures -//! -//! The `tests/fixtures/msgpack/` directory is reserved for canonical -//! byte streams produced by `sketchlib-go`. When those land, each -//! `*_round_trip` test below should grow a sibling `*_decodes_go_bytes` -//! test that loads the corresponding `.msgpack` file via -//! `include_bytes!` and asserts deserialize succeeds and field values -//! match the producer's expectations. - -use std::collections::HashSet; - -use asap_sketchlib::SetAggregator; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketch; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketchWire; -use asap_sketchlib::message_pack_format::portable::countminsketch_topk::CmsHeapItem; -use asap_sketchlib::message_pack_format::portable::countminsketch_topk::CountMinSketchWithHeap; -use asap_sketchlib::message_pack_format::portable::countminsketch_topk::{ - CountMinSketchInnerWire, CountMinSketchWithHeapWire, -}; -use asap_sketchlib::message_pack_format::portable::countsketch::CountSketch; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch; -use asap_sketchlib::message_pack_format::portable::delta_set_aggregator::SetAggregatorDelta; -use asap_sketchlib::message_pack_format::portable::hll::HllSketch; -use asap_sketchlib::message_pack_format::portable::hll::HllVariant; -use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; -use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketchWire; -use asap_sketchlib::message_pack_format::portable::kll::KllSketch; -use asap_sketchlib::message_pack_format::portable::kll::KllSketchData; - -// ===== round-trip: every wire-format-aligned type ===== - -#[test] -fn count_min_sketch_round_trip() { - let mut s = CountMinSketch::new(3, 64); - s.update("alpha", 1.0); - s.update("beta", 2.0); - s.update("alpha", 4.0); - let bytes = s.to_msgpack().expect("encode"); - let restored = CountMinSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.rows, 3); - assert_eq!(restored.cols, 64); -} - -#[test] -fn count_min_sketch_with_heap_round_trip() { - let mut s = CountMinSketchWithHeap::new(3, 64, 8); - s.update("hot", 100.0); - s.update("warm", 10.0); - let bytes = s.to_msgpack().expect("encode"); - let restored = CountMinSketchWithHeap::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.rows, 3); - assert_eq!(restored.cols, 64); - assert_eq!(restored.heap_size, 8); -} - -#[test] -fn count_sketch_round_trip() { - let mut s = CountSketch::new(3, 64); - s.update("k1", 1.0); - s.update("k2", 2.0); - let bytes = s.to_msgpack().expect("encode"); - let restored = CountSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.rows, 3); - assert_eq!(restored.cols, 64); -} - -#[test] -fn dd_sketch_round_trip() { - let mut s = DdSketch::new(0.01); - s.update(1.0); - s.update(10.0); - s.update(100.0); - let bytes = s.to_msgpack().expect("encode"); - let restored = DdSketch::from_msgpack(&bytes).expect("decode"); - // `count` is not on the wire; recover it by summing the bucket array. - assert_eq!(restored.total_count(), 3); -} - -#[test] -fn hll_sketch_round_trip() { - let mut s = HllSketch::new(HllVariant::Regular, 8); - s.update(b"a"); - s.update(b"b"); - s.update(b"c"); - let bytes = s.to_msgpack().expect("encode"); - let restored = HllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.registers.len(), s.registers.len()); -} - -#[test] -fn kll_sketch_round_trip() { - let mut s = KllSketch::with_seed(200, 0x5EED_0800); - for i in 0..100 { - s.update(i as f64); - } - let bytes = s.to_msgpack().expect("encode"); - let restored = KllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.k, 200); - assert_eq!(restored.count(), 100); -} - -#[test] -fn hydra_kll_sketch_round_trip() { - let mut s = HydraKllSketch::with_seed(2, 4, 200, 0x5EED_0900); - s.update("a", 1.0); - s.update("a", 2.0); - s.update("b", 3.0); - let bytes = s.to_msgpack().expect("encode"); - let restored = HydraKllSketch::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.rows, 2); - assert_eq!(restored.cols, 4); -} - -#[test] -fn set_aggregator_round_trip() { - let mut s = SetAggregator::new(); - s.update("web"); - s.update("api"); - let bytes = s.to_msgpack().expect("encode"); - let restored = SetAggregator::from_msgpack(&bytes).expect("decode"); - assert_eq!(restored.values.len(), 2); - assert!(restored.values.contains("web")); -} - -#[test] -fn set_aggregator_delta_round_trip() { - let mut added = HashSet::new(); - added.insert("a".to_string()); - let mut removed = HashSet::new(); - removed.insert("b".to_string()); - let dr = SetAggregatorDelta { added, removed }; - let bytes = dr.to_msgpack().expect("encode"); - let restored = SetAggregatorDelta::from_msgpack(&bytes).expect("decode"); - assert!(restored.added.contains("a")); - assert!(restored.removed.contains("b")); -} - -// ===== DTO-level structural sanity ===== -// -// Verify that the DTO field shapes still match what `sketchlib-go` -// expects (map keys / nesting). A producer that drops a field would -// trip these. - -#[test] -fn count_min_wire_shape() { - let wire = CountMinSketchWire { - sketch: vec![vec![1.0, 2.0]; 3], - rows: 3, - cols: 2, - }; - let bytes = rmp_serde::to_vec(&wire).unwrap(); - let restored: CountMinSketchWire = rmp_serde::from_slice(&bytes).unwrap(); - assert_eq!(restored.rows, 3); - assert_eq!(restored.cols, 2); - assert_eq!(restored.sketch.len(), 3); -} - -#[test] -fn count_min_with_heap_wire_shape() { - let wire = CountMinSketchWithHeapWire { - sketch: CountMinSketchInnerWire { - sketch: vec![vec![0.0; 4]; 2], - rows: 2, - cols: 4, - }, - topk_heap: vec![CmsHeapItem { - key: "hot".to_string(), - value: 42.0, - }], - heap_size: 8, - }; - let bytes = rmp_serde::to_vec(&wire).unwrap(); - let restored: CountMinSketchWithHeapWire = rmp_serde::from_slice(&bytes).unwrap(); - assert_eq!(restored.heap_size, 8); - assert_eq!(restored.topk_heap.len(), 1); - assert_eq!(restored.topk_heap[0].key, "hot"); -} - -#[test] -fn hydra_kll_wire_shape() { - let wire = HydraKllSketchWire { - rows: 2, - cols: 3, - sketches: vec![ - vec![ - KllSketchData { - k: 200, - sketch_bytes: vec![], - }; - 3 - ]; - 2 - ], - }; - let bytes = rmp_serde::to_vec(&wire).unwrap(); - let restored: HydraKllSketchWire = rmp_serde::from_slice(&bytes).unwrap(); - assert_eq!(restored.rows, 2); - assert_eq!(restored.cols, 3); - assert_eq!(restored.sketches.len(), 2); - assert_eq!(restored.sketches[0].len(), 3); -} - -// ===== golden-bytes placeholders: an ignored, uncovered gap ===== -// -// The five tests below are **empty and ignored**. They verify nothing today. -// -// What they need is a msgpack payload produced by `sketchlib-go` and checked in -// under `tests/fixtures/msgpack/`. That fixture cannot be generated here: this -// repository has no Go toolchain and no vendored copy of the Go encoder, so any -// bytes produced locally would be this crate's own output compared against -// itself — which is precisely the thing a cross-language golden exists to rule -// out. Writing such bytes would make the tests pass while testing nothing, so -// they stay ignored until a real fixture lands. -// -// The related `asapv1_golden/*.hex` files *are* cross-checked with the Go repo -// (see `asapv1_golden/README.md`), so the ASAPv1 envelope has cross-language -// coverage; what is missing is the older msgpack facade. -// -// When the fixtures arrive, each test should: -// 1. include_bytes!("fixtures/msgpack/.msgpack") -// 2. ::from_msgpack(bytes) succeeds -// 3. assert specific field values match the Go producer - -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn count_min_decodes_go_bytes() { - // let bytes = include_bytes!("fixtures/msgpack/count_min.msgpack"); - // let s = CountMinSketch::from_msgpack(bytes).unwrap(); - // assert_eq!(s.rows, EXPECTED_ROWS); -} - -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn dd_sketch_decodes_go_bytes() {} - -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn hll_sketch_decodes_go_bytes() {} - -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn kll_sketch_decodes_go_bytes() {} - -#[ignore = "gap: needs a sketchlib-go-produced msgpack fixture; none can be generated in this repo"] -#[test] -fn hydra_kll_decodes_go_bytes() {} diff --git a/tests/pbt/count_min.rs b/tests/pbt/count_min.rs index 4ba71f32..c5fb821d 100644 --- a/tests/pbt/count_min.rs +++ b/tests/pbt/count_min.rs @@ -1,5 +1,4 @@ -//! Property tests for Count-Min: the native matrix sketch and the portable -//! `CountMinSketch` that carries it over the wire. +//! Property tests for Count-Min. //! //! Cormode and Muthukrishnan, J. Algorithms '05. //! @@ -10,11 +9,8 @@ //! oracle, which is independent of the sketch's own bookkeeping. use crate::support::{ - PROBE_KEYS, close, edge_dimension, edge_rows, grid, keyed_updates, keys, matrices_close, + PROBE_KEYS, edge_dimension, edge_rows, grid, keys, }; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketch as PortableCountMinSketch; -use asap_sketchlib::message_pack_format::portable::countminsketch::CountMinSketchDelta; use asap_sketchlib::{ CountMinSketch, DataInput, DefaultMatrixI32, FastPath, MatrixFastHash, MatrixStorage, QuickMatrixI64, RegularPath, Vector2D, hash_for_matrix, @@ -72,93 +68,6 @@ fn columns_of(rows: usize, cols: usize, k: u64) -> Vec { (0..rows).map(|r| hashed.col_for_row(r, cols)).collect() } -fn cms_of(rows: usize, cols: usize, updates: &[(String, f64)]) -> PortableCountMinSketch { - let mut s = PortableCountMinSketch::new(rows, cols); - for (k, v) in updates { - s.update(k, *v); - } - s -} - -// ===== Sequence-law scaffolding ===== - -/// A transition the sequence law draws from. Every variant is a public entry -/// point of `CountMinSketch`. -#[derive(Debug, Clone)] -enum Op { - Insert(u64, i64), - Merge(Vec<(u64, i64)>), - ApplyDelta(Vec<(u64, i64)>), - RoundTrip, - /// Merges a `rows + .0` by `cols + .1` peer, a geometry `merge` refuses. - MergeMismatched(usize, usize), -} - -fn seq_stream(max: usize) -> impl Strategy> { - prop::collection::vec((0u64..DOMAIN, 1i64..8), 0..max) -} - -/// Inserts dominate, so the sequence reaches a deep state; the transitions -/// that rebuild the backend stay frequent enough to land between them. -fn seq_op() -> impl Strategy { - prop_oneof![ - 16 => (0u64..DOMAIN, 1i64..8).prop_map(|(k, w)| Op::Insert(k, w)), - 4 => seq_stream(20).prop_map(Op::Merge), - 2 => seq_stream(20).prop_map(Op::ApplyDelta), - 2 => Just(Op::RoundTrip), - 1 => prop_oneof![ - Just((1usize, 0usize)), - Just((0usize, 1usize)), - Just((1usize, 1usize)), - Just((3usize, 2usize)), - ] - .prop_map(|(over_rows, over_cols)| Op::MergeMismatched(over_rows, over_cols)), - ] -} - -/// `"0"` through `"31"`, the string keys the portable sketch takes. -fn key_domain() -> Vec { - (0..DOMAIN).map(|k| k.to_string()).collect() -} - -fn cms_stream_of( - rows: usize, - cols: usize, - keys: &[String], - stream: &[(u64, i64)], -) -> PortableCountMinSketch { - let mut sketch = PortableCountMinSketch::new(rows, cols); - for (k, w) in stream { - sketch.update(&keys[*k as usize], *w as f64); - } - sketch -} - -/// Every non-zero cell of `sketch`, as the delta that adds it to a peer. -fn delta_of(sketch: &PortableCountMinSketch) -> CountMinSketchDelta { - let mut cells = Vec::new(); - for (r, row) in sketch.sketch().iter().enumerate() { - for (c, count) in row.iter().enumerate() { - if *count != 0.0 { - cells.push((r as u32, c as u32, *count as i64)); - } - } - } - CountMinSketchDelta { - rows: sketch.rows() as u32, - cols: sketch.cols() as u32, - cells, - ..CountMinSketchDelta::default() - } -} - -/// The column a string key occupies on each row, from the public hash entry -/// point rather than from the sketch's own bookkeeping. -fn str_columns_of(rows: usize, cols: usize, key: &str) -> Vec { - let hashed = hash_for_matrix(rows, cols, &DataInput::Str(key)); - (0..rows).map(|r| hashed.col_for_row(r, cols)).collect() -} - /// The lightest row's mass in the cell `k` occupies: the tightest ceiling a /// Count-Min estimate of `k` can reach, own mass included. fn ceiling_for(rows: usize, columns: &[Vec], truth: &HashMap, k: u64) -> i64 { @@ -182,72 +91,6 @@ fn cm_fast_of(rows: usize, cols: usize, stream: &[(u64, i32)]) -> CmFast { sketch } -/// The four invariants every state in the sequence holds: the geometry it was -/// built with, non-negative cells, every row carrying the model's whole mass, -/// and each key's estimate between its truth and its lightest row's mass. -fn holds_model( - sketch: &PortableCountMinSketch, - model: &[i64], - keys: &[String], - columns: &[Vec], - geometry: (usize, usize), - what: &str, -) -> Result<(), TestCaseError> { - let (rows, cols) = geometry; - prop_assert_eq!(sketch.rows(), rows, "{}: rows", what); - prop_assert_eq!(sketch.cols(), cols, "{}: cols", what); - - let cells = sketch.sketch(); - prop_assert_eq!(cells.len(), rows, "{}: rows of the matrix", what); - - let total: i64 = model.iter().sum(); - for (r, row) in cells.iter().enumerate() { - prop_assert_eq!(row.len(), cols, "{}: width of row {}", what, r); - prop_assert!( - row.iter().all(|cell| *cell >= 0.0), - "{}: row {} holds a negative cell: {:?}", - what, - r, - row - ); - let carried: f64 = row.iter().sum(); - prop_assert_eq!( - carried, - total as f64, - "{}: row {} carries {} of the model's {}", - what, - r, - carried, - total - ); - } - - let mut column_mass = vec![0i64; rows * cols]; - for (k, mass) in model.iter().enumerate() { - for (r, col) in columns[k].iter().enumerate() { - column_mass[r * cols + col] += *mass; - } - } - - for (k, own) in model.iter().enumerate() { - let est = sketch.estimate(&keys[k]).round() as i64; - let ceiling = (0..rows) - .map(|r| column_mass[r * cols + columns[k][r]]) - .min() - .expect("a sketch has at least one row"); - prop_assert!( - est >= *own && est <= ceiling, - "{}: key {} estimated {} outside [{}, {}]", - what, - keys[k], - est, - own, - ceiling - ); - } - Ok(()) -} - proptest! { // ===== Stream order must not reach the state ===== @@ -320,110 +163,8 @@ proptest! { prop_assert_eq!(cm_fast_cells(&by_hashes), cm_fast_cells(&by_loop)); } - // ===== Merge algebra, on the portable representation ===== - - #[test] - fn count_min_merge_is_commutative( - rows in 1usize..6, - cols in 4usize..64, - a in keyed_updates(30), - b in keyed_updates(30), - ) { - let mut ab = cms_of(rows, cols, &a); - ab.merge(&cms_of(rows, cols, &b)).expect("merge"); - let mut ba = cms_of(rows, cols, &b); - ba.merge(&cms_of(rows, cols, &a)).expect("merge"); - - prop_assert_eq!(ab.sketch(), ba.sketch()); - } - - #[test] - fn count_min_merge_is_associative( - rows in 1usize..6, - cols in 4usize..64, - a in keyed_updates(20), - b in keyed_updates(20), - c in keyed_updates(20), - ) { - let mut left = cms_of(rows, cols, &a); - left.merge(&cms_of(rows, cols, &b)).expect("merge"); - left.merge(&cms_of(rows, cols, &c)).expect("merge"); - - let mut right = cms_of(rows, cols, &b); - right.merge(&cms_of(rows, cols, &c)).expect("merge"); - let mut right_full = cms_of(rows, cols, &a); - right_full.merge(&right).expect("merge"); - - prop_assert!(matrices_close(&left.sketch(), &right_full.sketch())); - } - - #[test] - fn count_min_empty_is_a_merge_identity( - rows in 1usize..6, - cols in 4usize..64, - a in keyed_updates(30), - ) { - let base = cms_of(rows, cols, &a); - let mut merged = base.clone(); - merged.merge(&PortableCountMinSketch::new(rows, cols)).expect("merge"); - - prop_assert_eq!(merged.sketch(), base.sketch()); - } - - #[test] - fn count_min_merge_matches_streaming_the_concatenation( - rows in 1usize..6, - cols in 4usize..64, - a in keyed_updates(30), - b in keyed_updates(30), - ) { - let mut merged = cms_of(rows, cols, &a); - merged.merge(&cms_of(rows, cols, &b)).expect("merge"); - - let concatenated: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); - let streamed = cms_of(rows, cols, &concatenated); - - prop_assert!(matrices_close(&merged.sketch(), &streamed.sketch())); - } - - #[test] - fn count_min_never_underestimates( - rows in 1usize..6, - cols in 4usize..64, - updates in keyed_updates(40), - ) { - let s = cms_of(rows, cols, &updates); - for (k, _) in &updates { - let truth: f64 = updates.iter().filter(|(x, _)| x == k).map(|(_, v)| v).sum(); - let est = s.estimate(k); - prop_assert!( - est >= truth || close(est, truth), - "key {}: estimate {} < truth {}", k, est, truth - ); - } - } - // ===== Wire ===== - #[test] - fn count_min_round_trip_preserves_every_cell( - rows in 1usize..8, - cols in 1usize..128, - updates in keyed_updates(40), - ) { - let s = cms_of(rows, cols, &updates); - - let bytes = s.to_msgpack().expect("encode"); - let restored = PortableCountMinSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.rows(), s.rows()); - prop_assert_eq!(restored.cols(), s.cols()); - prop_assert_eq!(restored.sketch(), s.sketch()); - for (k, _) in &updates { - prop_assert_eq!(restored.estimate(k), s.estimate(k), "key {}", k); - } - } - #[test] fn count_min_round_trips_at_edge_geometries( rows in edge_rows(), @@ -517,38 +258,6 @@ proptest! { } } - // ===== A merge only ever raises an estimate ===== - - #[test] - fn count_min_merge_never_lowers_either_sides_estimate( - rows in 1usize..6, - cols in 1usize..16, - a in keyed_updates(30), - b in keyed_updates(30), - ) { - let left = cms_of(rows, cols, &a); - let right = cms_of(rows, cols, &b); - let mut merged = left.clone(); - merged.merge(&right).expect("merge"); - - let probes = a - .iter() - .chain(b.iter()) - .map(|(k, _)| k.as_str()) - .chain(["0", "!"]); - for k in probes { - let after = merged.estimate(k); - prop_assert!( - after >= left.estimate(k), - "key {}: merged {} below the left estimate {}", k, after, left.estimate(k) - ); - prop_assert!( - after >= right.estimate(k), - "key {}: merged {} below the right estimate {}", k, after, right.estimate(k) - ); - } - } - // ===== Fixed-dimension storage lands in the cells the dynamic one does ===== // // `DefaultMatrixI32` and `QuickMatrixI64` fold the column from the @@ -600,86 +309,6 @@ proptest! { } } - // ===== A state that survives being rebuilt ===== - // - // Merge, apply_delta and a msgpack round trip each replace the backend - // mid-stream. The model is the exact ledger of everything inserted so far, - // and every invariant is re-checked after every transition. - - #[test] - fn count_min_holds_its_model_across_a_sequence_of_transitions( - rows in 1usize..6, - cols in 1usize..9, - ops in prop::collection::vec(seq_op(), 0..80), - ) { - let keys = key_domain(); - let columns: Vec> = keys - .iter() - .map(|k| str_columns_of(rows, cols, k)) - .collect(); - let mut model = vec![0i64; keys.len()]; - let mut sketch = PortableCountMinSketch::new(rows, cols); - holds_model(&sketch, &model, &keys, &columns, (rows, cols), "before the first op")?; - - for (step, op) in ops.iter().enumerate() { - match op { - Op::Insert(k, w) => { - sketch.update(&keys[*k as usize], *w as f64); - model[*k as usize] += *w; - } - Op::Merge(stream) => { - let other = cms_stream_of(rows, cols, &keys, stream); - sketch.merge(&other).expect("a peer of the same geometry merges"); - for (k, w) in stream { - model[*k as usize] += *w; - } - } - Op::ApplyDelta(stream) => { - let other = cms_stream_of(rows, cols, &keys, stream); - let mut merged = sketch.clone(); - merged.merge(&other).expect("a peer of the same geometry merges"); - - sketch.apply_delta(&delta_of(&other)).expect("an in-range delta applies"); - for (k, w) in stream { - model[*k as usize] += *w; - } - - prop_assert_eq!( - sketch.sketch(), merged.sketch(), - "step {}: apply_delta and the merge it stands for left different matrices", step - ); - } - Op::RoundTrip => { - let bytes = sketch.to_msgpack().expect("encode"); - let restored = PortableCountMinSketch::from_msgpack(&bytes).expect("decode"); - prop_assert_eq!( - restored.sketch(), sketch.sketch(), - "step {}: the decoded matrix differs", step - ); - sketch = restored; - } - Op::MergeMismatched(over_rows, over_cols) => { - let before = sketch.sketch(); - let peer = PortableCountMinSketch::new(rows + over_rows, cols + over_cols); - prop_assert!( - sketch.merge(&peer).is_err(), - "step {}: merging a {}x{} peer into a {}x{} sketch was accepted", - step, peer.rows(), peer.cols(), rows, cols - ); - prop_assert_eq!( - sketch.sketch(), before, - "step {}: a refused merge still changed the matrix", step - ); - } - } - - holds_model( - &sketch, &model, &keys, &columns, (rows, cols), - &format!("step {}, {:?}", step, op), - )?; - } - } - // ===== merge_max, on the disjoint key sets it documents ===== // // Distinct keys still share cells, so the maxed matrix is not the summed diff --git a/tests/pbt/count_sketch.rs b/tests/pbt/count_sketch.rs index 168651d7..0222bef6 100644 --- a/tests/pbt/count_sketch.rs +++ b/tests/pbt/count_sketch.rs @@ -1,13 +1,10 @@ -//! Property tests for Count Sketch: the native matrix sketch and the portable -//! `CountSketch` that carries it over the wire. +//! Property tests for Count Sketch. //! //! Charikar, Chen, Farach-Colton, ICALP '02. use crate::support::{ - PROBE_KEYS, edge_dimension, edge_rows, grid, keyed_updates, keys, matrices_close_at_mass, + PROBE_KEYS, edge_dimension, edge_rows, grid, keys, }; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::countsketch::CountSketch as PortableCountSketch; use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; use proptest::prelude::*; @@ -29,30 +26,11 @@ fn weighted(max: usize) -> impl Strategy> { prop::collection::vec((0u64..512, 1i32..8), 0..max) } -fn cs_of(rows: usize, cols: usize, updates: &[(String, f64)]) -> PortableCountSketch { - let mut s = PortableCountSketch::new(rows, cols); - for (k, v) in updates { - s.update(k, *v); - } - s -} - /// Signed weights small enough that no `i32` cell can overflow. fn signed_weights(max: usize) -> impl Strategy> { prop::collection::vec((0u64..512, -1_000i32..1_000), 0..max) } -/// Integral weights, so every portable cell stays exact in `f64`. -fn integral_updates(max: usize) -> impl Strategy> { - prop::collection::vec(("[a-z]{1,4}", -1_000i32..1_000), 0..max) - .prop_map(|v| v.into_iter().map(|(k, w)| (k, f64::from(w))).collect()) -} - -/// Power-of-two widths the packed-64 hash carries at these row counts. -fn packed_cols() -> impl Strategy { - prop_oneof![Just(2usize), Just(4), Just(8), Just(16), Just(32)] -} - /// The cell and sign a key holds in each row, read off a sketch holding that /// key alone. fn signature(rows: usize, cols: usize, key: u64) -> Vec<(usize, i32)> { @@ -134,44 +112,8 @@ proptest! { prop_assert_eq!(cs_cells(&by_weight), cs_cells(&by_repeat)); } - // ===== Merge algebra, on the portable representation ===== - - #[test] - fn count_sketch_merge_matches_streaming_the_concatenation( - rows in 1usize..6, - cols in 4usize..64, - a in keyed_updates(30), - b in keyed_updates(30), - ) { - let mut merged = cs_of(rows, cols, &a); - merged.merge(&cs_of(rows, cols, &b)).expect("merge"); - - let concatenated: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); - let streamed = cs_of(rows, cols, &concatenated); - let mass: f64 = concatenated.iter().map(|(_, w)| w.abs()).sum(); - - prop_assert!(matrices_close_at_mass(merged.sketch(), streamed.sketch(), mass)); - } - // ===== Wire ===== - #[test] - fn count_sketch_round_trip_preserves_every_cell( - rows in 1usize..8, - cols in 1usize..128, - updates in keyed_updates(40), - ) { - let s = cs_of(rows, cols, &updates); - - let bytes = s.to_msgpack().expect("encode"); - let restored = PortableCountSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.sketch(), s.sketch()); - for (k, _) in &updates { - prop_assert_eq!(restored.estimate(k), s.estimate(k), "key {}", k); - } - } - #[test] fn count_sketch_round_trips_at_edge_geometries( rows in edge_rows(), @@ -333,48 +275,4 @@ proptest! { ); } } - - // ===== Cells are linear in the stream ===== - - #[test] - fn cells_of_two_streams_add_to_the_cells_of_their_concatenation( - rows in 1usize..6, - cols in packed_cols(), - a in integral_updates(30), - b in integral_updates(30), - ) { - let left = cs_of(rows, cols, &a); - let right = cs_of(rows, cols, &b); - let concatenated: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); - let both = cs_of(rows, cols, &concatenated); - - for r in 0..rows { - for c in 0..cols { - prop_assert_eq!( - left.sketch()[r][c] + right.sketch()[r][c], both.sketch()[r][c], - "cell ({}, {})", r, c - ); - } - } - } - - #[test] - fn negating_every_weight_negates_every_cell( - rows in 1usize..6, - cols in packed_cols(), - updates in integral_updates(40), - ) { - let forward = cs_of(rows, cols, &updates); - let negated: Vec<_> = updates.iter().map(|(k, v)| (k.clone(), -v)).collect(); - let backward = cs_of(rows, cols, &negated); - - for r in 0..rows { - for c in 0..cols { - prop_assert_eq!( - backward.sketch()[r][c], -forward.sketch()[r][c], - "cell ({}, {})", r, c - ); - } - } - } } diff --git a/tests/pbt/ddsketch.rs b/tests/pbt/ddsketch.rs index 70a2241f..f99cad3a 100644 --- a/tests/pbt/ddsketch.rs +++ b/tests/pbt/ddsketch.rs @@ -1,5 +1,4 @@ -//! Property tests for DDSketch: the native sketch and the portable `DdSketch` -//! that carries it over the wire. +//! Property tests for DDSketch. //! //! Masson, Rim, Lee, VLDB '19. The model is the paper's relative-error //! guarantee, not a transcription of the bucket arithmetic: an answer sits @@ -7,12 +6,8 @@ use crate::support::{QUANTILES, extreme_values}; use asap_sketchlib::DdSketch; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::ddsketch::DdSketch as PortableDdSketch; use proptest::prelude::*; -const QUANTILE_PROBES: [f64; 7] = [0.0, 0.01, 0.25, 0.5, 0.75, 0.99, 1.0]; - /// First dense allocation, in buckets, and the slot the seeding value lands in. /// The Go producer's `Buckets.ensure` seeds a chunk this wide centred on the /// first index, and the `positive_store_counts` / `positive_store_offset` bytes are compared @@ -61,12 +56,6 @@ fn signed_values(max: usize) -> impl Strategy> { }) } -/// A signed stream or no stream at all: merge and the wire format both answer -/// for an empty operand. -fn maybe_signed_values(max: usize) -> impl Strategy> { - prop_oneof![1 => Just(Vec::new()), 9 => signed_values(max)] -} - fn dd_of(alpha: f64, values: &[f64]) -> DdSketch { let mut s = DdSketch::new(alpha); for v in values { @@ -75,14 +64,6 @@ fn dd_of(alpha: f64, values: &[f64]) -> DdSketch { s } -fn portable_of(alpha: f64, values: &[f64]) -> PortableDdSketch { - let mut s = PortableDdSketch::new(alpha); - for v in values { - s.update(*v); - } - s -} - fn sorted(values: &[f64]) -> Vec { let mut v = values.to_vec(); v.sort_by(|a, b| a.partial_cmp(b).expect("the generator excludes NaN")); @@ -124,24 +105,6 @@ proptest! { prop_assert_eq!(sketch.get_count(), values.len() as u64, "total count"); } - #[test] - fn ddsketch_portable_keeps_negative_and_zero_observations( - alpha in alpha(), - values in signed_values(120), - ) { - let sketch = portable_of(alpha, &values); - let negatives = values.iter().filter(|v| **v < 0.0).count() as u64; - let zeros = values.iter().filter(|v| **v == 0.0).count() as u64; - let positives = values.iter().filter(|v| **v > 0.0).count() as u64; - - prop_assert_eq!( - sketch.negative_store_counts.iter().sum::(), negatives, "negative store" - ); - prop_assert_eq!(sketch.zero_count, zeros, "zero bucket"); - prop_assert_eq!(sketch.store_counts.iter().sum::(), positives, "positive store"); - prop_assert_eq!(sketch.total_count(), values.len() as u64, "total count"); - } - /// `sum` is carried on the wire and read back through `sum()`, so it is /// the running total of the stream, not an estimate off the buckets. #[test] @@ -160,7 +123,7 @@ proptest! { } /// The first dense allocation is one `GROW_CHUNK` centred on the value's - /// bucket, in both implementations and in both signed stores. The layout + /// bucket, in both signed stores. The layout /// reaches the wire as `positive_store_counts` / `positive_store_offset`, where the Go /// producer's bytes have to match it for the same input. #[test] @@ -169,13 +132,10 @@ proptest! { value in 1e-6f64..1e9, ) { let native = dd_of(alpha, &[value, -value]); - let portable = portable_of(alpha, &[value, -value]); for (label, counts) in [ ("native positive", native.positive_store_counts()), ("native negative", native.negative_store_counts()), - ("portable positive", portable.store_counts.as_slice()), - ("portable negative", portable.negative_store_counts.as_slice()), ] { prop_assert_eq!(counts.len(), GROW_CHUNK, "{} store width", label); prop_assert_eq!(counts[GROW_CHUNK_CENTRE], 1, "{} store centre", label); @@ -260,59 +220,8 @@ proptest! { } } - /// The portable twin answers the floor(q*(n-1)) order statistic, and it - /// carries no min/max, so the bound applies at every q including 0 and 1. - #[test] - fn ddsketch_portable_signed_answers_stay_within_alpha_of_the_exact_order_statistic( - alpha in alpha(), - values in signed_values(50), - ) { - let sketch = portable_of(alpha, &values); - let ordered = sorted(&values); - - for q in probe_quantiles(40) { - let want = ordered[(q * (ordered.len() - 1) as f64).floor() as usize]; - let got = sketch.quantile(q).expect("a non-empty sketch answers"); - prop_assert!( - (got - want).abs() <= want.abs() * (alpha + 1e-12), - "alpha={} q={} n={}: got {}, exact order statistic {}", - alpha, q, ordered.len(), got, want - ); - } - } - // ===== Merge ===== - #[test] - fn ddsketch_merge_is_count_additive_and_commutative( - alpha in alpha(), - a in maybe_signed_values(40), - b in maybe_signed_values(40), - ) { - let mut ab = portable_of(alpha, &a); - ab.merge(&portable_of(alpha, &b)).expect("merge"); - let mut ba = portable_of(alpha, &b); - ba.merge(&portable_of(alpha, &a)).expect("merge"); - - prop_assert_eq!(ab.total_count(), (a.len() + b.len()) as u64); - prop_assert_eq!(ab.total_count(), ba.total_count()); - prop_assert_eq!(&ab.store_counts, &ba.store_counts); - prop_assert_eq!(ab.store_offset, ba.store_offset); - prop_assert_eq!(&ab.negative_store_counts, &ba.negative_store_counts); - prop_assert_eq!(ab.negative_store_offset, ba.negative_store_offset); - prop_assert_eq!(ab.zero_count, ba.zero_count); - prop_assert_eq!( - ab.zero_count, - a.iter().chain(&b).filter(|v| **v == 0.0).count() as u64, - "the merged zero bucket" - ); - prop_assert_eq!( - ab.negative_store_counts.iter().sum::(), - a.iter().chain(&b).filter(|v| **v < 0.0).count() as u64, - "the merged negative store" - ); - } - /// Merging keeps the paper's guarantee against the combined stream: the /// merged answer is still within alpha of the union's order statistic. #[test] @@ -341,8 +250,8 @@ proptest! { } /// Two alphas are two index mappings, so the bucket indices of one sketch - /// mean nothing under the other's gamma. Both implementations refuse the - /// merge and leave the receiver as it was. + /// mean nothing under the other's gamma. The merge is refused and leaves + /// the receiver as it was. #[test] fn ddsketch_merge_refuses_a_different_index_mapping( alpha in alpha(), @@ -358,13 +267,6 @@ proptest! { ); prop_assert_eq!(native.get_count(), values.len() as u64, "refused merge changed the count"); - let mut portable = portable_of(alpha, &values); - prop_assert!( - portable.merge(&portable_of(other_alpha, &values)).is_err(), - "portable alpha {} merged a sketch built with alpha {}", alpha, other_alpha - ); - prop_assert_eq!(portable.total_count(), values.len() as u64, "refused merge changed the count"); - prop_assert!( dd_of(alpha, &values).merge(&dd_of(alpha, &values)).is_ok(), "a shared mapping must still merge" @@ -373,27 +275,6 @@ proptest! { // ===== Wire ===== - #[test] - fn ddsketch_round_trip_preserves_buckets_and_quantiles( - alpha in alpha(), - values in maybe_signed_values(120), - ) { - let s = portable_of(alpha, &values); - - let bytes = s.to_msgpack().expect("encode"); - let restored = PortableDdSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.total_count(), s.total_count()); - prop_assert_eq!(&restored.store_counts, &s.store_counts); - prop_assert_eq!(restored.store_offset, s.store_offset); - prop_assert_eq!(&restored.negative_store_counts, &s.negative_store_counts); - prop_assert_eq!(restored.negative_store_offset, s.negative_store_offset); - prop_assert_eq!(restored.zero_count, s.zero_count); - for q in QUANTILE_PROBES { - prop_assert_eq!(restored.quantile(q), s.quantile(q), "q={}", q); - } - } - #[test] fn ddsketch_round_trips_under_extreme_values( alpha in prop_oneof![ diff --git a/tests/pbt/hll.rs b/tests/pbt/hll.rs index c19e9162..a91e0c04 100644 --- a/tests/pbt/hll.rs +++ b/tests/pbt/hll.rs @@ -20,15 +20,9 @@ //! `impl` block, so the two variants share one //! monomorphization. Estimation expands over both. //! -//! The merge laws cover both representations: `HyperLogLogImpl` itself and -//! `HllSketch`, the portable one with a register array of its own. -//! //! Flajolet, Fusy, Gandouet, Meunier, AofA '07. use crate::support::keys; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::hll::HllSketch; -use asap_sketchlib::message_pack_format::portable::hll::HllVariant; use asap_sketchlib::sketches::hll::{HyperLogLogHipImpl, HyperLogLogImpl}; use asap_sketchlib::{Classic, DataInput, Ertl, HyperLogLog}; use proptest::prelude::*; @@ -448,14 +442,20 @@ macro_rules! hip_laws { const M: usize = <$buckets>::NUM_REGISTERS; const P: u32 = <$buckets>::PRECISION as u32; - /// The registers, read back through the wire format. The struct + /// The registers, read back from the ASAPv1 payload. The struct /// holds them privately behind the `kxq0`/`kxq1` accumulators, and /// those accumulators are what the increment law checks, so the /// envelope is the one view of the registers that stays independent /// of it. fn registers_of(sketch: &S) -> Vec { - let bytes = sketch.to_msgpack().expect("encode"); - HllSketch::from_msgpack(&bytes).expect("decode").registers + let bytes = sketch.to_asapv1().expect("encode"); + let at = 8 + bytes[7] as usize; + let metadata_len = + u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) as usize; + let payload = &bytes[at + 8 + metadata_len..]; + let (registers, _, _, _): (serde_bytes::ByteBuf, f64, f64, f64) = + rmp_serde::from_slice(payload).expect("decode"); + registers.into_vec() } #[test] @@ -567,99 +567,9 @@ hip_laws!(hip_p6, BucketsP6); hip_laws!(hip_p8, BucketsP8); hip_laws!(hip_p10, BucketsP10); -fn byte_items(max: usize) -> impl Strategy>> { - prop::collection::vec(prop::collection::vec(any::(), 1..16), 0..max) -} - -fn portable_hll_of(precision: u32, items: &[Vec]) -> HllSketch { - let mut s = HllSketch::new(HllVariant::Regular, precision); - for i in items { - s.update(i); - } - s -} - -proptest! { - // ===== Merge ===== - - #[test] - fn hll_register_merge_is_commutative_and_idempotent( - precision in 4u32..12, - a in byte_items(50), - b in byte_items(50), - ) { - let mut ab = portable_hll_of(precision, &a); - ab.merge(&portable_hll_of(precision, &b)).expect("merge"); - let mut ba = portable_hll_of(precision, &b); - ba.merge(&portable_hll_of(precision, &a)).expect("merge"); - - prop_assert_eq!(&ab.registers, &ba.registers); - - let before = ab.registers.clone(); - let again = ab.clone(); - ab.merge(&again).expect("merge"); - prop_assert_eq!(&ab.registers, &before); - } - - #[test] - fn hll_merge_equals_streaming_the_concatenation( - precision in 4u32..12, - a in byte_items(50), - b in byte_items(50), - ) { - let mut merged = portable_hll_of(precision, &a); - merged.merge(&portable_hll_of(precision, &b)).expect("merge"); - - let concatenated: Vec<_> = a.iter().chain(b.iter()).cloned().collect(); - let streamed = portable_hll_of(precision, &concatenated); - - prop_assert_eq!(&merged.registers, &streamed.registers); - } - - #[test] - fn hll_empty_is_a_merge_identity( - precision in 4u32..12, - a in byte_items(50), - ) { - let base = portable_hll_of(precision, &a); - let mut merged = base.clone(); - merged.merge(&HllSketch::new(HllVariant::Regular, precision)).expect("merge"); - - prop_assert_eq!(&merged.registers, &base.registers); - } -} - -fn hll_variant() -> impl Strategy { - prop_oneof![ - Just(HllVariant::Regular), - Just(HllVariant::Datafusion), - Just(HllVariant::Hip), - ] -} - proptest! { // ===== Wire ===== - #[test] - fn hll_round_trip_preserves_registers( - variant in hll_variant(), - precision in 4u32..14, - items in byte_items(64), - ) { - let mut s = HllSketch::new(variant, precision); - for item in &items { - s.update(item); - } - - let bytes = s.to_msgpack().expect("encode"); - let restored = HllSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.variant, s.variant); - prop_assert_eq!(restored.precision, s.precision); - prop_assert_eq!(&restored.registers, &s.registers); - prop_assert_eq!(restored.estimate(), s.estimate()); - } - #[test] fn hyperloglog_round_trips_including_the_empty_sketch(stream in keys(400)) { type Hll = HyperLogLog; diff --git a/tests/pbt/hydra.rs b/tests/pbt/hydra.rs index bb628d37..8e20ec4a 100644 --- a/tests/pbt/hydra.rs +++ b/tests/pbt/hydra.rs @@ -20,20 +20,13 @@ //! subkey names. Everything else about a subpopulation answer — that it never //! falls below the records it counts, that a generalization never reads below //! its specialization — follows from those two. -//! -//! `HydraKllSketch` is a second, unrelated grid in the portable module: no -//! schema and no fan-out, one string key routed by `xxh32(key, row)` and a -//! median of the row KLLs. Its accuracy is characterized in the end-to-end -//! quantile battery; the two laws here pin its routing and its reduction. use asap_sketchlib::common::input::{HydraCounter, HydraQuery}; -use asap_sketchlib::message_pack_format::portable::hydra_kll::HydraKllSketch; use asap_sketchlib::{ CountMinSketch, DataInput, FastPath, HYDRA_SEED_INDEX, Hydra, MatrixFastHash, Vector2D, hash_for_matrix_seeded, }; use proptest::prelude::*; -use xxhash_rust::xxh32::xxh32; /// The value every record carries. const PAYLOAD: &str = "pkt"; @@ -263,37 +256,6 @@ fn grid_cols() -> impl Strategy { ] } -/// The compaction-coin seed every portable grid is built with, so a cell's -/// answer is a function of its stream alone. -const KLL_SEED: u64 = 0x5EED_2000; - -/// Wide enough that the streams below are retained whole, so no cell compacts. -const KLL_K: u16 = 256; - -/// Keys the portable grid is streamed and probed with. Few and short, so a -/// narrow grid makes them share cells. -const KLL_KEYS: [&str; 5] = ["a", "b", "c", "", "a:b"]; - -/// The column a portable key occupies on each row. -fn kll_columns(rows: usize, cols: usize, key: &str) -> Vec { - (0..rows) - .map(|row| xxh32(key.as_bytes(), row as u32) as usize % cols) - .collect() -} - -/// A stream of `(key index, value)` for the portable grid. -fn kll_stream(max: usize) -> impl Strategy> { - prop::collection::vec((0usize..KLL_KEYS.len(), 0.0f64..1000.0), 1..max) -} - -fn kll_filled(rows: usize, cols: usize, stream: &[(usize, f64)]) -> HydraKllSketch { - let mut grid = HydraKllSketch::with_seed(rows, cols, KLL_K, KLL_SEED); - for (key, value) in stream { - grid.update(KLL_KEYS[*key], *value); - } - grid -} - proptest! { // ===== Each subpopulation lands in the cell its labelled subkey names ===== // @@ -418,73 +380,6 @@ proptest! { } } - // ===== The portable grid routes each row by its own seeded hash ========= - // - // `HydraKllSketch` carries no schema and does not fan out: one key reaches - // one cell per row, at `xxh32(key, row) % cols`. The oracle counts the - // stream into that map and reads each cell's retained count back, so a row - // seeded alike as its neighbours — or a routing that ignores the row — is - // an unequal count rather than a merely worse estimate. - - #[test] - fn portable_hydra_kll_routes_each_update_to_the_cell_its_row_hash_names( - rows in 1usize..5, - cols in 1usize..9, - stream in kll_stream(24), - ) { - let grid = kll_filled(rows, cols, &stream); - - let mut oracle = vec![vec![0u64; cols]; rows]; - for (key, _) in &stream { - for (row, col) in kll_columns(rows, cols, KLL_KEYS[*key]).into_iter().enumerate() { - oracle[row][col] += 1; - } - } - - for (r, row) in oracle.iter().enumerate() { - for (c, want) in row.iter().enumerate() { - prop_assert_eq!( - grid.sketch[r][c].count(), *want, - "cell ({}, {}) of a {}x{} portable grid", r, c, rows, cols - ); - } - } - } - - // ===== The portable grid answers a median of its row cells ============== - // - // As above, an order statistic: the tie rule for an even row count is not - // part of the contract. An empty cell answers 0, which is below every value - // the stream carries, so a query for an absent key is a real probe of the - // reduction rather than a vacuous one. - - #[test] - fn portable_hydra_kll_answers_a_median_of_its_row_cells( - rows in 1usize..5, - cols in 1usize..9, - stream in kll_stream(24), - q in 0.0f64..=1.0, - ) { - let grid = kll_filled(rows, cols, &stream); - - for key in KLL_KEYS { - let row_estimates: Vec = kll_columns(rows, cols, key) - .into_iter() - .enumerate() - .map(|(r, c)| grid.sketch[r][c].quantile(q)) - .collect(); - - let seen = grid.quantile(key, q); - let at_or_below = row_estimates.iter().filter(|e| **e <= seen).count(); - let at_or_above = row_estimates.iter().filter(|e| **e >= seen).count(); - prop_assert!( - 2 * at_or_below >= rows && 2 * at_or_above >= rows, - "key {:?} at q {} read {}, no median of its row estimates {:?}", - key, q, seen, row_estimates - ); - } - } - // ===== Wire ===== // // A grid of counters cut to size costs no more to build than any other diff --git a/tests/pbt/kll.rs b/tests/pbt/kll.rs index 831be95a..0881c769 100644 --- a/tests/pbt/kll.rs +++ b/tests/pbt/kll.rs @@ -1,5 +1,4 @@ -//! Property tests for KLL: the native sketch and the portable `KllSketch` -//! that carries it over the wire. +//! Property tests for KLL. //! //! Karnin, Lang, Liberty, FOCS '16. Compaction is randomized, so a law may //! not name the items it keeps. What survives: a quantile is monotone in q, @@ -13,12 +12,8 @@ use crate::support::{QUANTILES, close, extreme_values}; use asap_sketchlib::Kll; -use asap_sketchlib::message_pack_format::MessagePackCodec; -use asap_sketchlib::message_pack_format::portable::kll::KllSketch; use proptest::prelude::*; -const QUANTILE_PROBES: [f64; 7] = [0.0, 0.01, 0.25, 0.5, 0.75, 0.99, 1.0]; - /// Compaction empties level 0 once it holds more than `k` items, so a stream /// this much longer than `k` has compacted many times over and carries items /// at several weights. @@ -90,12 +85,8 @@ fn top_weight(s: &Kll) -> f64 { (1u64 << (s.num_levels() - 1)) as f64 } -fn kll_of(k: u16, values: &[f64]) -> KllSketch { - let mut s = KllSketch::with_seed(k, 0xC0FFEE); - for v in values { - s.update(*v); - } - s +fn kll_of(k: u16, values: &[f64]) -> Kll { + native_kll(i32::from(k), 0xC0FFEE, values) } proptest! { @@ -107,7 +98,7 @@ proptest! { values in prop::collection::vec(-1e5f64..1e5, 0..8), ) { let s = kll_of(k, &values); - prop_assert_eq!(s.count(), values.len() as u64); + prop_assert_eq!(s.count(), values.len()); } #[test] @@ -117,7 +108,7 @@ proptest! { b in prop::collection::vec(-1e5f64..1e5, 0..400), ) { let mut merged = kll_of(k, &a); - merged.merge(&kll_of(k, &b)).expect("merge"); + merged.merge(&kll_of(k, &b)); let n = (a.len() + b.len()) as f64; let got = merged.count() as f64; @@ -160,26 +151,6 @@ proptest! { // ===== Wire ===== - #[test] - fn kll_round_trip_preserves_count_and_quantiles( - k in 8u16..256, - values in prop::collection::vec(-1e6f64..1e6, 0..300), - ) { - let mut s = KllSketch::with_seed(k, 0x5EED); - for v in &values { - s.update(*v); - } - - let bytes = s.to_msgpack().expect("encode"); - let restored = KllSketch::from_msgpack(&bytes).expect("decode"); - - prop_assert_eq!(restored.k(), s.k()); - prop_assert_eq!(restored.count(), s.count()); - for q in QUANTILE_PROBES { - prop_assert_eq!(restored.quantile(q), s.quantile(q), "q={}", q); - } - } - #[test] fn kll_round_trips_under_extreme_values( k in prop_oneof![Just(1i32), Just(8), Just(64), Just(200), Just(1_024)], diff --git a/tests/pbt/set_aggregator.rs b/tests/pbt/set_aggregator.rs index ebec4f23..41e5f606 100644 --- a/tests/pbt/set_aggregator.rs +++ b/tests/pbt/set_aggregator.rs @@ -4,7 +4,6 @@ //! is a `std::collections::HashSet` built from the stream on the test side. use asap_sketchlib::SetAggregator; -use asap_sketchlib::message_pack_format::MessagePackCodec; use proptest::prelude::*; use std::collections::HashSet; @@ -170,17 +169,17 @@ proptest! { // ===== Wire ===== #[test] - fn a_msgpack_round_trip_preserves_the_key_set(stream in awkward_keys(120)) { + fn an_asapv1_round_trip_preserves_the_key_set(stream in awkward_keys(120)) { let s = aggregator_of(&stream); - let bytes = s.to_msgpack().expect("encode"); - let restored = SetAggregator::from_msgpack(&bytes).expect("decode"); + let bytes = s.to_asapv1().expect("encode"); + let restored = SetAggregator::from_asapv1(&bytes).expect("decode"); prop_assert_eq!(&restored.values, &s.values, "round trip moved the set"); prop_assert_eq!(restored.values.len(), s.values.len()); - let twice = SetAggregator::from_msgpack( - &restored.to_msgpack().expect("re-encode") + let twice = SetAggregator::from_asapv1( + &restored.to_asapv1().expect("re-encode") ).expect("decode a re-encode"); prop_assert_eq!(&twice.values, &s.values, "the second round trip moved the set"); } From 708e2e3ba9184e8d54921380975bb993c5d73809 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:36:20 +0000 Subject: [PATCH 3/4] refactor!: remove serde derives from sketch types (#208) Sketches, their storage (`Vector1D`/`2D`/`3D`, `BitMatrix`, `CommonHeap`, the fixed-structure macros) and their inputs stop implementing `Serialize` and `Deserialize`; serde stays on the ASAPv1 metadata and payload types. `CountMinHll` and `MicroCm` lose `serialize_to_bytes` / `deserialize_from_bytes`, and `serde-big-array` is dropped. KMV's decoder and the golden tests that built known states through serde now construct them directly; the known-state encode checks move into each sketch's `wire.rs`. Raw-serde tests go; tests of shared decode paths move to ASAPv1 or the rebuild they exercised. Closes #208 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + Cargo.toml | 1 - docs/api/api_bloom.md | 8 - docs/api/api_coco_sketch.md | 2 +- docs/api/api_common_heap.md | 4 +- docs/api/api_common_input.md | 4 +- docs/api/api_countminsketch_hll.md | 8 +- docs/api/api_ddsketch.md | 4 - docs/api/api_eh_bucket_sketch.md | 2 +- docs/api/api_elastic_sketch.md | 2 +- docs/api/api_hydra.md | 4 +- docs/api/api_microscope.md | 9 +- docs/api/api_space_saving.md | 3 +- docs/api/api_uniform_sampling.md | 2 - docs/asapv1_wire_format.md | 2 +- docs/features.md | 51 +++-- docs/tests.md | 56 ++--- src/asapv1/admission_rate.rs | 12 +- src/common/hash.rs | 5 +- src/common/heap.rs | 117 +---------- src/common/input.rs | 10 +- src/common/structure_utils.rs | 18 +- src/common/structures/bit_matrix.rs | 108 ++-------- src/common/structures/fixed_structure.rs | 89 +------- src/common/structures/heap.rs | 16 +- src/common/structures/vector1d.rs | 3 +- src/common/structures/vector2d.rs | 36 +--- src/common/structures/vector3d.rs | 158 +------------- src/lib.rs | 6 - src/sketch_framework/eh_bucket_sketch.rs | 15 +- src/sketch_framework/hydra.rs | 9 +- src/sketch_framework/nitro.rs | 12 +- src/sketch_framework/univmon.rs | 7 +- src/sketch_framework/univmon_q.rs | 6 +- src/sketches/bloom.rs | 73 ------- src/sketches/bloom/wire.rs | 24 +++ src/sketches/coco.rs | 8 +- src/sketches/count_min_fold.rs | 70 +----- src/sketches/count_sketch_fold.rs | 65 +----- src/sketches/countminsketch.rs | 9 +- src/sketches/countminsketch_hll.rs | 208 +----------------- src/sketches/countsketch.rs | 9 +- src/sketches/countsketch_l2_norm.rs | 5 +- src/sketches/countsketch_l2_norm/wire.rs | 20 ++ src/sketches/ddsketch.rs | 143 +------------ src/sketches/ddsketch/wire.rs | 43 ++++ src/sketches/elastic.rs | 8 +- src/sketches/hll.rs | 17 +- src/sketches/kll.rs | 142 +------------ src/sketches/kll_dynamic.rs | 9 +- src/sketches/kmv.rs | 5 +- src/sketches/kmv/wire.rs | 42 +--- src/sketches/microscope.rs | 257 +---------------------- src/sketches/microscope/cell.rs | 48 +---- src/sketches/mode.rs | 5 +- src/sketches/space_saving.rs | 54 ++--- src/sketches/space_saving/wire.rs | 36 ++++ src/sketches/uniform.rs | 5 +- src/sketches/uniform/wire.rs | 23 ++ tests/asapv1_golden.rs | 113 ++++------ tests/ddsketch_signed.rs | 4 +- tests/e2e/heavy_hitters.rs | 106 +--------- tests/e2e/membership.rs | 33 ++- tests/e2e/nitro.rs | 155 +------------- tests/hll_custom_precision.rs | 31 +-- tests/pbt/count_min_hll.rs | 16 -- tests/pbt/microscope.rs | 87 +------- tests/pbt/support.rs | 14 +- 68 files changed, 422 insertions(+), 2259 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee5bf2dc..b6c88904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -341,6 +341,11 @@ signals a backwards-compatible change. `HydraKllSketch`, their deltas and helpers, and the `xxhash-rust` dependency. `SetAggregator` and `SetAggregatorDelta` move to `sketches::set_aggregator`; the ASAPv1 framing is private. +- **serde on sketch types** (breaking). Sketches, their storage and their + inputs no longer implement `Serialize` / `Deserialize`; ASAPv1 is the one + serialized form. `CountMinHll` and `MicroCm` lose + `serialize_to_bytes` / `deserialize_from_bytes`, and the + `serde-big-array` dependency goes. - **Dead and duplicate APIs** (breaking). `Kll::wire_k` (use `k()`); `PRECOMPUTED_SAMPLE_RATE_1PERCENT` and module `common::precompute_sample2`; module `common::hashspec` (`HashSpec`, `derive_index`, `derive_sign`, diff --git a/Cargo.toml b/Cargo.toml index dbafd6ae..77489e38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ all-features = true [dependencies] rand = "0.9.3" serde = { version = "1.0", features = ["derive"] } -serde-big-array = "0.5" serde_bytes = "0.11" rmp-serde = "1.3.0" twox-hash = "2.1.2" diff --git a/docs/api/api_bloom.md b/docs/api/api_bloom.md index 05d9cae1..64dbf5e3 100644 --- a/docs/api/api_bloom.md +++ b/docs/api/api_bloom.md @@ -120,14 +120,6 @@ word count that disagrees with the declared dimensions, and any bit set in a row's trailing padding past `cols` — unreachable by `contains`, but counted by `count_ones`, so it would skew `fill_ratio` and `estimated_fpp` alone. -Independently of ASAPv1, `Bloom` is plain `Serialize`/`Deserialize`; that form is -`{ bits, inserted, mode }`, where `mode` is `"regular"` or `"fast"`. Decoding -into the other hash path fails rather than producing a filter that reports its -own members absent, and so does a row count past `BLOOM_MAX_SLICES`. `BitMatrix` -carries `{ words, rows, cols }` only — the word stride and the fold masks are -recomputed on decode, and a payload whose word count disagrees with its -dimensions is rejected there rather than panicking later. - ## Examples ```rust diff --git a/docs/api/api_coco_sketch.md b/docs/api/api_coco_sketch.md index fe22065a..cf5e1466 100644 --- a/docs/api/api_coco_sketch.md +++ b/docs/api/api_coco_sketch.md @@ -113,7 +113,7 @@ that holds nothing. Both methods require `H: HashProfile`. A zero dimension, a table whose bucket count disagrees with `rows`/`cols`, a payload length that disagrees with the declared geometry, and a `nil` key carrying a -non-zero value are all rejected on both sides. `CocoSketch` also derives serde. +non-zero value are all rejected on both sides. Row `i` hashes at seed index `i`, so the wire covers `1 <= d <= 20` (`MATRIX_MAX_ROWS`, the seed list length): past that, row `i` and row `i + 20` diff --git a/docs/api/api_common_heap.md b/docs/api/api_common_heap.md index 58f3eefd..40717c2c 100644 --- a/docs/api/api_common_heap.md +++ b/docs/api/api_common_heap.md @@ -52,8 +52,8 @@ No dedicated merge method on `TopKHeap`; reconciliation is sketch-specific. ## Serialization -No dedicated byte API helpers. Derived serde carries the heap and its capacity; -the key index is derived data and is rebuilt on load rather than stored. +None of its own; a sketch holding a `TopKHeap` carries its entries in its +ASAPv1 payload. ## Examples diff --git a/docs/api/api_common_input.md b/docs/api/api_common_input.md index 0d569b29..3eb81345 100644 --- a/docs/api/api_common_input.md +++ b/docs/api/api_common_input.md @@ -61,8 +61,8 @@ fn merge(&mut self, other: &HydraCounter) -> Result<(), String> ## Serialization -- `DataInput`, `OwnedDataInput`, `HeapEntry`, and `HydraCounter` derive serde traits. -- `HydraQuery` is query-only and does not derive serde in current code. +None of these types implements serde's `Serialize` / `Deserialize`; a sketch +holding them serializes through its own `to_asapv1`. ## Variants (Core) diff --git a/docs/api/api_countminsketch_hll.md b/docs/api/api_countminsketch_hll.md index f29c9f7c..482b8f79 100644 --- a/docs/api/api_countminsketch_hll.md +++ b/docs/api/api_countminsketch_hll.md @@ -79,13 +79,7 @@ returns an error otherwise, leaving `self` untouched. ## Serialization -```rust -fn serialize_to_bytes(&self) -> Result, RmpEncodeError> -fn deserialize_from_bytes(bytes: &[u8]) -> Result -``` - -Plain MessagePack, **not** an ASAPv1 envelope: no `kind_id` is allocated for -this sketch while it is experimental. +None: no `kind_id` is allocated for this sketch while it is experimental. ## Accuracy diff --git a/docs/api/api_ddsketch.md b/docs/api/api_ddsketch.md index a5c816b2..625d4bb8 100644 --- a/docs/api/api_ddsketch.md +++ b/docs/api/api_ddsketch.md @@ -70,10 +70,6 @@ past `i32`, a non-zero offset on an empty store, counts that overflow the total, and scalars inconsistent with that total; `to_asapv1` enforces the same rules, so the format never emits bytes it would refuse to read back. -Independently of ASAPv1, `DdSketch` is plain `Serialize`/`Deserialize` for use -where it is nested in another type (`EhBucketSketch`); that form skips the four -running scalars. - ## Deltas ```rust diff --git a/docs/api/api_eh_bucket_sketch.md b/docs/api/api_eh_bucket_sketch.md index 03dc5d6b..eb5784a0 100644 --- a/docs/api/api_eh_bucket_sketch.md +++ b/docs/api/api_eh_bucket_sketch.md @@ -83,7 +83,7 @@ and `0x01 0x03` (HIP) are rejected, and `Kll` is compact, so `0x06 0x01` (dynamic) is rejected. The dispatch is the same in every build: a decoder built without `experimental` rejects `0x0d 0x00` with an error naming the variant and the feature, and its encoder can never emit it. -An unrecognized kind_id is rejected. `EhBucketSketch` also derives serde. +An unrecognized kind_id is rejected. ## Examples diff --git a/docs/api/api_elastic_sketch.md b/docs/api/api_elastic_sketch.md index 5fab26f8..06d84976 100644 --- a/docs/api/api_elastic_sketch.md +++ b/docs/api/api_elastic_sketch.md @@ -343,7 +343,7 @@ both sides past it. A zero dimension, a `heavy_buckets` disagreeing with the heavy table, a free bucket that still names a flow, a payload length that disagrees with the declared geometry, and a `nil` flow id whose `vote_pos` is non-zero are all rejected on -both sides. `ElasticSketch` also derives serde. +both sides. ## Examples diff --git a/docs/api/api_hydra.md b/docs/api/api_hydra.md index 69fea4ac..30b0f5c3 100644 --- a/docs/api/api_hydra.md +++ b/docs/api/api_hydra.md @@ -75,9 +75,7 @@ A grid mixing counter variants, UnivMon cells mixing key variants, a cell whose geometry differs from the prototype's, a `prototype` holding data, a declared grid the storage does not match, a row count past `MATRIX_MAX_ROWS`, and a payload length that disagrees with the declared geometry are all rejected -on both sides. `Hydra` also derives -serde, which is the nested codec used when a Hydra is embedded in a larger -serde value. +on both sides. ## Examples diff --git a/docs/api/api_microscope.md b/docs/api/api_microscope.md index 8dea65fd..6fa70a96 100644 --- a/docs/api/api_microscope.md +++ b/docs/api/api_microscope.md @@ -111,14 +111,7 @@ clocks would add unrelated sub-windows. `self` is untouched on error. ## Serialization -```rust -fn serialize_to_bytes(&self) -> Result, RmpEncodeError> -fn deserialize_from_bytes(bytes: &[u8]) -> Result -``` - -Plain MessagePack, **not** an ASAPv1 envelope: no `kind_id` is allocated while -this sketch is experimental. The layout is re-derived from the parameters on -load and a payload whose stored cell depth disagrees with it is refused. +None: no `kind_id` is allocated while this sketch is experimental. ## Cost diff --git a/docs/api/api_space_saving.md b/docs/api/api_space_saving.md index bb154d04..b679c38b 100644 --- a/docs/api/api_space_saving.md +++ b/docs/api/api_space_saving.md @@ -124,8 +124,7 @@ fn from_asapv1(bytes: &[u8]) -> Result These produce/consume the **ASAPv1** wire envelope (kind `0x18 0x00`) — see the [ASAPv1 wire format spec](../asapv1_wire_format.md). The impl is bounded on `H: HashProfile`, so a summary built with an unprofiled hasher cannot serialize -at all. `SpaceSaving` also implements `serde::Serialize` / `Deserialize` for -in-Rust use; both forms carry the same state. +at all. A summary travels as `capacity`, `total`, the unmonitored ceiling, and one `(key, count, error)` triple per monitored key. The bucket and counter lists and diff --git a/docs/api/api_uniform_sampling.md b/docs/api/api_uniform_sampling.md index 0346098e..133c6773 100644 --- a/docs/api/api_uniform_sampling.md +++ b/docs/api/api_uniform_sampling.md @@ -56,8 +56,6 @@ hash-spec group. The payload is `[priorities, values, total_seen, rng_state]`, emitted in ascending priority. `rng_state` is the SplitMix64 word the next priority is drawn from, so a decoded sampler continues the same draw sequence. -Also derives serde for the internal Rust-only codec. - ## Examples ```rust diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 5f81a26c..00fb1418 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -695,7 +695,7 @@ Rules 3 and 4 are enforced on the **encode** side too: a matrix past `MATRIX_MAX | 1 | `keys` | array | one key per heap entry; element type = `key_type`; homogeneous | | 2 | `heap_counts` | array | parallel to `keys`; `f64` when `counter_type` is `"f64"`, else `i64` | -**The heap's index does not reach the wire.** `TopKHeap` is a capacity-bounded min-heap beside a digest-to-position index; `slots` and `positions` are `#[serde(skip)]` in the type itself and are rebuilt from the entries on load, so the payload carries the entries only — the same argument §3.5 makes for Space-Saving's arenas, with the same benefit: with no index on the wire, no crafted payload can point one out of bounds or into a cycle. The entry count is `len(keys)` (derived, so not stored), and the heap's array order and its `is_full` state are recomputed. +**The heap's index does not reach the wire.** `TopKHeap` is a capacity-bounded min-heap beside a digest-to-position index; `slots` and `positions` are rebuilt from the entries on load, so the payload carries the entries only — the same argument §3.5 makes for Space-Saving's arenas, with the same benefit: with no index on the wire, no crafted payload can point one out of bounds or into a cycle. The entry count is `len(keys)` (derived, so not stored), and the heap's array order and its `is_full` state are recomputed. **Metadata vs payload.** Everything the payload's structure depends on is configuration and lives in the metadata, in this canonical order (hash-spec group first, then structural params; Go must mirror it verbatim): diff --git a/docs/features.md b/docs/features.md index b831e285..373eab0f 100644 --- a/docs/features.md +++ b/docs/features.md @@ -67,7 +67,7 @@ This document provides a high-level overview of implemented and planned features ### Serialization -**MessagePack (rmp-serde)** - Serialization support across most sketch types +**ASAPv1** - `to_asapv1` / `from_asapv1` on every sketch the ASAPv1 registry marks implemented ### Sampling @@ -87,31 +87,30 @@ Insertion throughput measured on 10,000,000 Zipf-distributed `int64` values (s=1 ### Serialization -MessagePack (`rmp-serde`) support. **serde support** means the type derives `Serialize`/`Deserialize` and can be used with any serde-compatible serializer. **Built-in helpers** (`to_asapv1` / `from_asapv1`) provide one-call round-tripping without requiring users to depend on `rmp-serde` directly. These helpers emit the self-describing **ASAPv1** wire envelope (see the [ASAPv1 wire format spec](./asapv1_wire_format.md)) for every sketch the spec's `kind_id` registry marks *implemented*; NitroBatch, CountMinSketchFold, CountSketchFold, HashSketchEnsemble, ExponentialHistogramUnivMon and OctoSketch are not converted. - -| Component | serde support | Built-in helpers | -| --- | --- | --- | -| CountMinSketch | Yes | Yes | -| Count / CountSketchWithL2Norm | Yes | Yes | -| HyperLogLog (all variants) | Yes | Yes | -| DDSketch | Yes | Yes | -| KLL / KllDynamic | Yes | Yes | -| KMV | Yes | Yes | -| Elastic | Yes | Yes | -| Coco | Yes | Yes | -| UniformSampling | Yes | Yes | -| CountMinSketchFold / CountSketchFold | Yes | In Progress | -| CountMinSketchWithHeap / CountSketchWithHeap | In Progress | Yes | -| CountMinSketchWithSpaceSaving / CountSketchWithSpaceSaving | No | Yes | -| SpaceSaving | Yes | Yes | -| Bloom | Yes | Yes | -| Hydra | Yes | Yes | -| UnivMon | Yes | Yes | -| Two-tier UnivMon | No | Yes | -| UnivMonQ (experimental) | Internal wire DTO | Yes | -| NitroBatch | Yes | In Progress | -| EhBucketSketch | Yes | Yes | -| ExponentialHistogram | No | Yes | +`to_asapv1` / `from_asapv1` emit and read the self-describing **ASAPv1** wire envelope (see the [ASAPv1 wire format spec](./asapv1_wire_format.md)) for every sketch the spec's `kind_id` registry marks *implemented*. Sketch types do not implement serde's `Serialize` / `Deserialize`. NitroBatch, CountMinSketchFold, CountSketchFold, CountMinHll, MicroCm, HashSketchEnsemble, ExponentialHistogramUnivMon and OctoSketch have no serialized form. + +| Component | ASAPv1 | +| --- | --- | +| CountMinSketch | Yes | +| Count / CountSketchWithL2Norm | Yes | +| HyperLogLog (all variants) | Yes | +| DDSketch | Yes | +| KLL / KllDynamic | Yes | +| KMV | Yes | +| Elastic | Yes | +| Coco | Yes | +| UniformSampling | Yes | +| CountMinSketchWithHeap / CountSketchWithHeap | Yes | +| CountMinSketchWithSpaceSaving / CountSketchWithSpaceSaving | Yes | +| SpaceSaving | Yes | +| Bloom | Yes | +| Hydra | Yes | +| UnivMon | Yes | +| Two-tier UnivMon | Yes | +| UnivMonQ (experimental) | Yes | +| EhBucketSketch | Yes | +| ExponentialHistogram | Yes | +| SetAggregator | Yes | ### API Stability diff --git a/docs/tests.md b/docs/tests.md index 453dd702..bc8d9b9d 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -165,7 +165,6 @@ Wire tests: [`src/sketches/kll/wire.rs`](../src/sketches/kll/wire.rs) | `kll_dynamic_kind_id_rejected_by_compact` | The dynamic kind_id is refused by the compact decoder. | Verifies a crafted envelope carrying valid compact metadata and payload under `0x06 0x01` fails to decode as a `Kll`. | | `cdf_cached_matches_uncached_across_lifecycle` | The memoized CDF agrees with the from-scratch rebuild at every lifecycle stage. | For `Kll::::with_seed(200, 8, 42)`, verifies `quantile_cached` equals `quantile` while empty at `{0.1, 0.5, 0.9}`, at `{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}` after 20,000 uniform samples in `[0, 1000)` (seed `0xACAC_0001`), and at `{0.1, 0.5, 0.9}` after merging a seed-43 sketch over 5,000 samples in `[-5000, -4000)` (seed `0xACAC_0002`); after `clear`, `quantile_cached(0.5)` is `0.0` and `count` is `0`. | | `cdf_cache_invalidated_by_subsequent_updates` | Updates after a cached query invalidate the cache instead of serving a stale CDF. | Takes `quantile_cached(0.5)` on a `with_seed(200, 8, 42)` sketch over 10,000 uniform samples in `[0, 100)` (seed `0xACAC_0003`), then inserts `999_999.0` 10,000 times and verifies the cached median rises by more than `50.0`. | -| `kllwire_serde_rejects_crafted_dimensions` | The nested serde decoder refuses an out-of-range `k` before it sizes anything. | Encodes a `KLLWire` with no items, `levels` of `[0, 0]`, `k` of `usize::MAX`, `m` of `8`, `num_levels` `1`, and `Coin::from_seed(1)`, and verifies it does not decode as a `Kll` rather than overflowing `compute_max_capacity`. | | `from_portable_state_reproduces_source_exactly` | Reconstruction from portable state is bit-exact, not a lossy replay. | Extracts the compacted `items` and `levels` from a `Kll::::with_k(200)` sketch over 200,000 updates of `i * 0.0007 + 3.0`, and verifies `from_portable_state` reproduces the source's quantiles at `{0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1}` exactly; an empty reconstruction is verified to report `num_levels` `1` with `levels[0]` at `max_capacity`. | | `seeded_sketches_are_byte_identical` | Same seed and same input give byte-identical sketches. | Feeds 5,000 uniform samples in `[0, 1_000_000)` (seed `7`) to two `with_k_and_seed(200, 42)` sketches and verifies their serialized bytes are equal. | | `different_seeds_produce_different_bytes` | The seed reaches `toss` rather than sitting unused on the struct. | Feeds the same 5,000 uniform samples in `[0, 1_000_000)` (seed `11`) to `with_k_and_seed(200, 1)` and `with_k_and_seed(200, 2)` and verifies their serialized bytes differ. | @@ -233,15 +232,12 @@ Delta tests: [`src/sketches/ddsketch/delta.rs`](../src/sketches/ddsketch/delta.r | `representative_within_alpha_at_bucket_edges` | Every bucket's representative is within `alpha` of both of its edges. | For `alpha` in `{0.001, 0.01, 0.05, 0.1}` and bucket indices `{-100, -1, 0, 1, 7, 500}`, verifies `bin_representative(k)` lies in `[lower_bound(k), lower_bound(k + 1)]` and its relative error against each edge is at most `alpha + 1e-9`. | | `merge_alpha_mismatch_is_a_real_runtime_error` | Merging across index mappings is an error, and a matched merge is not. | Verifies merging an `alpha = 0.02` sketch into an `alpha = 0.01` sketch holding `5.0` returns `Err`, while merging two `alpha = 0.01` sketches holding `3.0` and `7.0` returns `Ok` and leaves `get_count()` at `2`. | | `untrackable_extreme_is_dropped` | A finite value outside the indexable range is not recorded, so the dense store never spans the gap. | For an `alpha = 0.01` sketch over `1..=2000`, adds `max_indexable * 10.0` and `min_indexable / 10.0` from `ddsketch_indexable_bounds(0.01)` and verifies both `get_count()` and the store's bucket span are unchanged, then that `max_indexable / 2.0` is still recorded and raises the count by `1`. | -| `serde_round_trip_keeps_the_running_scalars` | The derived serde form carries every scalar the buckets do not determine. | Round-trips an `alpha = 0.01` sketch over `[0.25, 1, 2, 3, 10, 50, 100, 1000]` through `rmp-serde` and verifies the decoded sketch matches on `get_count`, `sum`, `min`, `max`, `alpha`, `positive_store_counts`, and `positive_store_offset`, and on quantiles at `{0, 0.25, 0.5, 0.9, 1}`. | -| `serde_round_trip_leaves_the_sketch_usable` | A decoded sketch keeps ingesting on top of the state it came back with. | Adds `5.0` to the decode of an `alpha = 0.01` sketch over `[0.25, 1, 2, 3, 10, 50, 100, 1000]` and verifies `get_count` rises by `1`, `sum` by `5.0`, and that the bucket counts still sum to `get_count`. | -| `serde_refuses_scalars_that_disagree_with_the_store` | Serde checks the scalars against the store the way the ASAPv1 decoder does. | Encodes a crafted state at `alpha = 0.01` whose store holds `[1, 2]` at offset `-3` while `sum` is `0.0` with the empty-store sentinels `+inf`/`-inf` for `min`/`max`, and verifies the decode fails with a complaint naming "DDSketch scalars". | -| `serde_refuses_an_out_of_range_alpha` | An `alpha` outside `(0, 1)` is refused on the way in, not at the first query. | Encodes a crafted state at `alpha = 1.5` with an empty store at offset `0` and the empty-store scalars, and verifies the decode fails with a complaint naming "alpha". | | `decode_rejects_each_payload_rule` | A `0x05 0x80` payload breaking any delta rule fails to decode. | At `alpha = 0.01`, verifies indices `[-35418, 35448]` (`key(min_v)`, `key(max_v)`) in both stores decode, and that index `-35419`, index `35449`, and negative-store index `35449` each fail. | | `decode_rejects_another_kind_and_new_another_alpha` | A delta decodes only under its own kind_id, and `new` checks `alpha`. | Verifies a delta envelope decodes and fails with its kind_id relabelled `0x05 0x00`, and that `new` refuses `alpha` `1.0`. | | `merge_delta_folds_the_scalars` | Applying a delta folds each representative, and `0.0` for zeros, into the scalars. | Merges a count of `2` at bucket `10` into an empty sketch and verifies `min`, `max` equal `r = 1.01 * gamma^10` and `sum` equals `2r`; then merges a `zero_count` of `3` and verifies `min` `0.0`, `zero_count` `3` and `get_count()` `5`. | | `merge_delta_rejects_without_changing_the_state` | A rejected delta leaves the sketch unchanged. | Verifies a delta whose `alpha` is one ulp above `0.01` fails on an `alpha = 0.01` sketch, and that on a sketch holding `u64::MAX - 1` samples a delta whose negative count overflows the total and a delta whose `zero_count` overflows it each fail, leaving the count and the serialized bytes as they were. | | `merge_delta_rejects_a_store_span_past_i32` | A delta that would grow a store past `i32` fails instead of overflowing. | Verifies a delta at index `-10` onto a decoded state whose positive store is `[1]` at offset `i32::MAX - 1` fails, leaving the serialized bytes as they were. | +| `known_state_matches_golden` | Known stores and scalars serialize to the golden bytes. | Builds the positive and signed `alpha = 0.01` states through `DdSketchState` and verifies `to_asapv1` equals `asapv1_golden/ddsketch_positive_a001.hex` and `ddsketch_signed_a001.hex`. | ### CountMinSketchWithHeap @@ -387,6 +383,7 @@ Wire tests: [`src/sketches/countsketch_l2_norm/wire.rs`](../src/sketches/countsk | `count_sketch_with_l2_norm_metadata_rejects_unknown_and_missing_keys` | An unexpected metadata key and a missing required one both fail closed. | Encodes the nine `CountSketchWithL2NormMetadata` fields plus a `bogus_field`, and the same fields with `cols` omitted, and verifies neither decodes as `CountSketchWithL2NormMetadata`. | | `count_sketch_with_l2_norm_empty_has_one_encoding` | A sketch holding nothing has exactly one encoding. | Verifies a fresh `3x32` sketch and one cleared after an insert serialize to identical bytes. | | `count_sketch_with_l2_norm_rejects_too_many_rows` | CountSketchWithL2Norm carries the seed list's row bound on both sides. | Verifies a sketch past `MATRIX_MAX_ROWS` fails to serialize, that a crafted envelope of that geometry fails to decode with a message naming `MATRIX_MAX_ROWS`, and that the boundary row count still serializes. | +| `known_state_matches_golden` | Known counts, L2 accumulators and seed index serialize to the golden bytes. | Builds a `2x4` sketch with counts `[[127, 128, 65535, -32768], [-32, -33, i32::MIN, i64::MIN]]`, row L2 `[65536, i64::MAX]` and seed index `7`, and verifies `to_asapv1` equals `asapv1_golden/count_l2hh_2x4_seed7.hex`. | ### CountMinSketchWithSpaceSaving @@ -486,7 +483,6 @@ The `FoldCell` and `FoldEntry` tests live here because both types are declared i | `hierarchical_merge_four_sketches` | Four epochs merge down to full width and stay exact. | Builds four `3x1024` level-2 sketches (256 physical columns) over `U64` ranges `0..10`, `10..20`, `20..30`, and `30..40` inserted once each, then verifies `hierarchical_merge` reports `fold_level()` `0` and matches the standard `3x1024` CMS estimate for all 40 keys. | | `unfold_full_matches_flat_counters` | `unfold_full` changes the geometry, not the counters. | For a `3x256` level-4 sketch over `I32(0..30)` inserted once each, verifies `unfold_full()` reports `fold_level()` `0` and `fold_cols()` `256`, and that `to_flat_counters()` is unchanged. | | `to_flat_counters_matches_standard_cms` | The flat counter view is the full-width matrix. | For `rows=3`, `cols=128`, `fold_level=3` over `I32(0..20)` inserted once each, verifies `to_flat_counters()` has the same length as the standard sketch's storage slice, then zips the two and verifies equality cell by cell. | -| `serde_round_trip_keeps_the_cells_the_geometry_and_the_heap` | The derived serde form carries the cells, the geometry and the heap. | For `rows=3`, `full_cols=4096`, `fold_level=4`, `heap_size=16` fed a seeded (`0x5EED_C45C`) Zipf(`domain = 2_000`, `exponent = 1.2`) stream of `20_000` samples - first checking the fixture actually produced collided cells - round-trips through `rmp_serde` and verifies `rows`, `fold_cols`, `full_cols`, `fold_level`, `total_entries`, `collided_cells` and `to_flat_counters()` are unchanged, that all `2_000` keys estimate identically, that the heap holds the same `16` residents in the same order with the same counts, and that the rebuilt (`#[serde(skip)]`) key index finds its own resident. | | `sparse_subwindow_has_few_collisions` | A sparse sub-window keeps its cells mostly uncollided. | Inserts 50 distinct `U64` keys into a `3x4096` level-4 sketch (256 physical columns) and verifies `total_entries()` is at most `rows * 50` and at least `rows * 45`, and that `collided_cells()` is under `30`. | | `heap_tracks_heavy_hitters` | The top-k heap carries the heaviest key at its estimate. | Inserts `Str("heavy")` 100 times, `Str("medium")` 10 times, and `Str("light")` once into a `3x1024` level-3 sketch with `heap_size=5`, then verifies the heap is not empty and its `"heavy"` entry reads `100`; the other two keys' entries are not checked. | | `heap_survives_same_level_merge` | A same-level merge refreshes the heap to the merged estimate. | Inserts `Str("user_x")` 50 times into one `3x1024` level-3 `heap_size=5` sketch and 70 times into another, then verifies the heap entry after `merge_same_level` reads `120`. | @@ -530,7 +526,6 @@ The shared `FoldCell` and `FoldEntry` behaviour is covered by the cell tests in | `hierarchical_merge_four_sketches` | Four epochs merge down to full width and stay exact. | Builds four `3x1024` level-2 sketches (256 physical columns) over `U64` ranges `0..10`, `10..20`, `20..30`, and `30..40` inserted once each, then verifies `hierarchical_merge` reports `fold_level()` `0` and matches the standard `3x1024` Count Sketch estimate for all 40 keys. | | `unfold_full_matches_flat_counters` | `unfold_full` changes the geometry, not the counters. | For a `3x256` level-4 sketch over `I32(0..30)` inserted once each, verifies `unfold_full()` reports `fold_level()` `0` and `fold_cols()` `256`, and that `to_flat_counters()` is unchanged. | | `to_flat_counters_matches_standard_cs` | The flat counter view matches the full-width matrix on a second geometry. | For `rows=3`, `cols=128`, `fold_level=3` over `I32(0..20)` inserted once each, verifies `to_flat_counters()` has the same length as the standard sketch's storage slice, then zips the two and verifies equality cell by cell. | -| `serde_round_trip_keeps_the_cells_the_geometry_and_the_heap` | The derived serde form carries the cells, the geometry and the heap. | For `rows=3`, `full_cols=4096`, `fold_level=4`, `heap_size=16` fed a seeded (`0x5EED_C55C`) Zipf(`domain = 2_000`, `exponent = 1.2`) stream of `20_000` samples - first checking the fixture actually produced collided cells - round-trips through `rmp_serde` and verifies `rows`, `fold_cols`, `full_cols`, `fold_level`, `total_entries`, `collided_cells` and `to_flat_counters()` are unchanged, that all `2_000` keys estimate identically, that the heap holds the same `16` residents in the same order with the same counts, and that the rebuilt (`#[serde(skip)]`) key index finds its own resident. | | `sparse_subwindow_has_few_collisions` | A sparse sub-window keeps its cells mostly uncollided. | Inserts 50 distinct `U64` keys into a `3x4096` level-4 sketch (256 physical columns) and verifies `total_entries()` is at most `rows * 50` and at least `rows * 45`, and that `collided_cells()` is under `30`. | | `heap_tracks_heavy_hitters` | The top-k heap carries the heaviest key at its estimate. | Inserts `Str("heavy")` 100 times, `Str("medium")` 10 times, and `Str("light")` once into a `3x1024` level-3 sketch with `heap_size=5`, then verifies the heap is not empty and its `"heavy"` entry reads `100`; the other two keys' entries are not checked. | | `heap_survives_same_level_merge` | A same-level merge refreshes the heap to the merged estimate. | Inserts `Str("user_x")` 50 times into one `3x1024` level-3 `heap_size=5` sketch and 70 times into another, then verifies the heap entry after `merge_same_level` reads `120`. | @@ -562,7 +557,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_weighted_arrival_matches_repeating_it` | `update_weighted` matches that many single inserts. | Builds two summaries over the same stream, one key-at-a-time and one weighted per key, and verifies equal totals, equal residency, and equal per-key estimates. | | `a_summary_larger_than_the_domain_is_exact` | A summary with room for the whole domain is exact. | With capacity above the distinct count, verifies every estimate equals the true count, every error is `0`, and `min_count` stays `0`. | | `a_merge_keeps_the_total_and_never_reads_low` | Merge conserves the inserted mass and stays one-sided. | Partitions the stream into two shards, merges their summaries, and verifies the merged total equals the whole stream length, residency stays within capacity, and no monitored key reads below either side's truth. | -| `a_serde_round_trip_preserves_every_answer` | Serde round trip preserves state and the rebuilt key index. | Round-trips through `rmp-serde` and verifies residency, capacity, total, `min_count`, and per-key estimate/error all match, then confirms the decoded summary still takes inserts against its rebuilt index. | +| `an_asapv1_round_trip_preserves_every_answer` | An ASAPv1 round trip preserves state and the rebuilt key index. | Round-trips through `to_asapv1` / `from_asapv1` and verifies residency, capacity, total, `min_count`, and per-key estimate/error all match, then confirms the decoded summary still takes inserts against its rebuilt index. | | `an_empty_summary_answers` | An empty summary answers every query without panicking. | Verifies `len`/`total`/`min_count` are zero, point queries return `0`, `is_guaranteed` is false, and `top_k`/`entries` are empty. | | `a_weighted_arrival_matches_repeating_it_under_eviction` | A weighted arrival matches repeating it once the summary is evicting. | Feeds the same Zipf stream key by key into two 64-counter summaries, one arrival at a time and one `update_weighted` per key, verifies the run saturates all 64 counters, and that `total`, `min_count`, and the whole sorted key/count/error entry list agree. | | `update_batch_matches_repeated_updates_and_a_zero_weight_is_inert` | `update_batch` is the loop it replaces, and a zero weight records nothing. | Verifies a 128-counter summary filled by `update_batch` matches one filled one at a time on `total` and on `top_k(usize::MAX)`, then that `update_weighted` at weight `0` for a resident key and for an absent one leaves `total`, residency, and `top_k` untouched. | @@ -570,8 +565,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_merge_into_an_under_full_summary_keeps_the_ceiling_honest` | A merge carries the peer's ceiling even when there are counters to spare. | Merges a one-counter summary that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, then verifies residency is `1` and below capacity, `upper_bound` for the dropped key 7 still covers its true `10`, and `is_guaranteed` is false for the key that survived. | | `a_merge_chain_stays_one_sided_against_the_truth` | A chain of merges over asymmetric capacities stays one-sided. | Shards the stream three ways by position into summaries of capacity 64, 512, and 7, merges them in turn, and verifies the merged `total` equals the stream length, residency stays within capacity, every true key's `upper_bound` covers its true count, every monitored key reads at or above its truth, and every key `is_guaranteed` accepts truly outranks the largest dropped key. | | `a_merge_picks_the_same_survivors_every_time` | The merge depends on what the summaries hold, not on arrival order. | Merges two 200-counter summaries over keys `0..150` and `100..250`, verifies the union fills the capacity at 200 with exactly the 50 shared keys at count `2`, that five repeats give an identical key/count/error list, and that reversing both arrival orders gives the same list. | -| `a_round_trip_carries_the_merged_ceiling` | The ceiling a merge established survives the wire. | Merges a 4-counter summary of the stream's second half into a 32-counter summary of its first half, round-trips through `rmp-serde`, and verifies `min_count`, `total`, and residency are unchanged, every key's estimate matches, and every true key's decoded `upper_bound` still covers its true count. | -| `crafted_state_fails_closed` | Serialized state no run could have produced is refused. | Verifies hand-built payloads with a zero capacity, entries over the capacity, a counter at `0`, an error above its count, a repeated key, a ceiling of `200` over the lowest count of `3`, and a `total` of `2` under a counter's `9` all fail to decode, while a well-formed two-counter state answers `top_k` and `estimate`. | +| `a_round_trip_carries_the_merged_ceiling` | The ceiling a merge established survives the wire. | Merges a 4-counter summary of the stream's second half into a 32-counter summary of its first half, round-trips through ASAPv1, and verifies `min_count`, `total`, and residency are unchanged, every key's estimate matches, and every true key's decoded `upper_bound` still covers its true count. | | `a_fresh_summary_is_well_formed` | A fresh summary passes the structural check. | For `with_capacity(4)`, verifies `validate()` succeeds, `min_count` is `0`, and `capacity` is `4`. | | `a_capacity_of_zero_floors_at_one` | A requested capacity of zero floors at one counter. | Verifies `with_capacity(0)` reports a capacity of `1`, and that after two distinct keys `validate()` passes, residency is `1`, and the survivor reads `2`. | | `a_weighted_arrival_displaces_the_minimum_and_starts_above_it` | A weighted eviction starts the arrival above the count it displaced. | In a two-counter summary holding `1` at 5 and `2` at 2, `add(3, 4)` is verified to evict key 2, leaving key 3 at `6` with error `2`, key 2 at `0`, key 1 at `5`, `min_count` at `5`, `total` at `11`, and `validate()` passing. | @@ -585,14 +579,14 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_key_that_re_enters_after_a_merge_never_reads_low` | A key re-entering a merged summary comes back above the ceiling. | After a merge drops key 7 at its true 12, one further insert is verified to seat it at `min_count + 1` with `min_count` as its error, at or above its true `13`, with `validate()` passing. | | `randomized_operations_keep_the_structure_sound` | Randomized weighted inserts keep the structure sound at every step. | Drives 4,000 `update_weighted` calls of weight 1, 3, 11, or 97 over a 96-key domain at capacities 1, 2, 7, 64, and 257, calling `validate()` after every step, then checks residency equals `min(capacity, distinct)`, `total` equals the inserted mass, the walk is the right length and descending, and every truth is bracketed by its counter's estimate and error. | | `randomized_merges_keep_the_structure_sound` | Randomized merges keep the structure sound and one-sided. | For five capacity/domain pairs, merges two independently fuzzed summaries into a fuzzed one and then inserts 200 more keys, calling `validate()` after each merge and after the inserts, and checks the error sandwich against the combined truth, `total` equal to the combined mass, and residency within capacity. | -| `a_decoded_summary_rebuilds_both_link_directions` | Decoding rebuilds the bucket and counter links, not just the counts. | Round-trips a fuzzed 48-counter summary through `rmp-serde` and verifies the decoded summary passes `validate()`, matches residency, `min_count`, and `total`, still brackets every truth, and walks the same key/count set. | +| `a_decoded_summary_rebuilds_both_link_directions` | Decoding rebuilds the bucket and counter links, not just the counts. | Rebuilds a fuzzed 48-counter summary from its triples and verifies the decoded summary passes `validate()`, matches residency, `min_count`, and `total`, still brackets every truth, and walks the same key/count set. | | `a_crafted_state_fails_closed` | `rebuild` refuses state the algorithm could not have produced. | Verifies a zero capacity, entries over the capacity, a counter at `0`, an error above its count, a repeated key, a `discarded_max` of `u64::MAX` over the lowest count of `3`, and a `total` of `0` under a counter's `9` are each rejected with the matching complaint. | | `a_declared_capacity_is_not_allocated_on_decode` | A huge declared capacity is not allocated on decode. | A state declaring a capacity of `1 << 40` with a single entry is verified to rebuild, pass `validate()`, report that capacity, and hold one counter. | | `a_non_utf8_byte_key_is_monitored_and_queried` | A byte array that is not UTF-8 is a key like any other. | Inserts `[0xff, 0x00, 0xfe]` three times and `[0x00, 0x80]` at weight 2 into a 4-counter summary, and verifies `validate()` passes, the repeats share one counter, both keys are held as their raw bytes, each estimates its weight, and an unseen byte key reads `0`. | | `a_byte_key_and_a_string_key_are_separate_counters` | `Bytes(b"abc")` and `Str("abc")` are two keys, not one. | Seats both in a 4-counter summary at weights 5 and 2 and verifies residency is `2`, the byte key reads `5`, and the string key reads `2` through `Str` and `String` alike. | | `an_evicted_byte_key_keeps_its_bound` | An evicted byte key still reads under the ceiling. | In a 2-counter summary holding a byte key at 3 and another at 2, a third byte key is verified to displace the smallest, leaving the evicted key at `estimate` `0` with an `upper_bound` at or above its true `2`, a `min_count` of `3`, and the newcomer at `3` carrying an error of `2`. | | `a_merge_pairs_byte_keys_by_their_bytes` | A merge pairs byte keys by their bytes. | Merges a peer holding the shared byte key at 2 and its own at 7 into a summary holding the shared key at 5 and its own at 3, and verifies `validate()` passes, all three byte keys survive as their raw bytes, and the shared key reads `7`. | -| `a_serde_round_trip_keeps_a_byte_key` | A serde round trip carries raw bytes and re-hashes them to the digest a query reaches. | Round-trips a summary holding `[0xff, 0x00, 0xfe]` at 9 and five `0x00` bytes at 4 through `rmp-serde`, and verifies the decoded summary passes `validate()`, holds the same raw byte keys, answers both `DataInput::Bytes` queries at their weights, and reports the same `total`. | +| `a_rebuild_keeps_a_byte_key` | A rebuild carries raw bytes and re-hashes them to the digest a query reaches. | Rebuilds a summary holding `[0xff, 0x00, 0xfe]` at 9 and five `0x00` bytes at 4 from its triples, and verifies the rebuilt summary passes `validate()`, holds the same raw byte keys, answers both `DataInput::Bytes` queries at their weights, and reports the same `total`. | | `space_saving_envelope_structure_and_round_trip` | The envelope frames a summary under kind_id `0x18 0x00`, and the bytes are stable across a round trip. | For a 48-counter summary saturated by 30,000 weighted draws over 400 distinct keys, verifies the bytes open with the ASAPv1 magic, `envelope::VERSION`, a `kind_id_len` of `2`, and `0x18 0x00`, that re-serializing the decoded summary reproduces them exactly, and that residency, capacity, `total`, `min_count`, and `estimate`/`error`/`upper_bound`/`is_guaranteed` agree across all 400 keys. | | `space_saving_merged_ceiling_survives_the_wire` | The ceiling a merge leaves behind travels in `floor`, which the triples do not determine. | Merges a one-counter summary that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, then verifies the decoded `min_count` and the `upper_bound` on the dropped key 7 both match the source and still cover its true `10`, with identical re-serialized bytes; a payload carrying only the triples decodes to a `min_count` of `0` here. | | `space_saving_empty_round_trip` | A summary monitoring nothing has one encoding. | Verifies an empty 16-counter summary reports the pinned `EMPTY_KEY_TYPE` of `u64` in its metadata and decodes back to residency `0`, capacity `16`, `min_count` `0`, `total` `0`, and identical re-serialized bytes. | @@ -616,14 +610,15 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs `spa | `a_merge_saturates_a_key_only_the_other_side_holds` | A key arriving only from the peer inherits this side's ceiling and saturates. | Merges a 4-counter peer holding key 3 at `7` into the `u64::MAX - 3`-ceilinged 4-counter summary, and verifies `validate()` passes, key 3 reads `u64::MAX` with an error of `u64::MAX - 3`, and key 8 still reads `u64::MAX - 3`. | | `a_merge_saturates_a_key_only_this_side_holds` | A key only this side holds inherits the peer's ceiling and saturates. | Merges the `u64::MAX - 3`-ceilinged summary into a 4-counter summary holding key 3 at `7`, and verifies `validate()` passes, key 3 reads `u64::MAX` with an error of `u64::MAX - 3`, and key 8 reads `u64::MAX - 3`. | | `a_merge_saturates_the_ceiling` | The ceiling a merge leaves behind saturates rather than wrapping to a tiny bound. | Merges a 4-counter summary ceilinged at `10` (key 6 dropped, key 5 held) into one ceilinged at `u64::MAX - 3`, and verifies `validate()` passes, `min_count` is `u64::MAX`, and `upper_bound` for the dropped key 9 is at least the `u64::MAX - 4` it truly reached. | -| `a_serde_round_trip_carries_a_ceiling_no_counter_holds` | An under-full summary's ceiling lives only in `discarded_max`, and serde carries it. | Merges a one-counter peer that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, checks residency is below capacity with `min_count` at `30`, then round-trips through `rmp-serde` and verifies the decoded summary passes `validate()`, still reports `min_count` `30`, and covers key 7's true `10` in its `upper_bound`. | +| `a_rebuild_carries_a_ceiling_no_counter_holds` | An under-full summary's ceiling lives only in `discarded_max`, and a rebuild carries it. | Merges a one-counter peer that saw key 7 ten times and key 8 twenty times into an empty 33-counter summary, checks residency is below capacity with `min_count` at `30`, then rebuilds it from its triples and verifies the rebuilt summary passes `validate()`, still reports `min_count` `30`, and covers key 7's true `10` in its `upper_bound`. | | `a_key_that_only_ties_the_ceiling_is_not_guaranteed` | `is_guaranteed` is strict: tying the ceiling is not outranking it. | In a 2-counter summary holding keys 1 and 2 at 3 each, the arrival of key 3 is verified to leave `validate()` passing, key 1 at `3` with error `0`, `min_count` at `3`, and `is_guaranteed` false for both key 1 and key 3; two further key-1 inserts then take it to `5` against a `min_count` of `4`, at which point `is_guaranteed` accepts it. | | `clear_resets_every_answer` | `clear` leaves a summary that answers like a fresh one, ceiling and total included. | Clears a 2-counter summary that held key 1 at 5 and key 2 at 3 with key 3 evicting, and verifies `validate()` passes, `is_empty` holds, `len` is `0`, `capacity` is still `2`, `total`, `min_count`, `upper_bound`, and `error` are all `0`, `top_k(4)` and `entries` are empty, and that one further insert seats key 4 at `1` with a `total` of `1`. | | `colliding_keys_stay_distinct` | Keys sharing a digest are separate counters, each answering for itself. | Under a `OneDigest` hasher that files every key under one digest, inserts keys 10, 20, and 30 at weights 3, 5, and 1 into a 4-counter summary and verifies `validate()` passes, residency is `3`, the held keys are exactly `[10, 20, 30]`, each estimates its own weight, and an unseen key 40 reads `0`. | | `an_eviction_under_collision_keeps_the_index_straight` | An eviction drops the victim's key from the index before it seats the arrival under the same digest. | Under the `OneDigest` hasher, a 2-counter summary holding key 10 at 3 and key 20 at 2 takes key 30; verifies `validate()` passes, residency is `2`, the held keys are `[10, 30]`, key 30 reads `3` with an error of `2`, key 10 still reads `3`, and the evicted key 20 reads `0`. | | `a_merge_under_collision_pairs_by_key` | A merge pairs the two sides by key, not by the digest slot they share. | Under the `OneDigest` hasher, merges a 4-counter peer holding key 10 at 2 and key 30 at 7 into one holding key 20 at 3 and key 10 at 5, and verifies `validate()` passes, all three keys survive, the shared key 10 reads `7`, and keys 20 and 30 read `3` and `7`. | | `a_merge_under_collision_breaks_ties_by_key` | With one digest for everything, key order is the only tie-break left and it still fixes the survivors. | Under the `OneDigest` hasher, merges two 2-counter summaries holding keys 30 and 40 against keys 10 and 20, all at count 1, and verifies the survivors are `[10, 20]` and that swapping which side is merged into gives the same list. | -| `a_decoded_summary_under_collision_keeps_its_keys` | A decode compares keys rather than digests, so it seats all of them and still catches a repeat. | Under the `OneDigest` hasher, round-trips a 4-counter summary holding keys 10, 20, and 30 at weights 5, 3, and 9 through `rmp-serde` and verifies the decoded summary passes `validate()`, holds all three keys, and estimates each at its weight; then that a `SpaceSavingState` listing key 10 twice fails `rebuild` with a "same key twice" complaint. | +| `a_decoded_summary_under_collision_keeps_its_keys` | A decode compares keys rather than digests, so it seats all of them and still catches a repeat. | Under the `OneDigest` hasher, rebuilds a 4-counter summary holding keys 10, 20, and 30 at weights 5, 3, and 9 from its triples and verifies the decoded summary passes `validate()`, holds all three keys, and estimates each at its weight; then that a `SpaceSavingState` listing key 10 twice fails `rebuild` with a "same key twice" complaint. | +| `known_state_matches_golden` | Known triples, seated out of emitted order, serialize to the golden bytes. | Rebuilds the `i64` capacity-8 and `bytes` capacity-3 summaries of `space_saving_{i64_c8, bytes_c3}_matches_golden` from their triples in reverse and verifies `to_asapv1` equals each golden. | ### Bloom @@ -646,7 +641,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `sizing_meets_the_target_and_is_one_power_of_two_from_missing_it` | `dimensions_for` is pinned to its contract rather than to its own expression. | For `(n, p)` pairs from `(1, 0.5)` through `(10_000, 1e-12)`, verifies every slice is a power of two and the row count stays within `BLOOM_MAX_SLICES`, that the chosen geometry's `predicted_fpp(n)` meets the target, and that halving `cols` misses it. | | `degenerate_geometries_still_answer` | Degenerate geometries answer rather than panic. | Verifies a one-bit filter answers and reports an `estimated_fpp` of `1.0` once set, and that a single-row filter still holds every member it was given. | | `clearing_restores_an_empty_filter` | `clear` restores the constructed state and keeps the dimensions. | After clearing a filled filter, verifies it is empty with a zero insert count, unchanged rows and columns, and no member reported present. | -| `a_serde_round_trip_preserves_every_answer` | Serde round trip preserves every answer. | Round-trips through `rmp-serde` and verifies matching dimensions, insert count, and set bits, plus identical answers on members and non-members. | +| `an_asapv1_round_trip_preserves_every_answer` | An ASAPv1 round trip preserves every answer. | Round-trips through `to_asapv1` / `from_asapv1` and verifies matching dimensions, insert count, and set bits, plus identical answers on members and non-members. | | `storage_is_one_bit_per_cell` | Packed storage is one bit per cell, not one byte. | For an `8x4096` filter, verifies `bit_capacity` equals `rows * cols` and `size_in_bytes` is one eighth of it. | | `sizing_never_asks_for_more_slices_than_the_seed_list_has` | The slice count stops at the number of seeds the hasher has. | For 10,000 keys at targets `1e-7`, `1e-9`, and `1e-12`, verifies the row count stays within `BLOOM_MAX_SLICES` and that no two rows of the filled filter hold identical bits. | | `sizing_stays_inside_the_allocation_ceiling` | A target the seed list cannot reach yields the widest geometry that fits. | For `expected_items` up to `usize::MAX` and targets `f64::MIN_POSITIVE`, `0.0`, and `2.0`, verifies `cols` is at least one and a power of two, rows fall in `1..=BLOOM_MAX_SLICES`, and `rows * cols` stays within `BLOOM_MAX_BITS`. | @@ -654,8 +649,8 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `an_infinite_target_rate_is_rejected` | An infinite target rate panics. | Verifies `with_capacity(1_000, f64::INFINITY)` panics with "target false-positive rate must be finite". | | `a_negative_infinite_target_rate_is_rejected` | A negative infinite target rate panics. | Verifies `with_capacity(1_000, f64::NEG_INFINITY)` panics with "target false-positive rate must be finite". | | `the_two_hash_paths_agree_only_where_the_geometry_gives_each_row_its_own_hash` | Whether the paths land on the same bits is a property of the geometry. | Verifies `8 x 65_536` needs 136 hash bits, so both paths fall back to one seeded hash per row and set identical bits, while `7 x 65_536` fits a packed hash and the paths differ; also verifies that second shape is `BLOOM_DEFAULT_ROWS` by `BLOOM_DEFAULT_COLS`. | -| `a_filter_cannot_be_decoded_into_the_other_hash_path` | The serialized form carries which path built the filter. | Verifies regular-path bytes fail to decode as `Bloom` with an error naming both paths, fast-path bytes fail as `Bloom` and as the unannotated `Bloom`, and fast-path bytes decoded back into `Bloom` still report every member present. | -| `a_fast_path_serde_round_trip_preserves_every_answer` | The fast path round-trips like the regular one, tag and all. | Round-trips a fast-path filter sized for 20,000 keys at a 1% target and verifies matching dimensions, insert count, and set bits, plus identical answers on 20,000 non-member probes. | +| `a_filter_cannot_be_decoded_into_the_other_hash_path` | The ASAPv1 metadata carries which path built the filter. | Verifies regular-path bytes fail to decode as `Bloom`, fast-path bytes fail as `Bloom` and as the unannotated `Bloom`, and fast-path bytes decoded back into `Bloom` still report every member present. | +| `a_fast_path_asapv1_round_trip_preserves_every_answer` | The fast path round-trips like the regular one, tag and all. | Round-trips a fast-path filter sized for 20,000 keys at a 1% target through ASAPv1 and verifies matching dimensions, insert count, and set bits, plus identical answers on 20,000 non-member probes. | | `a_fast_path_union_equals_the_filter_of_the_concatenated_stream` | Merge is an exact union on the fast path too. | Splits 10,000 keys by parity across two `7 x 2^14` fast-path filters, merges them, and verifies the same set bits and insert count as a single-pass filter, with every key reported present. | | `merging_filters_of_different_widths_panics` | A merge across slice widths panics. | Verifies merging a `7 x 2^14` filter with a `7 x 2^13` one panics with "bit matrices must have the same dimensions". | | `merging_filters_of_different_slice_counts_panics` | A merge across slice counts panics. | Verifies merging a `7 x 2^14` filter with an `8 x 2^14` one panics with "bit matrices must have the same dimensions". | @@ -669,7 +664,6 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `the_default_geometry_is_the_documented_pair` | `Default` is the two public constants, and they name a filter the hash family can serve. | Verifies a default filter is `BLOOM_DEFAULT_ROWS x BLOOM_DEFAULT_COLS`, that its rows are within `BLOOM_MAX_SLICES`, its width a power of two, its bit capacity under `BLOOM_MAX_BITS`, and that it starts empty. | | `more_slices_than_the_seed_list_has_is_rejected_at_construction` | More slices than the seed list has is refused at construction, not at serialization. | Verifies `with_dimensions(BLOOM_MAX_SLICES + 1, 1024)` panics with a message naming `BLOOM_MAX_SLICES`. | | `the_seed_list_length_itself_is_a_legal_slice_count` | The bound is inclusive: the assert fires past it, not at it. | Verifies `with_dimensions(BLOOM_MAX_SLICES, 1024)` builds and reports that many rows. | -| `a_serde_payload_past_the_slice_cap_is_rejected` | The plain serde form is a second door into the filter and fails closed on the same bound. | Encodes the serialized form over a `BLOOM_MAX_SLICES + 1` row grid and verifies the `rmp-serde` decode returns an error naming `BLOOM_MAX_SLICES` rather than panicking inside the decoder. | | `both_rate_reporters_raise_the_per_slice_rate_to_the_slice_count` | Both rate reporters use the slice count as the exponent. | For a `BLOOM_MAX_SLICES x 1024` filter holding 400 keys at a fill strictly inside `(0, 1)`, verifies `estimated_fpp` equals `fill_ratio^rows` and `predicted_fpp(400)` equals `(1 - e^(-400/cols))^rows`. | | `an_empty_filter_predicts_nothing_and_a_saturated_one_predicts_everything` | The two ends of the rate scale are reported exactly. | Verifies an empty filter reports a zero fill ratio, a zero `estimated_fpp` and `predicted_fpp(0) == 0`, then fills a `3x8` filter until its fill ratio is `1.0` and verifies `estimated_fpp` is exactly `1.0`. | | `a_single_update_sets_exactly_one_bit_in_every_slice` | Partitioning: one key touches exactly one bit per slice, whatever the width. | Across widths `1`, `2`, `7`, `64`, `100` and `1024` by row counts `1`, `5` and `BLOOM_MAX_SLICES`, verifies one insert sets exactly `rows` bits on both hash paths, that `fill_ratio` equals `rows / (rows * cols)`, and that the key reads present. | @@ -697,6 +691,7 @@ Conformance: [`tests/conformance_kit.rs`](../tests/conformance_kit.rs) runs the | `non_power_of_two_widths_answer_membership_on_both_paths` | A width that is not a power of two folds the same bits on insert and on lookup. | For the geometries `1x1`, `3x65`, `7x100`, `5x127`, `9x129` and `5x1_000`, inserts keys `0..200` into a `Bloom` and a `Bloom` built by `with_dimensions` and verifies neither path loses a single member. | | `the_allocation_ceiling_is_the_documented_number` | The allocation ceiling is a documented number, and a target past it settles on the widest geometry that fits. | Verifies `BLOOM_MAX_BITS` is `1 << 31` and that `Bloom::::dimensions_for(1 << 40, 1e-9)` is `(20, 1 << 26)`, the ceiling divided by the slice count and rounded down to a power of two. | | `the_default_filter_has_the_documented_dimensions` | The default filter is the documented geometry. | Verifies `Bloom::::default()` reports rows and columns equal to `BLOOM_DEFAULT_ROWS` and `BLOOM_DEFAULT_COLS`, and that it is `is_empty`. | +| `known_state_matches_golden` | Known bit grids serialize to the golden bytes. | Builds a `Bloom` of 3x128 with words `[1, 2^63, 0, 0xff, u64::MAX, 0x10000]` and `inserted` `300`, and a `Bloom` of 5x32 with words `[1, 0x80000000, 0, 0xffffffff, 0x10000]` and `inserted` `9`, through `BitMatrix::from_words`, and verifies `to_asapv1` equals `bloom_regular_3x128.hex` and `bloom_fast_5x32.hex`. | ### Elastic @@ -856,6 +851,7 @@ Wire tests: [`src/sketches/uniform/wire.rs`](../src/sketches/uniform/wire.rs) | `uniform_sampling_rejects_unordered_priorities` | Entries are held in ascending priority, with ties broken by the value. | Verifies a payload of priorities `[9, 2, 5]` fails to decode, and that one with equal priorities whose values run `[8.0, 1.0]` fails too. | | `uniform_sampling_rejects_crafted_bytes_without_panicking` | Truncated, foreign and garbage bytes are errors, never panics. | Verifies six truncations of a valid envelope, 64 bytes of `0xff`, and a valid envelope carrying a garbage payload all fail to decode. | | `uniform_sampling_rejects_serializing_an_over_full_sampler` | The encode side refuses a sampler its own decoder would reject. | Verifies a sampler holding two samples whose `total_seen` is set to `1` fails to serialize. | +| `known_state_matches_golden` | Known entries and running scalars serialize to the golden bytes. | Builds a sampler at rate `0.25`, `total_seen` `20`, `rng_state` `0x0123456789abcdef` and entries `(3, -0.0)`, `(3, 0.0)`, `(2^63, 1e300)`, `(u64::MAX, NaN)`, and verifies `to_asapv1` equals `asapv1_golden/uniform_sampling_r0p25.hex`. | ### SetAggregator @@ -1120,7 +1116,7 @@ Wire tests: [`src/sketch_framework/eh_bucket_sketch/wire.rs`](../src/sketch_fram | `eh_bucket_sketch_rejects_crafted_blocks` | Crafted blocks fail closed with an error, never a panic. | Verifies a Count-Min triple whose descriptor is cut in half, and one whose state is three `0xc1` bytes, each fail to decode. | | `eh_bucket_sketch_rejects_a_custom_hash_profile_descriptor` | A descriptor naming a custom hash profile is refused, since the variant's decoder pins the profile of the type it rebuilds. | Verifies a triple built from a custom-profile Count-Min fails to decode. | | `eh_bucket_sketch_query_agrees_after_decode` | A decoded union answers a query the way the original did. | For every populated variant, verifies the source and the decode return the same answer for that variant's sample input. | -| `nested_ddsketch_survives_a_serde_round_trip` | The nested DDSketch keeps its running state through a serde round trip. | Feeds `10.0`, `20.0`, `30.0`, `40.0` into an `EhBucketSketch::DdSketch(DdSketch::new(0.01))`, round-trips it through `rmp-serde`, and verifies the decode matches on `get_count`, `sum`, `min`, and `max` and that its `q = 0.5` query lands inside `[10.0, 40.0]`. | +| `nested_ddsketch_survives_an_asapv1_round_trip` | The nested DDSketch keeps its running state through an ASAPv1 round trip. | Feeds `10.0`, `20.0`, `30.0`, `40.0` into an `EhBucketSketch::DdSketch(DdSketch::new(0.01))`, round-trips it through `to_asapv1` / `from_asapv1`, and verifies the decode matches on `get_count`, `sum`, `min`, and `max` and that its `q = 0.5` query lands inside `[10.0, 40.0]`. | ### ExponentialHistogramUnivMon @@ -1414,7 +1410,6 @@ Test file: [`src/common/heap.rs`](../src/common/heap.rs) | `re_scoring_a_resident_never_duplicates_it` | A resident is re-scored in place rather than seated twice. | Updates `hot` and `warm` 50 rounds each into a 4-slot heap and verifies the index stays consistent, residency is `2`, `find` locates `hot`, and its count reads `50`. | | `a_zero_capacity_heap_turns_everything_away` | A zero-capacity heap accepts nothing and stays consistent. | Verifies 32 updates into `TopKHeap::new(0)` all return `false`, the heap stays empty, `find` returns `None`, and the index is consistent. | | `clearing_drops_the_index_with_the_heap` | `clear` drops the index along with the entries. | After filling an 8-slot heap from 32 keys, verifies `clear()` leaves it empty with `find` returning `None` and a consistent index, then refills from 40 fresh keys and verifies the index is consistent again at residency `8`. | -| `a_decoded_heap_rebuilds_its_index` | The index is derived, so decoding rebuilds it rather than carrying it. | Round-trips a 16-slot heap fed 5,000 skewed draws through `rmp-serde` and verifies the decoded index is consistent, residency and capacity match, every resident is still found, and a further update re-scores a resident to `1_000_000` with the index still consistent. | | `the_indexed_heap_matches_the_rebuild_implementation` | Retention does not depend on how the index is maintained. | At capacities 0, 1, 2, 3, 7, 8, 64, and 257, runs 30,000 skewed draws over a 4,096-key domain through both the shipped heap and a reference that rebuilds its key index in full after every accepted update, and verifies at every step that the completeness flag agrees and that both arrays hold the same key and count at every position. | | `the_indexed_heap_matches_the_rebuild_implementation_on_string_keys` | The same agreement on the owned-key path and its different hash. | Runs 20,000 `flow::` string keys over a 2,048-key domain into a 32-slot heap and verifies the shipped and rebuild implementations agree on the completeness flag and on every position's key and count at every step. | | `the_two_agree_when_counts_move_in_both_directions` | Falling counts sink a resident back down and the two still agree. | Feeds a 16-slot heap 10,000 updates whose counts oscillate over `-30..=30` rather than only climbing, and verifies the shipped and rebuild implementations agree on the completeness flag and on every position at every step. | @@ -1429,7 +1424,6 @@ Test file: [`src/common/heap.rs`](../src/common/heap.rs) | `replace_root_with_returns_the_displaced_root` | `replace_root_with` hands back the displaced root and reports the new value's sift. | On the heap holding `[10, 20, 40, 50, 30]`, verifies `replace_root_with(60, ..)` returns `10`, reports `[(0,1),(1,4)]` with the first swap leaving index `0`, and leaves `[20, 30, 40, 50, 60]`. | | `update_at_with_reports_the_sift_in_both_directions` | `update_at_with` reports the sift in whichever direction it runs and reports nothing for an index it refuses. | On the heap holding `[20, 30, 40, 50, 60]`, verifies setting index `4` to `5` reports `[(1,4),(0,1)]` and leaves `[5, 20, 40, 50, 30]`, setting index `0` to `100` reports `[(0,1),(1,4)]` and leaves `[20, 30, 40, 50, 100]`, and that `update_at_with(5, ..)` returns `false`, reports no swap, and leaves the array alone. | | `a_parallel_array_follows_the_reported_swaps` | A caller that applies each reported swap keeps its own array aligned to the heap. | Pushes `50, 40, 30, 20, 10` tagged `a` through `e` into an 8-capacity `CommonHeap` through `push_back_with`, swapping a `Vec` on every report, and verifies the heap reads `[10, 20, 40, 50, 30]` against tags `['e','d','b','a','c']`; then sets index `0` to `100`, runs `update_at_with(0, ..)` and verifies `[20, 30, 40, 50, 100]` against tags `['d','c','b','a','e']`. | -| `a_decoded_heap_takes_string_keyed_updates` | A decoded heap's rebuilt index is readable from the `DataInput` update path. | Fills an 8-slot `TopKHeap` with `flow-0`..`flow-7` at counts `1..=8`, round-trips it through `rmp-serde`, verifies `find` locates every key at its count, then updates `flow-7` to `999` and verifies residency stays `8`, that key is seated exactly once, its count reads `999`, and the index is consistent. | | `retention_matches_a_heapless_oracle` | Which keys survive matches a scan of a plain vector. | At capacities 0, 1, 2, 3, 5, 8, 11, and 16, streams 4,000 skewed draws over a 64-key domain (seed `0x2f2f`) at the distinct counts `((step * 7_919) % 1_000_003) + 1` and verifies at every step that the completeness flag matches the oracle's, that the heap is ordered under `RetainLargest`, and that both hold the same sorted key/count set. | | `a_lookup_reads_the_key_at_every_position_in_a_bucket` | A bucket naming two positions is answered by the key at each, not by the first position listed. | Seats `U64(11)` at `10` and `U64(22)` at `20` in a 4-slot `TopKHeap`, rewrites `slots` so both positions carry `11`'s digest and lists that bucket as `[other, wanted]`, then verifies `find(&U64(11))` and `find_owned(&OwnedDataInput::U64(11))` both return `11`'s own position and its count still reads `10`. | @@ -1475,11 +1469,10 @@ Test file: [`src/common/structures/bit_matrix.rs`](../src/common/structures/bit_ | `union_takes_the_bitwise_or` | `union_from` is the bitwise or of the two grids. | Unions a `3x100` matrix holding `(0,5)` and `(1,99)` with one holding `(1,99)` and `(2,0)`, then verifies `count_ones` is `3` and all three cells read set. | | `union_across_geometries_panics` | A union across geometries panics. | Verifies `union_from` between a `3x100` and a `3x101` matrix panics with "bit matrices must have the same dimensions". | | `a_zero_dimension_is_rejected` | A zero dimension panics at construction. | Verifies `BitMatrix::new(0, 64)` panics with "a bit matrix needs both dimensions". | -| `a_round_trip_recomputes_the_derived_fields` | A decoded matrix folds hashes exactly as the original does. | Over the same 33 geometries, round-trips through `rmp-serde` and verifies the dimensions and every cell match, then drives one `hash_for_matrix` digest through `fast_insert` on both the original and the decoded matrix and verifies they set the same cells. | -| `a_payload_that_does_not_fit_its_dimensions_is_rejected` | A word count that disagrees with the dimensions fails at decode. | Verifies a `3x100` payload carrying 2 words is refused with "needs 6 words, got 2", one carrying 9 words is refused, and a zero-row payload is refused with "needs both dimensions". | -| `the_wire_form_carries_only_the_stored_fields` | The wire carries `words`, `rows`, and `cols` and nothing derived from them. | Verifies a fresh `3x100` matrix encodes byte for byte identically to a struct of just those three fields. | +| `a_rebuild_from_words_recomputes_the_derived_fields` | A matrix rebuilt from its words folds hashes exactly as the original does. | Over the same 33 geometries, rebuilds through `from_words(rows, cols, words())` and verifies the dimensions and every cell match, then drives one `hash_for_matrix` digest through `fast_insert` on both the original and the rebuilt matrix and verifies they set the same cells. | +| `a_payload_that_does_not_fit_its_dimensions_is_rejected` | A word count that disagrees with the dimensions fails in `from_words`. | Verifies a `3x100` grid of 2 words is refused with "needs 6 words, got 2", one of 9 words is refused, and a zero-row grid is refused with "needs both dimensions". | | `a_packed_64_grid_decodes_a_distinct_window_per_row` | Each row of a packed-64 grid reads its own window of the hash. | For a `5x1024` `BitMatrix`, verifies `hash_layout_for_matrix(5, 1024)` is `MatrixHashLayout::Packed64`, then `fast_insert`s 300 `U64` keys and verifies every pair of rows differs at some column; the per-row bit windows themselves are not compared. | -| `dimensions_whose_word_count_overflows_are_rejected` | Dimensions whose word count overflows fail at decode. | Verifies a crafted payload declaring `usize::MAX` rows by `128` columns with an empty `words` array is refused with an error containing "dimensions overflow". | +| `dimensions_whose_word_count_overflows_are_rejected` | Dimensions whose word count overflows fail in `from_words`. | Verifies `usize::MAX` rows by `128` columns with an empty `words` array are refused with an error containing "dimensions overflow". | ### MatrixHash (Common Structure) @@ -2007,8 +2000,6 @@ Test file: [`tests/e2e/nitro.rs`](../tests/e2e/nitro.rs) | `row_level_countmin_nitro_tracks_a_zipf_stream_within_the_combined_band` | Under collisions the row-level Count-Min estimate stays between its sampling floor and the sketch's own collision budget, for every key. | Feeds 200,000 Zipf(1.1) draws over a 512-key domain (stream seed `0x0117_2199`) into a `3x4096` `CountMinSketch, FastPath>` at rates `{1.0, 0.3, 0.1, 0.07}`, seed `0x0117_0000`, and for every distinct key requires `nitro_estimate` to lie in `[f - 4*sigma(f), f + CountMinSpec::new(3, 4096).simultaneous_bound(N, f, distinct, 1e-3) + 4*sigma(N)]`. Both halves are union-bounded, so `Tally::assert_none` tolerates zero violations. | | `row_level_countmin_nitro_separates_keys_of_similar_frequency` | With many keys of comparable frequency, no single hot flow can carry the per-key estimates. | Inserts each of the 16 keys `0x5EED_0000 + 7919*i` 20,000 times into a `3x4096` `CountMinSketch, FastPath>` at rates `{1.0, 0.5, 0.3, 0.1}` and seed `0x0117_1388`, then requires every key's `nitro_estimate` to lie in `[20_000 - 4*sigma(20_000), 20_000 + CountMinSpec::new(3, 4096).simultaneous_bound(320_000, 20_000, 16, 1e-3) + 4*sigma(320_000)]`, with `Tally::assert_none` tolerating zero violations. | | `nitro_saturates_oversized_weights_instead_of_wrapping` | An oversized scaled increment clamps into the counter's domain instead of wrapping negative. | Asserts `nitro_delta_saturated_i32(u64::MAX) == i32::MAX`, `nitro_delta_saturated_u32(u64::MAX) == u32::MAX`, and that both return `7` for `7`. It then feeds 4,000 updates at rate `1e-9` (so `1/rate` exceeds `i32::MAX`) through `NitroBatch::update_batch` over a `3x4096` Count-Min and through the row-level `nitro_update` path, but asserts only that `estimate_median` and `nitro_estimate` are `>= 0.0`; the saturated magnitude on those two paths is not pinned. | -| `a_serde_round_trip_after_every_update_reproduces_the_uninterrupted_run` | A sketch encoded and decoded after every update is cell-for-cell identical to one that was never interrupted, and its future agrees too. | At rates `0.3` and `0.07` (reciprocals that are not integers, so the stochastic-rounding stream is live), runs 600 updates of key `0xC0FF_EE01` into two `3x4096` `CountMinSketch, FastPath>` sketches at seed `0x0117_0000`, round-tripping one through `rmp_serde::to_vec_named` / `from_slice` after every single update, then asserts the two sketches' full 12,288-cell vectors are equal and their `nitro_estimate` values are equal; 200 further updates on both must again leave the cell vectors equal, which catches a decoded sketch that resumed from a reset rounding stream. | -| `a_payload_written_before_the_rounding_field_existed_still_decodes` | A map-encoded payload that lacks `Nitro::rounding_state` decodes with the default stream and keeps sampling. | Serializes a hand-written mirror of the `CountMinSketch` / `Vector2D` / `Nitro` field layout without `rounding_state` (`rows=3`, `cols=4096`, `mask_bits = 12`, `mask = 4095`, `is_nitro_mode = true`, `sampling_rate = 0.3`, `to_skip = 2`, `delta = 3`, `idx = 9`, and a single counter `data[17] = 4`) with `rmp_serde::to_vec_named`, decodes it as `CountMinSketch, FastPath>`, asserts the decoded cell vector equals the 12,288-entry `data` exactly, then feeds 2,000 further `nitro_update` calls and requires the row-0 mass minus the pre-existing `4` to pass `SamplingConfidenceSpec::new(0.3, 4.0).check` against a truth of `2_000`. | | `a_context_snapshot_restores_the_rest_of_the_stream_exactly` | A `NitroContext` snapshot restored onto a fresh sketch replays the remainder of the stream exactly. | At rates `0.3` and `0.07`, feeds 500 updates of key `0xC0FF_EE01` into a `3x4096` `CountMinSketch, FastPath>` at seed `0x0117_3A98`, takes `nitro().context()`, records the cell vector, and runs 500 more updates; a fresh sketch at the same rate and seed gets `nitro_mut().restore_context(snapshot)` and the same 500 updates, and its cell vector must equal the continuous run's cell-by-cell difference over that stretch exactly. | | `count_min_at_full_nitro_sampling_writes_exactly_what_a_plain_update_writes` | At rate `1.0` the Nitro insert path writes the same Count-Min counters a plain insert writes, and the row median never falls below the row minimum. | Feeds 40,000 Zipf(1.1) draws over a 2,048-key domain (stream seed `0x4E17_0001`) into a plain `4x2048` `CountMinSketch, FastPath>` via `update` and into a twin with `enable_nitro(1.0)` via `nitro_update`, then asserts the two 8,192-counter vectors are equal, and per distinct key that `estimate` agrees between the two sketches and that `nitro_estimate` is at least the true count and at least the plain `estimate`. | | `count_min_after_disable_nitro_leaves_the_sampled_update_path_inert` | `disable_nitro` makes the sampled insert path a no-op without disabling the ordinary one. | Feeds the 40,000-draw Zipf(1.1) stream (2,048-key domain, seed `0x4E17_0001`) into a `4x2048` `CountMinSketch, FastPath>` with `enable_nitro(1.0)`, snapshots the 8,192 counters, calls `disable_nitro()` and replays the whole stream through `nitro_update` — the counters must be unchanged — then replays it through `update` and requires the counters to change. | @@ -2043,7 +2034,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `bloom_fast_5x32_matches_golden` | A fast-path Bloom whose rows are narrower than a word serializes to its golden bytes. | Verifies a `Bloom` of 5 slices of 32 bits, words `[1, 0x80000000, 0, 0xffffffff, 0x10000]` and `inserted` `9`, set through its serde form, serializes to `asapv1_golden/bloom_fast_5x32.hex`, and that the decode reports those dimensions and `inserted`, every bit at `(r, c)` equal to bit `c % 64` of word `r * ceil(cols / 64) + c / 64`, and re-serializes identically. | +| `bloom_fast_5x32_matches_golden` | A fast-path Bloom whose rows are narrower than a word serializes to its golden bytes. | Verifies a `Bloom` of 5 slices of 32 bits, words `[1, 0x80000000, 0, 0xffffffff, 0x10000]` and `inserted` `9`, decodes from `asapv1_golden/bloom_fast_5x32.hex` with those dimensions and `inserted`, every bit at `(r, c)` equal to bit `c % 64` of word `r * ceil(cols / 64) + c / 64`, and re-serializes identically. | | `bloom_regular_3x128_matches_golden` | A regular-path Bloom of two words per row serializes to its golden bytes. | Runs the `bloom_fast_5x32_matches_golden` checks for a `Bloom` of 3 slices of 128 bits, words `[1, 2^63, 0, 0xff, u64::MAX, 0x10000]` and `inserted` `300`, against `asapv1_golden/bloom_regular_3x128.hex`. | | `cms_f64_fast_2x3_matches_golden` | A known f64 fast-path Count-Min matrix serializes to its checked-in golden bytes. | Verifies `CountMinSketch::, FastPath>::from_storage` over the `2x3` matrix `[[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]` serializes to `asapv1_golden/cms_f64_fast_2x3.hex`, and that decoding that fixture recovers the six cells in row-major order and re-serializes to the same bytes. | | `cms_i64_regular_2x3_admission_matches_golden` | A sampled Count-Min state closes its metadata with `admission_rate`. | Verifies the `cms_i64_regular_2x3` matrix with `with_admission_rate(0.1)` serializes to `asapv1_golden/cms_i64_regular_2x3_admission.hex`, and that decoding the fixture recovers the six cells, `admission_rate()` `0.1`, and a byte-identical re-serialization. | @@ -2067,7 +2058,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `csheap_i64_regular_2x4_strkeys_matches_golden` | A CountSketchWithHeap serializes to its golden bytes. | Verifies the `2x4` Count Sketch matrix with heap size `5` holding `{"alpha": 4294967296, "beta": 127, "delta": 127, "gamma": -33}` serializes to `asapv1_golden/csheap_i64_regular_2x4_strkeys.hex`, and that the decode holds that matrix and those entries and re-serializes identically. | | `csheap_i64_regular_2x4_tuplekeys_matches_golden` | CountSketchWithHeap keys made of typed columns travel as `bytes` tuple keys. | Verifies the `2x4` Count Sketch matrix with heap size `6` holding four of the same tuple keys at `-33`, `127`, `65536` and `127` serializes to `asapv1_golden/csheap_i64_regular_2x4_tuplekeys.hex`, and that the decode holds that matrix and those entries, reads every key back as a canonical tuple key, and re-serializes identically. | | `csspacesaving_f64_fast_2x4_i64keys_matches_golden` | A CountSketchWithSpaceSaving serializes to its golden bytes. | Verifies the `2x4` Count Sketch f64 matrix beside a capacity-`3` summary fed `-1:65536, i64::MIN:5, 128:5, 7:1, i64::MIN:1` (so `7` displaces `128`) serializes to `asapv1_golden/csspacesaving_f64_fast_2x4_i64keys.hex`, and that the decode holds that matrix, the entries `-1:65536/0, 7:6/5, i64::MIN:6/0`, total `65548` and ceiling `6`, and re-serializes identically. | -| `ddsketch_{positive, signed, empty}_a001_matches_golden` | The DDSketch fixtures serialize to their golden bytes. | For each, verifies an `alpha = 0.01` `DdSketch` with the fixture's stores, zero count and scalars set through its serde form serializes to `asapv1_golden/ddsketch__a001.hex`, and that decoding the fixture re-serializes to the same bytes. | +| `ddsketch_{positive, signed, empty}_a001_matches_golden` | The DDSketch fixtures decode to their known state and round-trip. | For the positive and signed fixtures, verifies `asapv1_golden/ddsketch__a001.hex` decodes to the fixture's stores, zero count and scalars and re-serializes to the same bytes; for the empty one, that `DdSketch::new(0.01)` serializes to it. | | `ddsketch_signed_a001_admission_matches_golden` | A sampled DDSketch state closes its metadata with `admission_rate`. | Verifies the `ddsketch_signed_a001` state with `with_admission_rate(0.125)` serializes to `asapv1_golden/ddsketch_signed_a001_admission.hex` and re-serializes identically after decode, and that the decode reports `admission_rate()` `0.125`, `get_count()` `18` and `sum()` `2523.90625`, unscaled. | | `ddsketch_signed_a001_delta_matches_golden` | A known DDSketch delta serializes to its golden bytes and merges into the signed state golden. | Verifies `DdSketchDelta::new(0.01, [-300, -100, -5, 7, 200, 400], [1, 127, 128, 256, 65536, 4294967296], [-208, -33, -1, 0, 130, 1000], [2, 3, 3, 1, 3, 1], 9)` serializes to `asapv1_golden/ddsketch_signed_a001_delta.hex`, that the fixture decodes and re-serializes identically, and that merged into `ddsketch_signed_a001` it gives the summed non-zero buckets of both stores, `zero_count` `16`, `get_count()` `4295033384`, `sum` folded in payload order from `r = 1.01 * gamma^k`, `min` `-r(1000)`, and `max` `r(400)`. | | `delta_result_empty_matches_golden` | An empty delta serializes to its golden bytes, which round-trip unchanged. | Verifies a `SetAggregatorDelta` with both sets empty serializes to `asapv1_golden/delta_result_empty.hex` (kind_id `0x08 0x80`), and that decoding the fixture yields two empty sets and re-serializes to the same bytes. | @@ -2087,8 +2078,8 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | `set_aggregator_empty_matches_golden` | An empty set serializes to its golden bytes, which round-trip unchanged. | Verifies an empty `SetAggregator` serializes to `asapv1_golden/set_aggregator_empty.hex`, and that decoding the fixture yields an empty set and re-serializes to the same bytes. | | `set_aggregator_strings_matches_golden` | A set's members serialize in ascending UTF-8 byte order to the golden bytes. | Verifies a `SetAggregator` of 11 members, among them `""`, a 31-byte fixstr, a 32-byte str8, a proper-prefix chain and two-, three- and four-byte code points, serializes to `asapv1_golden/set_aggregator_strings.hex`, and that decoding the fixture recovers the set and re-serializes to the same bytes. | | `space_saving_bytes_c3_matches_golden` | A full `bytes`-keyed Space-Saving summary serializes to its golden bytes. | Runs the `space_saving_i64_c8_matches_golden` checks for capacity `3` holding `(ff 00, 9, 0)`, `("", 4, 1)`, `("ab", 4, 0)` with `total` `20` and `discarded_max` `3`, where `min_count()` is `4`, against `asapv1_golden/space_saving_bytes_c3.hex`. | -| `space_saving_i64_c8_matches_golden` | An `i64`-keyed Space-Saving summary serializes to its golden bytes, its count tie by two's-complement bits. | Verifies a summary of capacity `8` holding `(-1, 70000, 0)`, `(300, 9, 2)`, `(0, 5, 0)`, `(1, 5, 1)`, `(i64::MIN, 5, 4)`, `(-129, 5, 0)` with `total` `70030` and `discarded_max` `4`, set through its serde form in reverse, serializes to `asapv1_golden/space_saving_i64_c8.hex`, and that the decode reports that capacity, total, `min_count()` `4` and those triples and re-serializes identically. | -| `uniform_sampling_r0p25_matches_golden` | A UniformSampling reservoir serializes to its golden bytes (`experimental`). | Verifies a sampler at rate `0.25` with `total_seen` `20`, `rng_state` `0x0123456789abcdef` and entries `(3, -0.0)`, `(3, 0.0)`, `(2^63, 1e300)`, `(u64::MAX, NaN)`, set through its serde form, serializes to `asapv1_golden/uniform_sampling_r0p25.hex`, and that the decode reports that rate, `total_seen` and the samples' bits and re-serializes identically. | +| `space_saving_i64_c8_matches_golden` | An `i64`-keyed Space-Saving summary serializes to its golden bytes, its count tie by two's-complement bits. | Verifies a summary of capacity `8` holding `(-1, 70000, 0)`, `(300, 9, 2)`, `(0, 5, 0)`, `(1, 5, 1)`, `(i64::MIN, 5, 4)`, `(-129, 5, 0)` with `total` `70030` and `discarded_max` `4`, decodes from `asapv1_golden/space_saving_i64_c8.hex` with that capacity, total, `min_count()` `4` and those triples and re-serializes identically. | +| `uniform_sampling_r0p25_matches_golden` | A UniformSampling reservoir serializes to its golden bytes (`experimental`). | Verifies a sampler at rate `0.25` with `total_seen` `20`, `rng_state` `0x0123456789abcdef` and entries `(3, -0.0)`, `(3, 0.0)`, `(2^63, 1e300)`, `(u64::MAX, NaN)`, decodes from `asapv1_golden/uniform_sampling_r0p25.hex` with that rate, `total_seen` and the samples' bits and re-serializes identically. | | `univmon_optimized_l5_e2_3x8_1x4_h6_matches_golden` | A two-tier UnivMon pyramid serializes to its golden bytes. | Verifies `UnivMonTwoTier::new(6, 2, 3, 8, 1, 4, 5)` with counters set by `apply_delta`, the heaps `{"alpha": 40, "beta": 7, "gamma": 7}`, `{"delta": 3}`, `{}`, `{"eps": 1}`, `{}` and `bucket_size` `70` serializes to `asapv1_golden/univmon_optimized_l5_e2_3x8_1x4_h6.hex`, and that the decode reports that layout, every layer's cells and heap, and re-serializes identically. | | `univmon_q_i32_l3_c8_r3_matches_golden` | A UnivMon-Q golden decodes to its state and re-serializes byte-identically. | Verifies `asapv1_golden/univmon_q_i32_l3_c8_r3.hex` decodes to the config `levels` `3`, `width` `8`, `cols_halving_period` `1`, `depth` `3`, `counter_bits` `32`, `candidates` `2`, `ordered_samples` `3`, `seed_index` `5`, `source_id()` `7`, `count()` `12`, `min()` `-2.5` and `max()` `5.0`, and re-serializes to the same bytes. | @@ -2128,10 +2119,9 @@ Test file: [`tests/hll_custom_precision.rs`](../tests/hll_custom_precision.rs) | --- | --- | --- | | `custom_precision_indexing_and_iteration` | A macro-generated register storage indexes, slices and iterates. | On an `HllBucketListP10` built by `impl_hll_registers!` outside the crate, sets index `0` to `7` and index `NUM_REGISTERS - 1` to `3`, verifies both read back, that `storage[0..2]` is `[7, 0]`, that `copy_from_slice` of `[1, 2]` into `1..3` lands in `as_slice()`, and that iterating the whole storage sums to `13`. | | `custom_precision_merges` | A merge at a custom precision estimates the union. | Fills `HyperLogLogImpl` with items `key-{i:08}` for `i` in `0..60_000` and a second with `40_000..100_000`, merges them, and verifies the estimate is within 2% relative of the 100,000-item union. | -| `custom_precision_round_trips_through_serde` | A custom-precision sketch round-trips through the derived serde impls. | Fills `HyperLogLogImpl` with 5,000 distinct `key-{i:08}` items and verifies an `rmp_serde::to_vec` / `from_slice` round trip preserves the register slice exactly and the `estimate()`. | | `custom_precision_round_trips_through_the_asapv1_wire_format` | The ASAPv1 envelope carries a custom precision and refuses a mismatched register count. | Over 5,000 distinct `key-{i:08}` items at `lg_k = 13`, verifies `to_asapv1` / `from_asapv1` preserves the registers and `estimate()` for both `Classic` and `Ertl`, preserves `estimate()` for `HyperLogLogHipImpl` so that its running `kxq0`, `kxq1` and `est` travel rather than being recomputed, and that a fresh `lg_k = 18` envelope fails to decode as `HllBucketListP13`. | | `custom_precisions_estimate_correctly` | Every macro-generated precision reports the right constants and estimates in band. | Through a shared `check_precision` helper, verifies for each precision that `PRECISION` is `lg_k`, `NUM_REGISTERS` is `1 << lg_k`, `RANK_HASH_BITS` is `64 - lg_k` and `INDEX_MASK` is `NUM_REGISTERS - 1`, that a default storage has that length, is non-empty and is zeroed, that at least one register is set after the inserts and none exceeds `RANK_HASH_BITS + 1`, and that the `Classic`, `Ertl` and HIP estimates all fall inside the tolerance; invoked at `lg_k` `4` with 100 items at `0.60`, `8` with 1,000 at `0.30`, `10` with 10,000 at `0.15`, `13` with 50,000 at `0.06` and `18` with 200,000 at `0.02`. | -| `large_precision_allocates_on_the_heap` | Register storage is allocated on the heap, so a 4 MiB precision constructs on a test thread. | Verifies a fresh `HyperLogLogImpl` — `1 << 22` registers, past the 2 MiB test-thread stack — reports `NUM_REGISTERS` registers that are all zero, and that an `HllBucketListP18` sketch over 1,000 `key-{i:08}` items survives an `rmp_serde` round trip with an identical register slice. | +| `large_precision_allocates_on_the_heap` | Register storage is allocated on the heap, so a 4 MiB precision constructs on a test thread. | Verifies a fresh `HyperLogLogImpl` — `1 << 22` registers, past the 2 MiB test-thread stack — reports `NUM_REGISTERS` registers that are all zero, and that an `HllBucketListP18` sketch over 1,000 `key-{i:08}` items survives an ASAPv1 round trip with an identical register slice. | ### Accuracy-Probe Regressions diff --git a/src/asapv1/admission_rate.rs b/src/asapv1/admission_rate.rs index 9be948db..ae6d9c34 100644 --- a/src/asapv1/admission_rate.rs +++ b/src/asapv1/admission_rate.rs @@ -3,8 +3,7 @@ //! producer admitted each update. HLL, Count-Min, Count Sketch and DDSketch //! carry it, as the last key of the map, and only when `p < 1`. -use serde::de::Error; -use serde::{Deserialize, Deserializer}; +use serde::Deserializer; use super::wire_key::exact; @@ -57,15 +56,6 @@ pub(crate) fn deserialize<'de, D: Deserializer<'de>>( exact(deserializer).map(Some) } -/// Reads a sketch's rate from its derived serde form: inside `(0, 1]`. -pub(crate) fn deserialize_rate<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result { - let p = f64::deserialize(deserializer)?; - to_key(p).map_err(D::Error::custom)?; - Ok(p) -} - #[cfg(test)] mod tests { use std::panic::{AssertUnwindSafe, catch_unwind}; diff --git a/src/common/hash.rs b/src/common/hash.rs index 6b59723c..74055728 100644 --- a/src/common/hash.rs +++ b/src/common/hash.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use std::fmt::Debug; use twox_hash::{XxHash3_64, XxHash3_128}; @@ -109,7 +108,7 @@ pub trait HashProfile { /// Default hasher using twox_hash (XxHash3). This is the built-in implementation /// used when no custom hasher is specified. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] pub struct DefaultXxHasher; /// The standard ProjectASAP hash profile (`docs/asapv1_wire_format.md` §2). @@ -360,7 +359,7 @@ fn mask_bits_for_cols(cols: usize) -> u32 { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] /// Layout used to store per-row fast-path hashes. pub enum MatrixHashLayout { /// Packs row hashes into one 64-bit value. diff --git a/src/common/heap.rs b/src/common/heap.rs index a4e06516..b6f9aa19 100644 --- a/src/common/heap.rs +++ b/src/common/heap.rs @@ -9,7 +9,7 @@ use crate::common::input::HeapEntry; use crate::common::structures::heap::PREALLOCATED_SLOTS; use crate::common::{CommonHeap, DigestBuildHasher, RetainLargest}; use crate::{DataInput, OwnedDataInput, hash64_seeded, hash64_seeded_owned}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde::{Serialize, de::DeserializeOwned}; use smallvec::SmallVec; use std::cmp::Ordering; use std::collections::HashMap; @@ -66,43 +66,18 @@ type Slot = SmallVec<[usize; 2]>; type Index = HashMap; -/// Serialized form. The index is derived from the heap, so it is rebuilt on -/// load rather than carried. -#[derive(Deserialize)] -#[serde(bound = "")] -struct TopKHeapState { - heap: CommonHeap, RetainLargest>, - heap_size: usize, -} - /// Bounded min-heap over [`HeapEntry`] with key lookup, retaining the `k` largest /// counts. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(from = "TopKHeapState", bound = "")] +#[derive(Clone, Debug)] pub struct TopKHeap { heap: CommonHeap, RetainLargest>, /// Digest of each resident, parallel to the heap array, so a sift patches /// the index without re-hashing the keys it moves. - #[serde(skip)] slots: Vec, - #[serde(skip)] positions: Index, heap_size: usize, } -impl From> for TopKHeap { - fn from(state: TopKHeapState) -> Self { - let mut heap = TopKHeap { - heap: state.heap, - slots: Vec::new(), - positions: Index::default(), - heap_size: state.heap_size, - }; - heap.rebuild_index(); - heap - } -} - impl TopKHeap { /// Creates a new TopKHeap with capacity `heap_size`. pub fn new(heap_size: usize) -> Self { @@ -268,24 +243,6 @@ impl TopKHeap { } } - /// Rebuilds `slots` and `positions` from the heap array. - fn rebuild_index(&mut self) { - let Self { - heap, - slots, - positions, - .. - } = self; - slots.clear(); - positions.clear(); - slots.reserve(heap.len()); - for (idx, item) in heap.iter().enumerate() { - let slot = hash64_seeded_owned(0, &item.key); - slots.push(slot); - positions.entry(slot).or_default().push(idx); - } - } - #[inline] fn should_accept_new(&self, count: C) -> bool { if self.heap_size == 0 { @@ -1009,76 +966,6 @@ mod index_invariants { assert_eq!(heap.len(), 8); } - /// The index is derived, so it is rebuilt on load rather than carried. A - /// decoded heap answers lookups and takes further updates. - #[test] - fn a_decoded_heap_rebuilds_its_index() { - let mut heap = TopKHeap::new(16); - let mut counts: HashMap = HashMap::new(); - for key in zipfish(5_000, 256, 0xd00d) { - let entry = counts.entry(key).or_default(); - *entry += 1; - heap.update(&DataInput::U64(key), *entry); - } - - let bytes = rmp_serde::to_vec(&heap).expect("serialize"); - let mut decoded: TopKHeap = rmp_serde::from_slice(&bytes).expect("deserialize"); - - decoded.assert_index_consistent("decoded"); - assert_eq!(decoded.len(), heap.len()); - assert_eq!(decoded.capacity(), heap.capacity()); - for item in heap.heap() { - assert!( - decoded.find_owned(&item.key).is_some(), - "decoded heap lost {:?}", - item.key - ); - } - - let resident = heap.heap()[0].key.clone(); - decoded.update_owned(&resident, 1_000_000); - decoded.assert_index_consistent("decoded then updated"); - let idx = decoded.find_owned(&resident).expect("still resident"); - assert_eq!(decoded.heap()[idx].count, 1_000_000); - } - - /// The shape every sketch uses after a decode: string keys reaching the - /// heap as [`DataInput`] through [`TopKHeap::update`], which hashes by a - /// different entry point from the one `rebuild_index` uses. A rebuilt index - /// the update path cannot read seats a resident key a second time. - #[test] - fn a_decoded_heap_takes_string_keyed_updates() { - let mut heap = TopKHeap::new(8); - for i in 0..8u64 { - heap.update(&DataInput::String(format!("flow-{i}")), i as i64 + 1); - } - assert_eq!(heap.len(), 8); - - let bytes = rmp_serde::to_vec(&heap).expect("serialize"); - let mut decoded: TopKHeap = rmp_serde::from_slice(&bytes).expect("deserialize"); - - for i in 0..8u64 { - let probe = DataInput::String(format!("flow-{i}")); - let idx = decoded - .find(&probe) - .unwrap_or_else(|| panic!("decoded heap cannot find flow-{i}")); - assert_eq!(decoded.heap()[idx].count, i as i64 + 1); - } - - decoded.update(&DataInput::String("flow-7".to_owned()), 999); - assert_eq!(decoded.len(), 8, "an update seated a resident key again"); - let seated = decoded - .heap() - .iter() - .filter(|item| item.key == OwnedDataInput::String("flow-7".to_owned())) - .count(); - assert_eq!(seated, 1, "flow-7 is seated {seated} times"); - let idx = decoded - .find(&DataInput::String("flow-7".to_owned())) - .expect("flow-7 is resident"); - assert_eq!(decoded.heap()[idx].count, 999); - decoded.assert_index_consistent("decoded then updated by data input"); - } } #[cfg(test)] diff --git a/src/common/input.rs b/src/common/input.rs index 6cf74c62..b9def6f8 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -2,7 +2,6 @@ //! This module defines value wrappers, heap-friendly items, and Hydra/UnivMon helpers. use crate::asapv1::admission_rate; -use serde::{Deserialize, Serialize}; use std::{ fmt, hash::{Hash, Hasher}, @@ -14,7 +13,7 @@ use crate::{ }; /// Input wrapper for sketch APIs (supports primitive and borrowed values). -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub enum DataInput<'a> { /// A signed 8-bit integer. I8(i8), @@ -53,7 +52,7 @@ pub enum DataInput<'a> { } /// Owned counterpart to `DataInput` for heap storage. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, PartialOrd)] +#[derive(Clone, Debug, PartialEq, PartialOrd)] pub enum OwnedDataInput { /// A signed 8-bit integer. I8(i8), @@ -320,7 +319,7 @@ impl<'a> fmt::Display for HydraQuery<'a> { } /// Counter variants supported by Hydra. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub enum HydraCounter { /// Count-Min-backed counter. CountMinSketch(CountMinSketch, FastPath>), @@ -509,8 +508,7 @@ impl HydraCounter { } /// A key-count pair used in heap-based sketches for tracking heavy hitters. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct HeapEntry { /// Stored key. pub key: OwnedDataInput, diff --git a/src/common/structure_utils.rs b/src/common/structure_utils.rs index 381a8a7a..eb86ad7e 100644 --- a/src/common/structure_utils.rs +++ b/src/common/structure_utils.rs @@ -3,7 +3,6 @@ //! Vector2D: //! Vector3D: //! CommonHeap: -use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use crate::LN_ONE_MINUS_U_TABLE; @@ -50,7 +49,7 @@ impl ToF64 for i32 { /// boundaries so the schedule is one continuous geometric stream. /// /// Reference: . -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct Nitro { /// Whether Nitro sampling is enabled. pub is_nitro_mode: bool, @@ -63,27 +62,16 @@ pub struct Nitro { /// Integer part of the weight one admitted slot carries. pub delta: u64, idx: usize, - /// Unused; retained so the serialized field order is unchanged. + /// Unused except as the fourth value [`Nitro::get_ctx`] reports. mask: usize, /// splitmix64 state for the stochastic-rounding draw, advanced once per /// admitted slot whose `1/p` is not an integer. - /// - /// Serialized (with a default for payloads written before this field - /// existed), because it is part of what the sketch will do next: a decoded - /// sketch that restarted this stream from a fixed constant would emit a - /// different weight sequence than the uninterrupted run. - #[serde(default = "default_rounding_state")] rounding_state: u64, } -/// Seed the rounding stream starts from when it is not seeded explicitly, and -/// the value a payload written before `rounding_state` existed decodes to. +/// Seed the rounding stream starts from when it is not seeded explicitly. const ROUNDING_SEED: u64 = 0x2545_F491_4F6C_DD1D; -fn default_rounding_state() -> u64 { - ROUNDING_SEED -} - /// The complete sampling state of a [`Nitro`]: everything that decides which /// slots it admits next and at what weight. /// diff --git a/src/common/structures/bit_matrix.rs b/src/common/structures/bit_matrix.rs index 393233a9..ac346696 100644 --- a/src/common/structures/bit_matrix.rs +++ b/src/common/structures/bit_matrix.rs @@ -9,7 +9,6 @@ use super::matrix_storage::{ MatrixFastHash, MatrixStorage, cols_mask, cols_mask_bits, fold_to_col, }; -use serde::{Deserialize, Serialize}; const BITS_PER_WORD: usize = u64::BITS as usize; @@ -24,42 +23,6 @@ pub struct BitMatrix { mask: u128, } -// Only the stored fields travel; `words_per_row`, `mask_bits` and `mask` are -// functions of `cols` and are recomputed on decode. -#[derive(Serialize)] -#[serde(rename = "BitMatrix")] -struct BitMatrixSer<'a> { - words: &'a [u64], - rows: usize, - cols: usize, -} - -#[derive(Deserialize)] -#[serde(rename = "BitMatrix")] -struct BitMatrixDe { - words: Vec, - rows: usize, - cols: usize, -} - -impl Serialize for BitMatrix { - fn serialize(&self, serializer: S) -> Result { - BitMatrixSer { - words: &self.words, - rows: self.rows, - cols: self.cols, - } - .serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for BitMatrix { - fn deserialize>(deserializer: D) -> Result { - let input = BitMatrixDe::deserialize(deserializer)?; - Self::from_words(input.rows, input.cols, input.words).map_err(serde::de::Error::custom) - } -} - impl BitMatrix { /// The packed words, one row after another, each row padded out to a whole /// number of words. @@ -469,18 +432,17 @@ mod tests { let _ = BitMatrix::new(0, 64); } - /// Only `words`, `rows` and `cols` travel; the word stride and the fold - /// masks are recomputed, so a decoded matrix folds hashes exactly as the - /// original did. + /// `from_words` takes only `words`, `rows` and `cols`; the word stride and + /// the fold masks are recomputed, so a rebuilt matrix folds hashes exactly + /// as the original did. #[test] - fn a_round_trip_recomputes_the_derived_fields() { + fn a_rebuild_from_words_recomputes_the_derived_fields() { for (rows, cols) in GEOMETRIES { let mut m = BitMatrix::new(rows, cols); for row in 0..rows { m.set(row, (row * 7) % cols); } - let bytes = rmp_serde::to_vec(&m).expect("encode"); - let decoded: BitMatrix = rmp_serde::from_slice(&bytes).expect("decode"); + let decoded = BitMatrix::from_words(rows, cols, m.words().to_vec()).expect("rebuild"); assert_eq!((decoded.rows(), decoded.cols()), (rows, cols)); for row in 0..rows { @@ -538,45 +500,22 @@ mod tests { } } - #[derive(serde::Serialize)] - #[serde(rename = "BitMatrix")] - struct CraftedMatrix { - words: Vec, - rows: usize, - cols: usize, - } - /// A payload whose word count disagrees with its dimensions fails at /// decode rather than later, as an index panic or a silent alias. #[test] fn a_payload_that_does_not_fit_its_dimensions_is_rejected() { - let short = CraftedMatrix { - words: vec![0; 2], - rows: 3, - cols: 100, - }; - let bytes = rmp_serde::to_vec(&short).expect("encode"); - let err = rmp_serde::from_slice::(&bytes).expect_err("short payload accepted"); + let short = BitMatrix::from_words(3, 100, vec![0; 2]); + let err = short.expect_err("short payload accepted"); assert!( err.to_string().contains("needs 6 words, got 2"), "unexpected error: {err}" ); - let long = CraftedMatrix { - words: vec![0; 9], - rows: 3, - cols: 100, - }; - let bytes = rmp_serde::to_vec(&long).expect("encode"); - assert!(rmp_serde::from_slice::(&bytes).is_err()); - - let empty = CraftedMatrix { - words: vec![], - rows: 0, - cols: 100, - }; - let bytes = rmp_serde::to_vec(&empty).expect("encode"); - let err = rmp_serde::from_slice::(&bytes).expect_err("zero rows accepted"); + let long = BitMatrix::from_words(3, 100, vec![0; 9]); + assert!(long.is_err()); + + let empty = BitMatrix::from_words(0, 100, vec![]); + let err = empty.expect_err("zero rows accepted"); assert!( err.to_string().contains("needs both dimensions"), "unexpected error: {err}" @@ -588,31 +527,12 @@ mod tests { /// right length for a grid of `usize::MAX` rows. #[test] fn dimensions_whose_word_count_overflows_are_rejected() { - let overflowing = CraftedMatrix { - words: vec![], - rows: usize::MAX, - cols: 128, - }; - let bytes = rmp_serde::to_vec(&overflowing).expect("encode"); - let err = rmp_serde::from_slice::(&bytes).expect_err("overflow accepted"); + let overflowing = BitMatrix::from_words(usize::MAX, 128, vec![]); + let err = overflowing.expect_err("overflow accepted"); assert!( err.to_string().contains("dimensions overflow"), "unexpected error: {err}" ); } - /// The wire carries the three stored fields and nothing derived from them. - #[test] - fn the_wire_form_carries_only_the_stored_fields() { - let m = BitMatrix::new(3, 100); - let stored = CraftedMatrix { - words: vec![0; 6], - rows: 3, - cols: 100, - }; - assert_eq!( - rmp_serde::to_vec(&m).expect("encode"), - rmp_serde::to_vec(&stored).expect("encode") - ); - } } diff --git a/src/common/structures/fixed_structure.rs b/src/common/structures/fixed_structure.rs index 2ba3fa3c..5cebaa3e 100644 --- a/src/common/structures/fixed_structure.rs +++ b/src/common/structures/fixed_structure.rs @@ -1,8 +1,6 @@ //! A fixed integer matrix. //! Size fixed at compile time and heap-backed via Box. -use serde::{Deserialize, Serialize}; - /// Quick-start row count for fixed matrix aliases. pub const QUICKSTART_ROWS: usize = 5; /// Quick-start column count for fixed matrix aliases. @@ -23,9 +21,7 @@ pub const DEFAULT_COLS: usize = 4096; /// an inconsistent hand-written impl silently produces wrong estimates. /// [`impl_hll_registers!`](crate::impl_hll_registers) enforces them at /// compile time. -pub trait HllRegisterStorage: - Clone + std::fmt::Debug + Default + Serialize + for<'de> Deserialize<'de> -{ +pub trait HllRegisterStorage: Clone + std::fmt::Debug + Default { /// HLL precision parameter. const PRECISION: usize; /// Number of hash bits reserved for register-rank computation. @@ -139,67 +135,6 @@ macro_rules! impl_hll_registers { } } - impl $crate::__private::serde::Serialize for $name { - fn serialize(&self, serializer: S) -> ::std::result::Result - where - S: $crate::__private::serde::Serializer, - { - $crate::__private::serde_big_array::BigArray::serialize( - &*self.registers, - serializer, - ) - } - } - - impl<'de> $crate::__private::serde::Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> ::std::result::Result - where - D: $crate::__private::serde::Deserializer<'de>, - { - // Fills a heap-allocated array element by element. The - // encoding is the same fixed-length sequence `BigArray` - // writes, but decoding never builds an `[u8; N]` on the - // stack -- see `zeroed_registers`. - struct RegisterVisitor; - - impl<'de> $crate::__private::serde::de::Visitor<'de> for RegisterVisitor { - type Value = $name; - - fn expecting( - &self, - formatter: &mut ::std::fmt::Formatter, - ) -> ::std::fmt::Result { - ::std::write!(formatter, "an array of {} bytes", $num_registers) - } - - fn visit_seq( - self, - mut seq: A, - ) -> ::std::result::Result - where - A: $crate::__private::serde::de::SeqAccess<'de>, - { - let mut out = <$name as ::std::default::Default>::default(); - for i in 0..$num_registers { - match seq.next_element::()? { - Some(byte) => out.registers[i] = byte, - None => { - return Err( - ::invalid_length( - i, &self, - ), - ); - } - } - } - Ok(out) - } - } - - deserializer.deserialize_tuple($num_registers, RegisterVisitor) - } - } - impl ::std::ops::Index for $name { type Output = u8; @@ -291,28 +226,6 @@ macro_rules! impl_fixed_matrix { } } - impl $crate::__private::serde::Serialize for $name { - fn serialize(&self, serializer: S) -> Result - where - S: $crate::__private::serde::Serializer, - { - $crate::__private::serde_big_array::BigArray::serialize(&*self.data, serializer) - } - } - - impl<'de> $crate::__private::serde::Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: $crate::__private::serde::Deserializer<'de>, - { - let data: [$counter; $rows * $cols] = - $crate::__private::serde_big_array::BigArray::deserialize(deserializer)?; - Ok(Self { - data: Box::new(data), - }) - } - } - impl $crate::MatrixStorage for $name { type Counter = $counter; diff --git a/src/common/structures/heap.rs b/src/common/structures/heap.rs index 7c42608a..d8aecab7 100644 --- a/src/common/structures/heap.rs +++ b/src/common/structures/heap.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use std::ops::{Index, IndexMut}; /// Trait defining heap ordering behavior. @@ -13,7 +12,7 @@ pub trait CommonHeapOrder { } /// Root is the smallest value; bounded heap retains the largest values. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct RetainLargest; impl CommonHeapOrder for RetainLargest { @@ -30,7 +29,7 @@ impl CommonHeapOrder for RetainLargest { } /// Root is the largest value; bounded heap retains the smallest values. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct RetainSmallest; impl CommonHeapOrder for RetainSmallest { @@ -46,7 +45,7 @@ impl CommonHeapOrder for RetainSmallest { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq)] /// Capacity-bounded binary heap with pluggable ordering. pub struct CommonHeap> { data: Vec, @@ -75,6 +74,15 @@ impl> CommonHeap { } } + /// Wraps `data` that already satisfies the heap invariant for `order`. + pub(crate) fn from_heap_vec(data: Vec, capacity: usize, order: O) -> Self { + Self { + data, + size: capacity, + order, + } + } + /// Returns the number of elements currently in the heap. #[inline] pub fn len(&self) -> usize { diff --git a/src/common/structures/vector1d.rs b/src/common/structures/vector1d.rs index 39d2187b..bab3630d 100644 --- a/src/common/structures/vector1d.rs +++ b/src/common/structures/vector1d.rs @@ -1,8 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::ops::{Index, IndexMut, Range}; /// Shared thin wrapper over `Vec` tailored for sketches. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct Vector1D { data: Vec, } diff --git a/src/common/structures/vector2d.rs b/src/common/structures/vector2d.rs index a6ec9a22..83f1bacb 100644 --- a/src/common/structures/vector2d.rs +++ b/src/common/structures/vector2d.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use std::ops::{Index, IndexMut}; use crate::{ @@ -6,7 +5,7 @@ use crate::{ compute_median_inline_f64, }; /// Shared thin wrapper over `Vec` tailored for sketches. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug)] pub struct Vector2D { data: Vec, rows: usize, @@ -16,39 +15,6 @@ pub struct Vector2D { nitro: Nitro, } -// Helper type for deserialization: we only read stored fields and recompute -// derived ones (mask_bits, mask) from rows/cols. -#[derive(Deserialize)] -struct Vector2DDeserialize { - data: Vec, - rows: usize, - cols: usize, - #[serde(default)] - nitro: Nitro, -} - -impl<'de, T> Deserialize<'de> for Vector2D -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let input = Vector2DDeserialize::deserialize(deserializer)?; - let mask_bits = super::matrix_storage::cols_mask_bits(input.cols); - let mask = super::matrix_storage::cols_mask(input.cols); - Ok(Self { - data: input.data, - rows: input.rows, - cols: input.cols, - mask_bits, - mask, - nitro: input.nitro, - }) - } -} - impl Vector2D { /// Creates an empty matrix with reserved capacity for `rows * cols` elements. /// The underlying storage is left uninitialized until `fill` or similar methods are called, diff --git a/src/common/structures/vector3d.rs b/src/common/structures/vector3d.rs index 87f67f53..5ab5cc1f 100644 --- a/src/common/structures/vector3d.rs +++ b/src/common/structures/vector3d.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use super::matrix_storage::cols_mask_bits; use crate::MatrixFastHash; @@ -27,9 +26,8 @@ use crate::MatrixFastHash; /// /// `Vector2D<[T; N]>` has exactly the same memory layout. The one thing /// `Vector3D` adds is a **`bucket_len` chosen at run time**: it comes from a -/// configuration value (a HyperLogLog precision, a sub-window count) that is -/// also read back from deserialized bytes, which a const generic cannot -/// express. +/// configuration value (a HyperLogLog precision, a sub-window count), which a +/// const generic cannot express. /// /// # Layout /// @@ -54,80 +52,6 @@ pub struct Vector3D { mask: u128, } -// Only the stored fields travel, in both directions; `mask_bits` and `mask` -// are functions of `cols` and are recomputed on decode, so tampered bytes -// cannot produce inconsistent routing. The two field sets must match: a -// self-describing encoding ignores the extra keys, but the compact one is -// positional and would reject its own output. -#[derive(Serialize)] -#[serde(rename = "Vector3D")] -struct Vector3DSer<'a, T> { - data: &'a [T], - rows: usize, - cols: usize, - bucket_len: usize, -} - -#[derive(Deserialize)] -#[serde(rename = "Vector3D")] -struct Vector3DDeserialize { - data: Vec, - rows: usize, - cols: usize, - bucket_len: usize, -} - -impl Serialize for Vector3D { - fn serialize(&self, serializer: S) -> Result { - Vector3DSer { - data: &self.data, - rows: self.rows, - cols: self.cols, - bucket_len: self.bucket_len, - } - .serialize(serializer) - } -} - -impl<'de, T> Deserialize<'de> for Vector3D -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let input = Vector3DDeserialize::deserialize(deserializer)?; - if input.rows == 0 || input.cols == 0 || input.bucket_len == 0 { - return Err(serde::de::Error::custom( - "Vector3D: rows, cols and bucket_len must all be non-zero", - )); - } - let expected = input - .rows - .checked_mul(input.cols) - .and_then(|n| n.checked_mul(input.bucket_len)) - .ok_or_else(|| { - serde::de::Error::custom("Vector3D: rows * cols * bucket_len overflows") - })?; - if input.data.len() != expected { - return Err(serde::de::Error::custom(format!( - "Vector3D: data length {} does not match rows * cols * bucket_len = {expected}", - input.data.len() - ))); - } - let mask_bits = cols_mask_bits(input.cols); - Ok(Self { - data: input.data, - rows: input.rows, - cols: input.cols, - bucket_len: input.bucket_len, - mask_bits, - mask: (1u128 << mask_bits) - 1, - }) - } -} - impl Vector3D { /// Creates a container sized for `rows * cols * bucket_len` elements, with the /// storage left empty. @@ -400,35 +324,6 @@ mod tests { assert_eq!(touched, 3); } - #[test] - fn serde_round_trip_preserves_shape_and_contents() { - let mut v: Vector3D = Vector3D::unfilled(2, 8, 3); - v.fill(0); - let at = (8 + 5) * 3 + 2; // bucket (1, 5), slot 2 - v.as_mut_slice()[at] = 200; - let bytes = rmp_serde::to_vec_named(&v).expect("serialize"); - let back: Vector3D = rmp_serde::from_slice(&bytes).expect("deserialize"); - assert_eq!((back.rows(), back.cols(), back.bucket_len()), (2, 8, 3)); - assert_eq!(back.as_slice(), v.as_slice()); - // Derived routing state is recomputed, not carried on the wire. - assert_eq!(back.mask_bits(), v.mask_bits()); - } - - /// The compact encoding is positional, so `Serialize` and `Deserialize` - /// must agree on the field set exactly. A self-describing encoding hides a - /// mismatch by ignoring the extra keys; this one rejects its own output. - #[test] - fn serde_round_trip_also_holds_for_the_compact_encoding() { - let mut v: Vector3D = Vector3D::unfilled(2, 8, 3); - v.fill(0); - v.as_mut_slice()[(8 + 5) * 3 + 2] = 200; - let bytes = rmp_serde::to_vec(&v).expect("serialize"); - let back: Vector3D = rmp_serde::from_slice(&bytes).expect("deserialize"); - assert_eq!((back.rows(), back.cols(), back.bucket_len()), (2, 8, 3)); - assert_eq!(back.as_slice(), v.as_slice()); - assert_eq!(back.mask_bits(), v.mask_bits()); - } - #[test] #[should_panic(expected = "overflows usize")] fn unfilled_rejects_dimensions_whose_product_overflows() { @@ -448,53 +343,4 @@ mod tests { let _ = v.bucket_slice(0, 5); } - #[test] - fn deserialize_rejects_a_data_length_that_contradicts_the_dimensions() { - #[derive(Serialize)] - struct Forged { - data: Vec, - rows: usize, - cols: usize, - bucket_len: usize, - } - // 2 * 8 * 3 = 48 elements are required; supply 47. - let forged = Forged { - data: vec![0u8; 47], - rows: 2, - cols: 8, - bucket_len: 3, - }; - let bytes = rmp_serde::to_vec_named(&forged).expect("serialize"); - let err = rmp_serde::from_slice::>(&bytes) - .expect_err("a short payload must be rejected"); - assert!( - err.to_string() - .contains("does not match rows * cols * bucket_len"), - "unexpected error: {err}" - ); - } - - #[test] - fn deserialize_rejects_zero_dimensions() { - #[derive(Serialize)] - struct Forged { - data: Vec, - rows: usize, - cols: usize, - bucket_len: usize, - } - let forged = Forged { - data: Vec::new(), - rows: 2, - cols: 0, - bucket_len: 3, - }; - let bytes = rmp_serde::to_vec_named(&forged).expect("serialize"); - let err = - rmp_serde::from_slice::>(&bytes).expect_err("zero cols must be rejected"); - assert!( - err.to_string().contains("must all be non-zero"), - "unexpected error: {err}" - ); - } } diff --git a/src/lib.rs b/src/lib.rs index b09b6537..dd11f5b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,12 +25,6 @@ pub mod sketches; #[cfg(test)] pub mod test_utils; -#[doc(hidden)] -pub mod __private { - pub use serde; - pub use serde_big_array; -} - pub use common::*; pub use asapv1::envelope::peek_kind_id; pub use sketch_framework::*; diff --git a/src/sketch_framework/eh_bucket_sketch.rs b/src/sketch_framework/eh_bucket_sketch.rs index c1330f2a..b0d5b0a8 100644 --- a/src/sketch_framework/eh_bucket_sketch.rs +++ b/src/sketch_framework/eh_bucket_sketch.rs @@ -2,7 +2,6 @@ use crate::asapv1::admission_rate; use crate::{DataInput, Vector2D}; -use serde::{Deserialize, Serialize}; use super::super::sketches::*; use super::UnivMon; @@ -10,7 +9,7 @@ use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; pub(crate) mod wire; -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] /// Norm choices used by EH merge policies. pub enum SketchNorm { /// L1-based merge policy. @@ -19,7 +18,7 @@ pub enum SketchNorm { L2, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] /// Erased sketch variants supported by exponential histograms. pub enum EhBucketSketch { /// Count-Min sketch. @@ -336,10 +335,10 @@ mod tests { assert!((10.0..=30.0).contains(&q50), "unexpected q50 {q50}"); } - /// `EhBucketSketch` derives its serde form from the variants it holds, so a - /// round trip of a nested DDSketch must carry the sketch's whole state. + /// An ASAPv1 round trip of a nested DDSketch carries the sketch's whole + /// state. #[test] - fn nested_ddsketch_survives_a_serde_round_trip() { + fn nested_ddsketch_survives_an_asapv1_round_trip() { let mut dd = EhBucketSketch::DdSketch(DdSketch::new(0.01)); for v in [10.0f64, 20.0, 30.0, 40.0] { dd.update(&DataInput::F64(v)); @@ -351,8 +350,8 @@ mod tests { _ => panic!("expected the DDSketch variant"), }; - let bytes = rmp_serde::to_vec(&dd).expect("encode"); - let restored: EhBucketSketch = rmp_serde::from_slice(&bytes).expect("decode"); + let bytes = dd.to_asapv1().expect("encode"); + let restored = EhBucketSketch::from_asapv1(&bytes).expect("decode"); let after = match &restored { EhBucketSketch::DdSketch(inner) => { (inner.get_count(), inner.sum(), inner.min(), inner.max()) diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index c1b98298..4a153775 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -20,7 +20,6 @@ //! - Manousis et al., VLDB 2022. //! -use serde::{Deserialize, Serialize}; use crate::Vector2D; use crate::input::{HydraCounter, HydraQuery}; @@ -57,10 +56,8 @@ fn push_escaped(out: &mut String, s: &str) { /// no longer collide, and a projection of a wide row can no longer alias the full /// key of a narrow one. /// -/// Serializes as a plain array of labels; the escaped cache is rebuilt and the -/// labels re-validated on decode via `TryFrom>`. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(try_from = "Vec", into = "Vec")] +/// `TryFrom>` validates the labels and builds the escaped cache. +#[derive(Clone, Debug)] pub struct KeySchema { labels: Vec, escaped_labels: Vec, @@ -184,7 +181,7 @@ impl KeySchema { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] /// Sketch grid for subpopulation queries. pub struct Hydra { /// Number of rows in the sketch grid. diff --git a/src/sketch_framework/nitro.rs b/src/sketch_framework/nitro.rs index 55dea103..2e6c51d5 100644 --- a/src/sketch_framework/nitro.rs +++ b/src/sketch_framework/nitro.rs @@ -8,7 +8,6 @@ use rand::rngs::SmallRng; use rand::{Rng, SeedableRng, rng}; -use serde::{Deserialize, Serialize}; use crate::{ CountMinSketch, CountSketch, DataInput, DefaultXxHasher, FastPath, FastPathHasher, @@ -119,7 +118,7 @@ impl NitroEstimate for CountSketch, FastPath> { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] /// Batch-oriented Nitro wrapper around a sketch target. pub struct NitroBatch { sampling_rate: f64, @@ -128,15 +127,10 @@ pub struct NitroBatch { inv_ln_one_minus_p: f64, /// Weight applied to each sampled update. pub delta: u64, - /// Live sampling RNG for [`NitroBatch::update_batch`]. Not serialized: a decoded - /// batch restarts it from the OS, so an `update_batch` stream does not resume - /// across a round trip. The cached path's state (`idx`, `to_skip`) is - /// serialized and does resume. - #[serde(skip)] - #[serde(default = "new_small_rng")] + /// Live sampling RNG for [`NitroBatch::update_batch`]. generator: SmallRng, idx: usize, - /// Unused; retained so the serialized field order is unchanged. + /// Unused except as the fourth value [`NitroBatch::get_ctx`] reports. mask: usize, sk: S, } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index dfe51f8e..66ce5d7c 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -31,7 +31,6 @@ use crate::common::{ }; use crate::octo_delta::UnivMonCellDelta; use crate::sketches::countsketch_l2_norm::CountSketchWithL2Norm; -use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; pub(crate) mod wire; @@ -40,7 +39,7 @@ const DEFAULT_ROWS: usize = 5; const DEFAULT_COLS: usize = 2048; const DEFAULT_LAYERS: usize = 8; -#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(crate) enum UnivMonUpdateMode { #[default] Unset, @@ -48,7 +47,7 @@ pub(crate) enum UnivMonUpdateMode { Terminal, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Clone, Debug)] /// UnivMon sketch pyramid for multi-metric stream estimation. pub struct UnivMon { /// Per-layer L2/heavy-hitter sketches. @@ -65,9 +64,7 @@ pub struct UnivMon { pub heap_size: usize, /// Total inserted weight (the exact L1 norm). pub total_weight: usize, - #[serde(default)] update_mode: UnivMonUpdateMode, - #[serde(default)] candidate_complete: Vec, } diff --git a/src/sketch_framework/univmon_q.rs b/src/sketch_framework/univmon_q.rs index cd5920fd..5895e570 100644 --- a/src/sketch_framework/univmon_q.rs +++ b/src/sketch_framework/univmon_q.rs @@ -33,7 +33,7 @@ const OCCURRENCE_HASH_SEED_DOMAIN: usize = usize::MAX / 3; static NEXT_SOURCE_ID: AtomicU64 = AtomicU64::new(1); /// Memory, accuracy, and hashing controls for [`UnivMonQ`]. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct UnivMonQConfig { /// Number of geometrically sampled logical substreams. pub layers: usize, @@ -238,13 +238,13 @@ impl HashLayout { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] enum Counters { I32(Vec), I64(Vec), } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] struct PackedCountSketch { width: usize, depth: usize, diff --git a/src/sketches/bloom.rs b/src/sketches/bloom.rs index 805a6db1..e8b9f9a0 100644 --- a/src/sketches/bloom.rs +++ b/src/sketches/bloom.rs @@ -19,7 +19,6 @@ use crate::{ BitMatrix, DataInput, DefaultXxHasher, FastPath, FastPathHasher, MATRIX_MAX_ROWS, MatrixStorage, RegularPath, SketchHasher, }; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; mod wire; @@ -67,58 +66,6 @@ pub struct Bloom { _hasher: PhantomData, } -#[derive(Serialize)] -#[serde(rename = "Bloom")] -struct BloomSer<'a> { - bits: &'a BitMatrix, - inserted: u64, - mode: &'static str, -} - -#[derive(Deserialize)] -#[serde(rename = "Bloom")] -struct BloomDe { - bits: BitMatrix, - inserted: u64, - mode: String, -} - -impl Serialize for Bloom { - fn serialize(&self, serializer: S) -> Result { - BloomSer { - bits: &self.bits, - inserted: self.inserted, - mode: Mode::MODE_TAG, - } - .serialize(serializer) - } -} - -impl<'de, Mode: BloomMode, H: SketchHasher> Deserialize<'de> for Bloom { - fn deserialize>(deserializer: D) -> Result { - let input = BloomDe::deserialize(deserializer)?; - if input.mode != Mode::MODE_TAG { - return Err(serde::de::Error::custom(format!( - "bloom filter was built on the {} hash path, cannot decode as {}", - input.mode, - Mode::MODE_TAG - ))); - } - if input.bits.rows() > BLOOM_MAX_SLICES { - return Err(serde::de::Error::custom(format!( - "bloom filter has {} slices, past BLOOM_MAX_SLICES {BLOOM_MAX_SLICES}", - input.bits.rows() - ))); - } - Ok(Self { - bits: input.bits, - inserted: input.inserted, - _mode: PhantomData, - _hasher: PhantomData, - }) - } -} - impl Default for Bloom { fn default() -> Self { Self::with_dimensions(BLOOM_DEFAULT_ROWS, BLOOM_DEFAULT_COLS) @@ -339,7 +286,6 @@ mod tests { use super::*; const WIDE_COLS: usize = 1 << 10; - const NARROW_COLS: usize = 16; fn keys(count: u64) -> Vec> { (0..count).map(DataInput::U64).collect() @@ -532,25 +478,6 @@ mod tests { ); } - /// The plain serde form carries the grid dimensions, so it is a second door - /// into a filter construction would refuse. It fails closed on the same - /// bound rather than panicking inside a decoder. - #[test] - fn a_serde_payload_past_the_slice_cap_is_rejected() { - let bytes = rmp_serde::to_vec(&BloomSer { - bits: &BitMatrix::new(BLOOM_MAX_SLICES + 1, NARROW_COLS), - inserted: 0, - mode: ::MODE_TAG, - }) - .expect("encode"); - let err = rmp_serde::from_slice::>(&bytes) - .expect_err("a filter past the slice cap must not decode"); - assert!( - err.to_string().contains("BLOOM_MAX_SLICES"), - "the slice cap must be named in the rejection, got {err}" - ); - } - /// Both rate reporters raise a per-slice probability to the slice count, so /// a reporter reading the wrong count would report a rate the filter cannot /// deliver. diff --git a/src/sketches/bloom/wire.rs b/src/sketches/bloom/wire.rs index c1512f33..fa792ea1 100644 --- a/src/sketches/bloom/wire.rs +++ b/src/sketches/bloom/wire.rs @@ -620,4 +620,28 @@ mod tests { "decoded filter delivered {measured}, over its predicted {predicted} (+{band})" ); } + + /// A known bit grid serializes to the `asapv1_golden/` bytes. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + use std::marker::PhantomData; + + fn bloom(rows: usize, cols: usize, words: &[u64], inserted: u64) -> Bloom { + Bloom { + bits: BitMatrix::from_words(rows, cols, words.to_vec()).expect("valid grid"), + inserted, + _mode: PhantomData, + _hasher: PhantomData, + } + } + + let regular = bloom::(3, 128, &[1, 1 << 63, 0, 0xff, u64::MAX, 0x10000], 300); + let golden = hex(include_str!("../../../asapv1_golden/bloom_regular_3x128.hex")); + assert_eq!(regular.to_asapv1().unwrap(), golden); + + let fast = bloom::(5, 32, &[1, 0x8000_0000, 0, 0xffff_ffff, 0x10000], 9); + let golden = hex(include_str!("../../../asapv1_golden/bloom_fast_5x32.hex")); + assert_eq!(fast.to_asapv1().unwrap(), golden); + } } diff --git a/src/sketches/coco.rs b/src/sketches/coco.rs index ac21d877..84e9c54f 100644 --- a/src/sketches/coco.rs +++ b/src/sketches/coco.rs @@ -16,30 +16,26 @@ use crate::{DataInput, DefaultXxHasher, SketchHasher, Vector2D}; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; -use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::marker::PhantomData; mod wire; /// One table slot: the key it currently represents and the mass attributed to it. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct CocoBucket { pub full_key: Option, pub val: u64, } -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct CocoSketch { pub cols: usize, pub rows: usize, pub table: Vector2D, /// Source of the tie-break and election draws. Not serialized: a decoded /// sketch draws from the thread generator. - #[serde(skip)] rng: CocoRng, - #[serde(skip)] _hasher: PhantomData, } diff --git a/src/sketches/count_min_fold.rs b/src/sketches/count_min_fold.rs index 50f884ef..c1edb6b2 100644 --- a/src/sketches/count_min_fold.rs +++ b/src/sketches/count_min_fold.rs @@ -15,7 +15,6 @@ //! additional approximation error — the accuracy is identical to a full-width //! CMS with W columns. -use serde::{Deserialize, Serialize}; use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap}; use std::marker::PhantomData; @@ -27,7 +26,7 @@ const LOWER_32_MASK: u64 = (1u64 << 32) - 1; // --------------------------------------------------------------------------- /// A single tagged counter in a folded cell. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct FoldEntry { /// Column index in the target full-width CMS (permanent address). pub full_col: u16, @@ -41,7 +40,7 @@ pub struct FoldEntry { /// - `Single` — exactly one `full_col` present (no heap allocation). /// - `Collided` — two or more distinct `full_col` values share this physical /// column; entries are stored in a `Vec`. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default)] pub enum FoldCell { /// Cell with no stored entries. #[default] @@ -200,8 +199,7 @@ impl<'a> Iterator for FoldCellIter<'a> { /// Each physical cell lazily tracks which full-CMS column(s) it holds, /// expanding only on real collisions. When sub-windows are merged the columns /// are "unfolded" back towards the full-width CMS. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct CountMinSketchFold { /// Number of hash functions (rows). Same across all fold levels. rows: usize, @@ -215,7 +213,6 @@ pub struct CountMinSketchFold { cells: Vec, /// Top-K heavy-hitter tracking heap. heap: TopKHeap, - #[serde(skip)] _hasher: PhantomData, } @@ -1077,67 +1074,6 @@ mod tests { } } - // -- Serde round trip --------------------------------------------------- - - /// The derived serde form is the only serialization CountMinSketchFold has, so the - /// round trip has to carry the whole sketch: the folded cells with their - /// permanent `full_col` tags, the geometry the unfold path reads, and the - /// heavy-hitter heap (whose index is rebuilt on load rather than carried). - #[test] - fn serde_round_trip_keeps_the_cells_the_geometry_and_the_heap() { - let rows = 3; - let full_cols = 4096; - let fold_level = 4; - let top_k = 16; - - let mut sk: CountMinSketchFold = - CountMinSketchFold::new(rows, full_cols, fold_level, top_k); - let stream = sample_zipf_u64(2_000, 1.2, 20_000, 0x5EED_C45C); - for &value in &stream { - sk.update_weighted(&DataInput::U64(value), 1); - } - assert!( - sk.collided_cells() > 0, - "the fixture should exercise collided cells, not just Single ones" - ); - - let bytes = rmp_serde::to_vec(&sk).expect("encode"); - let restored: CountMinSketchFold = rmp_serde::from_slice(&bytes).expect("decode"); - - assert_eq!(restored.rows(), sk.rows()); - assert_eq!(restored.fold_cols(), sk.fold_cols()); - assert_eq!(restored.full_cols(), sk.full_cols()); - assert_eq!(restored.fold_level(), sk.fold_level()); - assert_eq!(restored.total_entries(), sk.total_entries()); - assert_eq!(restored.collided_cells(), sk.collided_cells()); - assert_eq!(restored.to_flat_counters(), sk.to_flat_counters()); - - for value in 0..2_000u64 { - let key = DataInput::U64(value); - assert_eq!( - restored.estimate(&key), - sk.estimate(&key), - "estimate for {value} moved across the round trip" - ); - } - - let before = sk.heap().heap(); - let after = restored.heap().heap(); - assert_eq!(after.len(), before.len(), "heap size moved"); - assert_eq!(after.len(), top_k, "the fixture should fill the heap"); - for (a, b) in after.iter().zip(before.iter()) { - assert_eq!(a.key, b.key, "heap order moved across the round trip"); - assert_eq!(a.count, b.count, "heap count moved across the round trip"); - } - // The heap's key index is `#[serde(skip)]` and rebuilt on load, so a - // lookup has to work on the decoded side too. - let resident = before[0].key.clone(); - assert!( - restored.heap().find_owned(&resident).is_some(), - "the rebuilt heap index cannot find its own resident" - ); - } - // -- Memory efficiency -------------------------------------------------- #[test] diff --git a/src/sketches/count_sketch_fold.rs b/src/sketches/count_sketch_fold.rs index 0afadc09..e2b976d1 100644 --- a/src/sketches/count_sketch_fold.rs +++ b/src/sketches/count_sketch_fold.rs @@ -14,7 +14,6 @@ //! additional approximation error — the accuracy is identical to a full-width //! Count Sketch with W columns. -use serde::{Deserialize, Serialize}; use crate::count_min_fold::FoldCell; use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap, compute_median_inline_f64}; @@ -35,8 +34,7 @@ const LOWER_32_MASK: u64 = (1u64 << 32) - 1; /// /// Unlike `CountMinSketchFold`, insert applies a random ±1 sign per (row, key), and /// query returns the **median** of sign-corrected row estimates. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct CountSketchFold { /// Number of hash functions (rows). Same across all fold levels. rows: usize, @@ -50,7 +48,6 @@ pub struct CountSketchFold { cells: Vec, /// Top-K heavy-hitter tracking heap. heap: TopKHeap, - #[serde(skip)] _hasher: PhantomData, } @@ -828,66 +825,6 @@ mod tests { } } - // -- Serde round trip --------------------------------------------------- - - /// The derived serde form is the only serialization CountSketchFold has, so the - /// round trip has to carry the whole sketch: the folded cells with their - /// permanent `full_col` tags, the geometry the unfold path reads, and the - /// heavy-hitter heap (whose index is rebuilt on load rather than carried). - #[test] - fn serde_round_trip_keeps_the_cells_the_geometry_and_the_heap() { - let rows = 3; - let full_cols = 4096; - let fold_level = 4; - let top_k = 16; - - let mut sk: CountSketchFold = CountSketchFold::new(rows, full_cols, fold_level, top_k); - let stream = sample_zipf_u64(2_000, 1.2, 20_000, 0x5EED_C55C); - for &value in &stream { - sk.update_weighted(&DataInput::U64(value), 1); - } - assert!( - sk.collided_cells() > 0, - "the fixture should exercise collided cells, not just Single ones" - ); - - let bytes = rmp_serde::to_vec(&sk).expect("encode"); - let restored: CountSketchFold = rmp_serde::from_slice(&bytes).expect("decode"); - - assert_eq!(restored.rows(), sk.rows()); - assert_eq!(restored.fold_cols(), sk.fold_cols()); - assert_eq!(restored.full_cols(), sk.full_cols()); - assert_eq!(restored.fold_level(), sk.fold_level()); - assert_eq!(restored.total_entries(), sk.total_entries()); - assert_eq!(restored.collided_cells(), sk.collided_cells()); - assert_eq!(restored.to_flat_counters(), sk.to_flat_counters()); - - for value in 0..2_000u64 { - let key = DataInput::U64(value); - assert_eq!( - restored.estimate(&key), - sk.estimate(&key), - "estimate for {value} moved across the round trip" - ); - } - - let before = sk.heap().heap(); - let after = restored.heap().heap(); - assert_eq!(after.len(), before.len(), "heap size moved"); - assert_eq!(after.len(), top_k, "the fixture should fill the heap"); - for (a, b) in after.iter().zip(before.iter()) { - assert_eq!(a.key, b.key, "heap order moved across the round trip"); - assert_eq!(a.count, b.count, "heap count moved across the round trip"); - } - // The heap's key index is `#[serde(skip)]` and rebuilt on load, so a - // lookup has to work on the decoded side too. - let resident = before[0].key.clone(); - assert!( - restored.heap().find_owned(&resident).is_some(), - "the rebuilt heap index cannot find its own resident" - ); - } - // -- Memory efficiency -------------------------------------------------- #[test] diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 3e58b4ca..a4ff220e 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -8,11 +8,10 @@ //! Sketch and its Applications," J. Algorithms 55(1), 2005. //! -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::structure_utils::AdmittedRows; -use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::admit_all; use crate::octo_delta::{ CountMinSketchCellDelta, KeyedCountMinSketchCellDelta, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_MAX_PROMOTION_THRESHOLD, @@ -33,8 +32,7 @@ pub use crate::common::structures::fixed_structure::{QUICKSTART_COLS, QUICKSTART const LOWER_32_MASK: u64 = (1u64 << 32) - 1; /// A Count-Min Sketch for estimating item frequencies in a data stream. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound(serialize = "S: Serialize", deserialize = "S: Deserialize<'de>"))] +#[derive(Clone, Debug)] pub struct CountMinSketch< S: MatrixStorage = Vector2D, Mode = RegularPath, @@ -43,11 +41,8 @@ pub struct CountMinSketch< counts: S, rows: usize, cols: usize, - #[serde(skip)] _mode: PhantomData, - #[serde(skip)] _hasher: PhantomData, - #[serde(default = "admit_all", deserialize_with = "deserialize_rate")] admission_rate: f64, } diff --git a/src/sketches/countminsketch_hll.rs b/src/sketches/countminsketch_hll.rs index 3088e610..9ca80d57 100644 --- a/src/sketches/countminsketch_hll.rs +++ b/src/sketches/countminsketch_hll.rs @@ -36,8 +36,8 @@ //! //! # Status //! -//! Experimental: behind the `experimental` cargo feature, and its API and -//! wire form may change without a major version bump. For a per-key distinct +//! Experimental: behind the `experimental` cargo feature, and its API may +//! change without a major version bump. For a per-key distinct //! count on a stable API today, reach for `Hydra` with `HydraCounter::Hll`. //! //! # Related sketches @@ -59,10 +59,6 @@ use crate::sketches::hll::classic_estimate; use crate::{DataInput, DefaultXxHasher, SketchHasher, Vector3D}; -use rmp_serde::{ - decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice, to_vec_named, -}; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; const DEFAULT_ROWS: usize = 4; @@ -80,78 +76,14 @@ const HLL_SEED: usize = 1; /// selected `(row, col)` bucket holds a `2^precision`-register HyperLogLog that /// records the item. See the [module docs](crate::sketches::countminsketch_hll) for /// the supported queries and the performance notes for the optimization strategy. -#[derive(Clone, Debug, Serialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct CountMinHll { buckets: Vector3D, precision: u32, - #[serde(skip)] p_mask: u64, - #[serde(skip)] _hasher: PhantomData, } -// Seed struct: only the two authoritative fields are read from the wire. -// The derived p_mask is recomputed on load, and column routing is owned by -// Vector3D, so stale or tampered bytes cannot produce inconsistent routing. -#[derive(Deserialize)] -struct CountMinHllSeed { - buckets: Vector3D, - precision: u32, -} - -impl<'de, H: SketchHasher> Deserialize<'de> for CountMinHll { - fn deserialize>(deserializer: D) -> Result { - let CountMinHllSeed { buckets, precision } = CountMinHllSeed::deserialize(deserializer)?; - let rows = buckets.rows(); - let cols = buckets.cols(); - if !(1..=18).contains(&precision) { - return Err(serde::de::Error::custom(format!( - "precision {precision} out of range 1..=18" - ))); - } - let expected_depth = 1usize << precision; - if buckets.bucket_len() != expected_depth { - return Err(serde::de::Error::custom(format!( - "buckets bucket_len {} does not match 2^precision {} = {expected_depth}", - buckets.bucket_len(), - precision - ))); - } - if cols < 2 || !cols.is_power_of_two() { - return Err(serde::de::Error::custom(format!( - "cols ({cols}) must be a power of two and at least 2" - ))); - } - let col_bits = buckets.mask_bits() as usize; - let required_bits = rows.saturating_mul(col_bits); - if required_bits > 128 { - return Err(serde::de::Error::custom(format!( - "rows ({rows}) × column bits ({col_bits}) = {required_bits} exceeds the \ - 128-bit packed column hash; reduce rows or cols" - ))); - } - // A register holds a rank drawn from the 64 - precision bits below the - // register index, so `64 - precision + 1` is the largest an insert can - // write. Above that is a state no stream produces, and it skews the - // harmonic sum the estimator builds. `hll/wire.rs` holds the standalone - // sketch to the same bound. - let max_rank = (64 - precision + 1) as u8; - if let Some(&rank) = buckets.as_slice().iter().find(|&&rank| rank > max_rank) { - return Err(serde::de::Error::custom(format!( - "register value {rank} exceeds the maximum rank {max_rank} at \ - precision {precision}" - ))); - } - Ok(Self { - buckets, - precision, - p_mask: (1u64 << precision) - 1, - _hasher: PhantomData, - }) - } -} - impl Default for CountMinHll { fn default() -> Self { Self::with_dimensions(DEFAULT_ROWS, DEFAULT_COLS, DEFAULT_PRECISION) @@ -306,15 +238,6 @@ impl CountMinHll { Ok(()) } - /// Serializes the sketch into MessagePack bytes. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) - } - - /// Deserializes a sketch from MessagePack bytes. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) - } } #[cfg(test)] @@ -507,24 +430,6 @@ mod tests { ); } - #[test] - fn serialize_round_trip_preserves_estimates() { - let mut sk = CountMinHll::::with_dimensions(4, 32, 8); - let k = key("user_A"); - for i in 0..1500u64 { - sk.update(&k, &val(i)); - } - let bytes = sk.serialize_to_bytes().expect("serialize"); - let restored = - CountMinHll::::deserialize_from_bytes(&bytes).expect("decode"); - - assert_eq!(sk.rows(), restored.rows()); - assert_eq!(sk.cols(), restored.cols()); - assert_eq!(sk.precision(), restored.precision()); - assert_eq!(sk.as_storage().as_slice(), restored.as_storage().as_slice()); - assert_eq!(sk.estimate(&k), restored.estimate(&k)); - } - #[test] fn update_batch_matches_sequential_updates() { let k = key("user_A"); @@ -560,111 +465,4 @@ mod tests { assert_eq!(sk.rows(), 21); } - /// Every guard in `deserialize` gets a payload that trips it. - /// - /// These are forged, not merely decoded: the point is that a crafted - /// payload cannot produce a sketch whose accessors read out of bounds or - /// whose routing shifts past the packed hash. A test that decodes valid - /// bytes and asserts an invariant of the *constructor* proves nothing - /// about the decoder. - #[test] - fn deserialize_rejects_every_state_the_sketch_cannot_represent() { - #[derive(Serialize)] - struct Forged { - buckets: Vector3D, - precision: u32, - } - fn attempt(rows: usize, cols: usize, depth: usize, precision: u32) -> String { - let mut buckets: Vector3D = Vector3D::unfilled(rows, cols, depth); - buckets.fill(0); - let bytes = rmp_serde::to_vec_named(&Forged { buckets, precision }).expect("encode"); - CountMinHll::::deserialize_from_bytes(&bytes) - .err() - .map(|e| e.to_string()) - .unwrap_or_default() - } - - // A sane payload decodes, so the cases below fail for their own - // reason and not because the shape was wrong to begin with. - assert_eq!( - attempt(4, 32, 256, 8), - "", - "the control payload must decode" - ); - - for (label, err, want) in [ - ("precision 0", attempt(4, 32, 1, 0), "out of range"), - ("precision 19", attempt(4, 32, 1, 19), "out of range"), - ( - "bucket_len mismatch", - attempt(4, 32, 128, 8), - "does not match", - ), - ( - "cols not a power of two", - attempt(4, 17, 256, 8), - "power of two", - ), - ("single column", attempt(4, 1, 256, 8), "at least 2"), - // 22 rows x 6 column bits = 132 > 128. - ("rows past the hash", attempt(22, 64, 256, 8), "exceeds the"), - ] { - assert!( - err.contains(want), - "{label}: expected an error containing {want:?}, got {err:?}" - ); - } - } - - /// A register holds a rank of at most `64 - precision + 1`. A byte above - /// that is a state no insert produces, and it skews the harmonic sum every - /// estimate over that bucket is built from. `hll/wire.rs` holds the - /// standalone sketch to the same bound. - #[test] - fn deserialize_rejects_a_register_holding_an_impossible_rank() { - #[derive(Serialize)] - struct Forged { - buckets: Vector3D, - precision: u32, - } - fn attempt(precision: u32, rank: u8) -> Option { - let depth = 1usize << precision; - let mut buckets: Vector3D = Vector3D::unfilled(4, 32, depth); - buckets.fill(0); - buckets.as_mut_slice()[0] = rank; - let bytes = rmp_serde::to_vec_named(&Forged { buckets, precision }).expect("encode"); - CountMinHll::::deserialize_from_bytes(&bytes) - .err() - .map(|e| e.to_string()) - } - - for precision in [1u32, 8, 18] { - let max_rank = (64 - precision + 1) as u8; - assert!( - attempt(precision, max_rank).is_none(), - "precision {precision}: the largest rank an insert can write must decode" - ); - let err = attempt(precision, max_rank + 1).unwrap_or_else(|| { - panic!( - "precision {precision}: rank {} must be rejected", - max_rank + 1 - ) - }); - assert!(err.contains("exceeds the maximum rank"), "{err}"); - let err = attempt(precision, u8::MAX) - .unwrap_or_else(|| panic!("precision {precision}: rank 255 must be rejected")); - assert!(err.contains("exceeds the maximum rank"), "{err}"); - } - } - - #[test] - fn deserialize_recomputes_derived_fields() { - let sk = CountMinHll::::with_dimensions(4, 32, 8); - let bytes = sk.serialize_to_bytes().expect("serialize"); - let restored = - CountMinHll::::deserialize_from_bytes(&bytes).expect("decode"); - assert_eq!(restored.p_mask, (1u64 << 8) - 1); - // Column routing is owned by the storage and rebuilt from `cols`. - assert_eq!(restored.as_storage().mask_bits(), 5); // 32.ilog2() - } } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 8feb53b5..645ef81b 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -8,13 +8,12 @@ //! ICALP 2002. use crate::common::structure_utils::AdmittedRows; -use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::admit_all; use crate::{ DataInput, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, DefaultXxHasher, FastPath, FastPathHasher, FixedMatrix, MatrixFastHash, MatrixStorage, NitroTarget, QuickMatrixI64, QuickMatrixI128, RegularPath, SketchHasher, Vector2D, hash64_seeded, nitro_delta_saturated_i32, }; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use std::ops::Neg; @@ -27,8 +26,7 @@ use crate::common::structures::fixed_structure::{DEFAULT_COLS, DEFAULT_ROWS}; const LOWER_32_MASK: u64 = (1u64 << 32) - 1; /// A frequency-estimation sketch using random sign projections (Count Sketch). -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound(serialize = "S: Serialize", deserialize = "S: Deserialize<'de>"))] +#[derive(Clone, Debug)] pub struct CountSketch< S: MatrixStorage = Vector2D, Mode = RegularPath, @@ -37,11 +35,8 @@ pub struct CountSketch< counts: S, rows: usize, cols: usize, - #[serde(skip)] _mode: PhantomData, - #[serde(skip)] _hasher: PhantomData, - #[serde(default = "admit_all", deserialize_with = "deserialize_rate")] admission_rate: f64, } diff --git a/src/sketches/countsketch_l2_norm.rs b/src/sketches/countsketch_l2_norm.rs index 1e98bc42..cb61a845 100644 --- a/src/sketches/countsketch_l2_norm.rs +++ b/src/sketches/countsketch_l2_norm.rs @@ -3,7 +3,6 @@ //! [`CountSketchWithL2Norm`] is the per-layer sketch of `UnivMon` and //! `UnivMonTwoTier`; its L2 norm sets their heavy-hitter threshold. -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use crate::common::structures::matrix_storage::cols_mask_bits; @@ -17,15 +16,13 @@ pub(crate) mod wire; use crate::common::structures::fixed_structure::{DEFAULT_COLS, DEFAULT_ROWS}; /// Count Sketch augmented with per-row L2 norm tracking for heavy-hitter detection. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct CountSketchWithL2Norm { counts: Vector2D, row_l2_squared: Vector1D, rows: usize, cols: usize, seed_index: usize, - #[serde(skip)] _hasher: PhantomData, } diff --git a/src/sketches/countsketch_l2_norm/wire.rs b/src/sketches/countsketch_l2_norm/wire.rs index cf8aaade..b3b549c2 100644 --- a/src/sketches/countsketch_l2_norm/wire.rs +++ b/src/sketches/countsketch_l2_norm/wire.rs @@ -701,4 +701,24 @@ mod tests { right.to_asapv1().expect("serialize") ); } + + /// Known counts, L2 accumulators and seed index serialize to the + /// `asapv1_golden/` bytes. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + use std::marker::PhantomData; + + const COUNTS: [[i64; 4]; 2] = [[127, 128, 65535, -32768], [-32, -33, -2147483648, i64::MIN]]; + let sketch: CountSketchWithL2Norm = CountSketchWithL2Norm { + counts: Vector2D::from_fn(2, 4, |r, c| COUNTS[r][c]), + row_l2_squared: Vector1D::from_vec(vec![65536, i64::MAX]), + rows: 2, + cols: 4, + seed_index: 7, + _hasher: PhantomData, + }; + let golden = hex(include_str!("../../../asapv1_golden/count_l2hh_2x4_seed7.hex")); + assert_eq!(sketch.to_asapv1().unwrap(), golden); + } } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 9466e414..7095af9b 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -21,7 +21,6 @@ use crate::common::numerical::NumericalValue; use crate::common::structures::Vector1D; use crate::asapv1::admission_rate::{self, admit_all}; use crate::octo_delta::{DdSign, DdSketchBucketDelta}; -use serde::{Deserialize, Serialize}; /// The ASAPv1 DDSketch delta (kind_id `0x05 0x80`). mod delta; @@ -33,7 +32,7 @@ pub use delta::DdSketchDelta; // Number of buckets to grow by when expanding. const GROW_CHUNK: usize = 128; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] struct Buckets { counts: Vector1D, offset: i32, @@ -125,51 +124,33 @@ impl Buckets { /// ASAPv1 serialization lives in the `wire` submodule (kind_id `0x05 0x00`); it /// carries `alpha`, both bucket stores, the zero count and `sum` / `min` / /// `max`, and recovers `count` by summing them. -/// -/// The derived `Serialize` / `Deserialize` is a separate, Rust-internal form -/// used where a `DdSketch` is nested in another type. It carries the same state -/// as the ASAPv1 payload and rebuilds the index mapping and `count` on the way -/// in, refusing a state the ASAPv1 decoder would refuse. -#[derive(Debug, Serialize, Deserialize)] -#[serde(try_from = "DdSketchState")] +#[derive(Debug)] pub struct DdSketch { alpha: f64, - #[serde(skip)] gamma: f64, - #[serde(skip)] log_gamma: f64, - #[serde(skip)] inv_log_gamma: f64, positive_store: Buckets, - #[serde(skip)] count: u64, sum: f64, min: f64, max: f64, - #[serde(default)] negative_store: Buckets, - #[serde(default)] zero_count: u64, - #[serde(default = "admit_all")] admission_rate: f64, } -/// The fields a serialized [`DdSketch`] carries, in the order it emits them. -/// `gamma` / `log_gamma` / `inv_log_gamma` follow from `alpha` and `count` is -/// the sum of the buckets, so none of the four reaches the wire. -#[derive(Deserialize)] +/// The state a decoded [`DdSketch`] is rebuilt from. `gamma` / `log_gamma` / +/// `inv_log_gamma` follow from `alpha` and `count` is the sum of the buckets. struct DdSketchState { alpha: f64, positive_store: Buckets, sum: f64, min: f64, max: f64, - #[serde(default)] negative_store: Buckets, - #[serde(default)] zero_count: u64, - #[serde(default = "admit_all")] admission_rate: f64, } @@ -809,120 +790,4 @@ mod tests { assert_eq!(d.get_count(), count_before + 2); } - fn populated_sketch() -> DdSketch { - let mut sketch = DdSketch::new(0.01); - for v in [0.25f64, 1.0, 2.0, 3.0, 10.0, 50.0, 100.0, 1000.0] { - sketch.add(&v); - } - sketch - } - - /// The derived serde form carries every scalar the buckets do not - /// determine, so a round trip continues the run rather than resetting it. - #[test] - fn serde_round_trip_keeps_the_running_scalars() { - let sketch = populated_sketch(); - let bytes = rmp_serde::to_vec(&sketch).expect("encode"); - let restored: DdSketch = rmp_serde::from_slice(&bytes).expect("decode"); - - assert_eq!(restored.get_count(), sketch.get_count()); - assert_eq!(restored.sum(), sketch.sum()); - assert_eq!(restored.min(), sketch.min()); - assert_eq!(restored.max(), sketch.max()); - assert_eq!(restored.alpha(), sketch.alpha()); - assert_eq!( - restored.positive_store_counts(), - sketch.positive_store_counts() - ); - assert_eq!( - restored.positive_store_offset(), - sketch.positive_store_offset() - ); - for q in [0.0, 0.25, 0.5, 0.9, 1.0] { - assert_eq!( - restored.get_value_at_quantile(q), - sketch.get_value_at_quantile(q), - "quantile {q} moved across the round trip" - ); - } - } - - /// A decoded sketch keeps ingesting on top of the state it came back with. - #[test] - fn serde_round_trip_leaves_the_sketch_usable() { - let sketch = populated_sketch(); - let bytes = rmp_serde::to_vec(&sketch).expect("encode"); - let mut restored: DdSketch = rmp_serde::from_slice(&bytes).expect("decode"); - - restored.add(&5.0f64); - assert_eq!(restored.get_count(), sketch.get_count() + 1); - assert_eq!(restored.sum(), sketch.sum() + 5.0); - assert_eq!( - restored.positive_store_counts().iter().sum::(), - restored.get_count(), - "the recovered count drifted from the buckets" - ); - } - - /// The scalars are checked against the store the same way the ASAPv1 - /// decoder checks them: a populated store with an empty sketch's scalars - /// is refused rather than decoded into an inconsistent sketch. - #[test] - fn serde_refuses_scalars_that_disagree_with_the_store() { - #[derive(Serialize)] - struct CraftedState { - alpha: f64, - positive_store: Buckets, - sum: f64, - min: f64, - max: f64, - } - - let crafted = CraftedState { - alpha: 0.01, - positive_store: Buckets { - counts: Vector1D::from_vec(vec![1u64, 2]), - offset: -3, - }, - sum: 0.0, - min: f64::INFINITY, - max: f64::NEG_INFINITY, - }; - let bytes = rmp_serde::to_vec(&crafted).expect("encode crafted state"); - let err = rmp_serde::from_slice::(&bytes) - .expect_err("scalars disagreeing with the store must be refused"); - assert!( - err.to_string().contains("DDSketch scalars"), - "unexpected error: {err}" - ); - } - - /// An alpha outside `(0, 1)` gives a meaningless index mapping, so it is - /// refused on the way in rather than at the first query. - #[test] - fn serde_refuses_an_out_of_range_alpha() { - #[derive(Serialize)] - struct CraftedState { - alpha: f64, - positive_store: Buckets, - sum: f64, - min: f64, - max: f64, - } - - let crafted = CraftedState { - alpha: 1.5, - positive_store: Buckets { - counts: Vector1D::from_vec(Vec::new()), - offset: 0, - }, - sum: 0.0, - min: f64::INFINITY, - max: f64::NEG_INFINITY, - }; - let bytes = rmp_serde::to_vec(&crafted).expect("encode crafted state"); - let err = rmp_serde::from_slice::(&bytes) - .expect_err("an out-of-range alpha must be refused"); - assert!(err.to_string().contains("alpha"), "unexpected error: {err}"); - } } diff --git a/src/sketches/ddsketch/wire.rs b/src/sketches/ddsketch/wire.rs index 113549a7..c4d82cf4 100644 --- a/src/sketches/ddsketch/wire.rs +++ b/src/sketches/ddsketch/wire.rs @@ -665,4 +665,47 @@ mod tests { "bucket counts overflowing the total must not serialize" ); } + + /// Known stores and scalars serialize to the `asapv1_golden/` bytes. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + + let state = |positive: (&[u64], i32), negative: (&[u64], i32), zero_count, scalars| { + let (sum, min, max) = scalars; + let store = |(counts, offset): (&[u64], i32)| Buckets { + counts: Vector1D::from_vec(counts.to_vec()), + offset, + }; + DdSketch::try_from(DdSketchState { + alpha: 0.01, + positive_store: store(positive), + sum, + min, + max, + negative_store: store(negative), + zero_count, + admission_rate: 1.0, + }) + .expect("valid state") + }; + + let positive = state( + (&[1, 0, 127, 128, 300, 65536, 4294967296], -40), + (&[], 0), + 0, + (2181071000.0, 0.453125, 0.5078125), + ); + let golden = hex(include_str!("../../../asapv1_golden/ddsketch_positive_a001.hex")); + assert_eq!(positive.to_asapv1().unwrap(), golden); + + let signed = state( + (&[3, 0, 2], 310), + (&[5, 1], -208), + 7, + (2523.90625, -0.016, 515.0), + ); + let golden = hex(include_str!("../../../asapv1_golden/ddsketch_signed_a001.hex")); + assert_eq!(signed.to_asapv1().unwrap(), golden); + } } diff --git a/src/sketches/elastic.rs b/src/sketches/elastic.rs index d117b7e8..e1bea1ec 100644 --- a/src/sketches/elastic.rs +++ b/src/sketches/elastic.rs @@ -9,7 +9,6 @@ use crate::{DEFAULT_CANONICAL_SEED_INDEX, DataInput, DefaultXxHasher, SketchHash use super::{CountMinSketch, RegularPath}; use crate::Vector2D; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; mod wire; @@ -30,7 +29,7 @@ pub const DEFAULT_LIGHT_COLS: usize = 4096; /// One slot of the heavy part: the resident flow, its vote pair, and the flag /// marking that part of its size lives in the light layer. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Clone, Debug)] pub struct HeavyBucket { pub flow_id: String, pub vote_pos: i32, @@ -40,17 +39,14 @@ pub struct HeavyBucket { /// Heavy/light frequency estimator: a heavy hash table over flow ids backed by /// a Count-Min light layer that absorbs evicted and unelected flows. -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct ElasticSketch { pub heavy: Vec, pub light: CountMinSketch, RegularPath, H>, pub heavy_buckets: i32, /// Set by [`ElasticSketch::expand_heavy`] while copies of pre-expansion /// residents may still sit in the half they no longer hash to. - #[serde(default)] stale_copies: bool, - #[serde(skip)] _hasher: PhantomData, } diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 4a037028..2353c00e 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -33,14 +33,13 @@ // - Refactored into a generic `HyperLogLog` structure. // ---------------------------------------------------------------- -use crate::asapv1::admission_rate::{admit_all, deserialize_rate}; +use crate::asapv1::admission_rate::admit_all; use crate::structures::fixed_structure::{ HllRegisterStorage, HllRegistersP12, HllRegistersP14, HllRegistersP16, }; use crate::{ DEFAULT_CANONICAL_SEED_INDEX, DataInput, DefaultXxHasher, SketchHasher, hash64_seeded, }; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; mod delta; @@ -50,19 +49,15 @@ pub use wire::HllWireVariant; pub(crate) use wire::{HllMetadata, HllPayloadHip, HllPayloadPlain, standard_hll_metadata}; /// Generic HyperLogLog sketch parameterized by estimation variant, register storage, and hasher. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct HyperLogLogImpl< Variant, Registers: HllRegisterStorage, H: SketchHasher = DefaultXxHasher, > { registers: Registers, - #[serde(skip)] _marker: PhantomData, - #[serde(skip)] _hasher: PhantomData, - #[serde(default = "admit_all", deserialize_with = "deserialize_rate")] admission_rate: f64, } @@ -94,21 +89,19 @@ pub(crate) fn classic_estimate(registers: &[u8]) -> f64 { } /// Marker type selecting the classic HyperLogLog estimation algorithm. -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default)] pub struct Classic; /// Marker type selecting the Ertl MLE estimation algorithm (arXiv:1702.01284). -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default)] pub struct Ertl; /// HyperLogLog variant using the Historic Inverse Probability (HIP) estimator for improved accuracy. -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct HyperLogLogHipImpl { registers: Registers, kxq0: f64, kxq1: f64, est: f64, - #[serde(default = "admit_all", deserialize_with = "deserialize_rate")] admission_rate: f64, } diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index e2e1c205..b1b1f97b 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -11,7 +11,6 @@ //! - use rand::{Rng, rng}; -use serde::{Deserialize, Serialize}; use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; @@ -34,11 +33,10 @@ const DEFAULT_K: i32 = 200; /// Coin generates deterministic pseudo-random coin flips while amortizing /// calls to the RNG by consuming one bit at a time from a 64-bit buffer. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub(crate) struct Coin { state: u64, bit_cache: u64, - #[serde(default)] remaining_bits: u8, } @@ -109,7 +107,7 @@ impl Coin { } /// A single (value, normalized rank) pair in a CDF table. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct CdfEntry { value: f64, quantile: f64, @@ -848,15 +846,6 @@ impl Kll { out } - // -- Serialization ------------------------------------------------------- - // - // The ASAPv1 wire methods (`to_asapv1` / `from_asapv1`) - // live in the `wire` submodule, which authors the envelope + metadata + - // payload. The `serde::{Serialize, Deserialize}` impls below stay: they are - // the *nested* codec used when a `Kll` is embedded in a larger serde value - // (e.g. `HydraCounter::Kll`), which is a different concern from the - // standalone ASAPv1 envelope. - fn ensure_levels_sorted(&mut self) { if self.num_levels <= 1 { return; @@ -872,36 +861,6 @@ impl Kll { } } -/// Wire format for serialization (only the used portion of the buffer). -#[derive(Serialize, Deserialize)] -struct KLLWire { - items: Vec, - levels: Vec, - k: usize, - m: usize, - num_levels: usize, - co: Coin, -} - -impl Serialize for Kll { - fn serialize(&self, serializer: S) -> Result { - let used_start = self.levels[0]; - let used_end = self.levels[self.num_levels]; - let wire = KLLWire { - items: self.items[used_start..used_end].to_vec(), - levels: self.levels[..=self.num_levels] - .iter() - .map(|&l| l - used_start) - .collect(), - k: self.k, - m: self.m, - num_levels: self.num_levels, - co: self.coin.clone(), - }; - wire.serialize(serializer) - } -} - impl Kll { /// Inserts a value from a [`DataInput`] into a `Kll` sketch. /// @@ -926,83 +885,6 @@ impl Kll { } } -impl<'de, T: NumericalValue + Deserialize<'de>> Deserialize<'de> for Kll { - fn deserialize>(deserializer: D) -> Result { - use serde::de::Error as _; - let wire = KLLWire::::deserialize(deserializer)?; - - // Fail closed on crafted bytes. This nested serde path is reachable with - // untrusted input via `HydraCounter::Kll` (`Hydra::from_asapv1`) - // and direct `rmp_serde::from_slice::>`, so it must guard the same - // way the ASAPv1 decoder does: bound `k`/`m` to the range the constructor - // clamps to (else `compute_max_capacity` / `Vec::with_capacity` blow up), - // and validate the level layout (else the buffer math below underflows or - // indexes out of bounds, and `count()` overflows on a later query). - if wire.m < 2 || wire.m > wire.k || wire.k > MAX_K { - return Err(D::Error::custom(format!( - "KLL: k={}, m={} outside valid range (2 <= m <= k <= {MAX_K})", - wire.k, wire.m - ))); - } - if wire.num_levels == 0 - || wire.num_levels > MAX_LEVELS - || wire.levels.len() != wire.num_levels + 1 - || wire.levels.first() != Some(&0) - || wire.levels.windows(2).any(|w| w[0] > w[1]) - || wire.levels.last() != Some(&wire.items.len()) - { - return Err(D::Error::custom("KLL: inconsistent level layout")); - } - let sizes: Vec = wire.levels.windows(2).map(|w| w[1] - w[0]).collect(); - if checked_weighted_count(&sizes).is_none() { - return Err(D::Error::custom( - "KLL: level layout overflows weighted count", - )); - } - - let max_cap = compute_max_capacity(wire.k, wire.m); - let used_len = wire.items.len(); - if used_len > max_cap { - return Err(D::Error::custom(format!( - "KLL: {used_len} items exceed max_capacity {max_cap}" - ))); - } - let offset = max_cap - used_len; - - let mut items = vec![T::default(); max_cap].into_boxed_slice(); - items[offset..offset + used_len].copy_from_slice(&wire.items); - - let mut levels = vec![0usize; MAX_LEVELS + 1].into_boxed_slice(); - for (i, &l) in wire.levels.iter().enumerate() { - levels[i] = l + offset; - } - - let mut sketch = Kll { - items, - levels, - k: wire.k, - m: wire.m, - num_levels: wire.num_levels, - max_capacity: max_cap, - coin: wire.co, - // Wire format does not carry the explicit-seed flag — a - // round-tripped sketch keeps its current coin state but - // would re-randomize from time on a subsequent clear(). - // Callers that need clear()-determinism after a deserialize - // should rebuild via with_seed. - seed: None, - capacity_cache: [0; CAPACITY_CACHE_LEN], - top_height: 0, - level0_capacity: 0, - merge_buf: Vec::with_capacity(wire.k), - cdf_cache: None, - }; - sketch.rebuild_capacity_cache(); - sketch.ensure_levels_sorted(); - Ok(sketch) - } -} - /// The CDF for quantile queries. #[derive(Clone, Debug)] pub struct Cdf { @@ -1153,26 +1035,6 @@ mod tests { ); } - // Crafted nested-serde bytes (the `HydraCounter::Kll` path, reachable with - // untrusted input) with an out-of-range `k` must fail closed, not overflow - // `compute_max_capacity` / drive a huge allocation. - #[test] - fn kllwire_serde_rejects_crafted_dimensions() { - let wire = KLLWire:: { - items: vec![], - levels: vec![0, 0], - k: usize::MAX, - m: 8, - num_levels: 1, - co: Coin::from_seed(1), - }; - let bytes = rmp_serde::to_vec(&wire).expect("encode crafted wire"); - assert!( - rmp_serde::from_slice::>(&bytes).is_err(), - "an out-of-range k must be rejected by the nested serde decoder" - ); - } - // Direct reconstruction from level state must be BIT-EXACT: // identical quantiles to the source sketch (unlike a lossy // replay-through-`update()` reconstruction). diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index d4c93a15..4a5fa1da 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -12,7 +12,6 @@ //! //! - -use serde::{Deserialize, Serialize}; use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; @@ -27,7 +26,7 @@ const CAPACITY_DECAY: f64 = 2.0 / 3.0; const DEFAULT_K: i32 = 200; /// One entry in the cumulative distribution, storing a value and its mass. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct DynamicCdfEntry { value: f64, normalized_rank: f64, @@ -38,7 +37,7 @@ pub struct DynamicCdfEntry { /// Unlike [`Kll`](super::kll::Kll), which pre-allocates a fixed buffer and /// grows downward, `KllDynamic` appends items to a growable `Vector1D` and /// shifts elements during compaction. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct KllDynamic { items: Vector1D, // compactors, packed /// Stores the START index of each level in `items`. @@ -52,16 +51,12 @@ pub struct KllDynamic { /// a sketch replays identically across runs and `clear()` re-seeds from /// the same value. Not part of the wire format: it describes how the /// sketch was built, not what it currently holds. - #[serde(skip)] seed: Option, /// Cached level capacities by height (from top to bottom) - #[serde(skip)] capacity_cache: [u32; CAPACITY_CACHE_LEN], /// Tracks current top height so we can index into the cache quickly. - #[serde(skip)] top_height: usize, /// Cached capacity for level 0 to speed up hot-path updates - #[serde(skip)] level0_capacity: usize, } diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index bb1228f8..6b9b1b42 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -9,7 +9,6 @@ use crate::{ SketchHasher, }; -use serde::{Deserialize, Serialize}; use std::marker::PhantomData; mod wire; @@ -17,12 +16,10 @@ mod wire; // expect error bound to be less than 2% const KMV_DEFAULT_K: usize = 4096_usize; -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct Kmv { pub k: usize, pub retained_hashes: CommonHeap, - #[serde(skip)] _hasher: PhantomData, } diff --git a/src/sketches/kmv/wire.rs b/src/sketches/kmv/wire.rs index b318d769..8bb0f575 100644 --- a/src/sketches/kmv/wire.rs +++ b/src/sketches/kmv/wire.rs @@ -24,7 +24,7 @@ //! Retained values are the 64-bit digests `hash64_seeded` returns and travel at //! that width, neither widened nor narrowed. -use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice}; +use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; @@ -78,40 +78,13 @@ struct KmvPayload { hashes: Vec, } -/// The serde shape of [`CommonHeap`]'s own fields. -#[derive(Serialize)] -struct HeapSeed { - data: Vec, - size: usize, - order: RetainSmallest, -} - -/// Rebuilds the retained heap through [`CommonHeap`]'s serde form, so the -/// backing vector is sized from the payload and `k` sizes no allocation. A -/// descending run already satisfies the max-heap invariant, so the hashes seat -/// as they are. -fn rebuild_heap( - k: usize, - ascending: Vec, -) -> Result, RmpDecodeError> { - let len = ascending.len(); +/// Seats the retained hashes as the heap, with the backing vector sized from +/// the payload so `k` sizes no allocation. A descending run already satisfies +/// the max-heap invariant, so the hashes seat as they are. +fn rebuild_heap(k: usize, ascending: Vec) -> CommonHeap { let mut data = ascending; data.reverse(); - let seed = rmp_serde::to_vec_named(&HeapSeed { - data, - size: k, - order: RetainSmallest, - }) - .map_err(|err| RmpDecodeError::Uncategorized(err.to_string()))?; - let heap: CommonHeap = from_slice(&seed)?; - if heap.len() != len || heap.capacity() != k { - return Err(RmpDecodeError::Uncategorized(format!( - "KMV heap rebuild: {} of {len} hashes under a bound of {} against k {k}", - heap.len(), - heap.capacity() - ))); - } - Ok(heap) + CommonHeap::from_heap_vec(data, k, RetainSmallest) } // Wire serialization for KMV. `wire` is a descendant of the sketch module, so @@ -202,7 +175,7 @@ impl Kmv { } Ok(Kmv { k, - retained_hashes: rebuild_heap(k, decoded.hashes)?, + retained_hashes: rebuild_heap(k, decoded.hashes), _hasher: PhantomData, }) } @@ -220,6 +193,7 @@ impl Kmv { mod tests { use super::*; use crate::{DEFAULT_CANONICAL_SEED_INDEX, DataInput, DefaultXxHasher, OwnedDataInput}; + use rmp_serde::from_slice; fn populated(k: usize, keys: u64) -> Kmv { let mut sketch: Kmv = Kmv::new(k); diff --git a/src/sketches/microscope.rs b/src/sketches/microscope.rs index a98d7311..8cec824c 100644 --- a/src/sketches/microscope.rs +++ b/src/sketches/microscope.rs @@ -61,10 +61,6 @@ pub(crate) mod cell; pub use cell::{DeltaStrategy, MAX_SUB_WINDOWS, MAX_ZOOM_BASE, MicroLayout, MicroParams, Rounding}; use crate::{DataInput, DefaultXxHasher, MatrixFastHash, SketchHasher, Vector3D}; -use rmp_serde::{ - decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice, to_vec_named, -}; -use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use std::marker::PhantomData; @@ -75,7 +71,7 @@ use std::marker::PhantomData; /// places — when `n` advances, and how the partially expired oldest /// sub-window is weighted — so they are one type here rather than two /// sketches. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SubWindowClock { /// A sub-window is a fixed number of items. CountBased { @@ -268,84 +264,16 @@ impl SubWindowClock { /// /// See the [module docs](crate::sketches::microscope) for the layout rationale and the current /// status of this implementation. -#[derive(Clone, Debug, Serialize)] -#[serde(bound = "")] +#[derive(Clone, Debug)] pub struct MicroCm { cells: Vector3D, params: MicroParams, clock: SubWindowClock, rounding: Rounding, - #[serde(skip)] layout: MicroLayout, - #[serde(skip)] _hasher: PhantomData, } -// Only the authoritative fields are read from the wire; the layout is -// re-derived from `params` and then checked against the stored geometry, so -// a payload cannot describe a record shape the cells do not have. -#[derive(Deserialize)] -struct MicroCmSeed { - cells: Vector3D, - params: MicroParams, - clock: SubWindowClock, - rounding: Rounding, -} - -impl<'de, H: SketchHasher> Deserialize<'de> for MicroCm { - fn deserialize>(deserializer: D) -> Result { - let MicroCmSeed { - cells, - params, - clock, - rounding, - } = MicroCmSeed::deserialize(deserializer)?; - // Validate before deriving the layout: `MicroLayout::new` computes - // `t + 2`, which a crafted `t` could overflow. - MicroParams::checked(params.t, params.c).map_err(serde::de::Error::custom)?; - let layout = MicroLayout::new(params); - if cells.bucket_len() != layout.depth() { - return Err(serde::de::Error::custom(format!( - "cell bucket_len {} does not match the {} bytes the layout for t={}, c={} needs", - cells.bucket_len(), - layout.depth(), - params.t, - params.c - ))); - } - let cols = cells.cols(); - if cols < 2 || !cols.is_power_of_two() { - return Err(serde::de::Error::custom(format!( - "cols ({cols}) must be a power of two and at least 2" - ))); - } - let rows = cells.rows(); - let col_bits = cells.mask_bits() as usize; - let required_bits = rows.saturating_mul(col_bits); - if required_bits > 128 { - return Err(serde::de::Error::custom(format!( - "rows ({rows}) x column bits ({col_bits}) = {required_bits} exceeds the \ - 128-bit packed column hash; reduce rows or cols" - ))); - } - clock.checked().map_err(serde::de::Error::custom)?; - // Each record carries a zoom exponent and a shutter that the layout - // cannot constrain; a crafted pair panics or wraps on first use. - for (index, record) in cells.as_slice().chunks_exact(layout.depth()).enumerate() { - cell::validate(record, &layout) - .map_err(|detail| serde::de::Error::custom(format!("cell {index}: {detail}")))?; - } - Ok(Self { - cells, - params, - clock, - rounding, - layout, - _hasher: PhantomData, - }) - } -} - impl Default for MicroCm { /// A 4 x 1024 grid at `T = 12`, `c = 2`, over count-based sub-windows of /// 4096 items — a sliding window of roughly 49k items. @@ -386,7 +314,6 @@ impl MicroCm { ); // Both types are public with public fields, so a struct literal // arrives here without having passed their constructors' checks. - // `deserialize` validates the same two; so must this path. let params = MicroParams::checked(params.t, params.c).unwrap_or_else(|detail| panic!("{detail}")); clock.checked().unwrap_or_else(|detail| panic!("{detail}")); @@ -660,15 +587,6 @@ impl MicroCm { Ok(()) } - /// Serializes the sketch into MessagePack bytes. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) - } - - /// Deserializes a sketch from MessagePack bytes. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) - } } #[cfg(test)] @@ -697,7 +615,7 @@ mod tests { /// `MicroParams` and `SubWindowClock` are public with public fields, so a /// struct literal reaches `with_dimensions` without their constructors' - /// checks. `deserialize` rejects both; so must this path. + /// checks. #[test] #[should_panic(expected = "zoom base c must be in 2..=256")] fn with_dimensions_rejects_a_params_literal_the_layout_cannot_represent() { @@ -1004,173 +922,4 @@ mod tests { assert_eq!(a.as_storage().as_slice(), before.as_slice()); } - #[test] - fn serde_round_trip_preserves_the_answers() { - let mut sk = sketch(3, 32, 4, 100); - for i in 0..1_000u64 { - sk.update(&key(i % 17)); - } - let bytes = sk.serialize_to_bytes().expect("serialize"); - let back = MicroCm::::deserialize_from_bytes(&bytes).expect("decode"); - assert_eq!(back.sub_window(), sk.sub_window()); - assert_eq!(back.cell_bytes(), sk.cell_bytes()); - assert_eq!(back.as_storage().as_slice(), sk.as_storage().as_slice()); - for k in 0..17u64 { - assert_eq!( - back.estimate_with(&key(k), DeltaStrategy::Over), - sk.estimate_with(&key(k), DeltaStrategy::Over) - ); - } - } - - /// Every guard in `deserialize` gets a payload that trips it. - /// - /// Two of these — the zoom exponent and the shutter — are per-record - /// values the layout cannot constrain, so a crafted pair decodes into a - /// sketch that panics or wraps on first use rather than one that is - /// merely wrong. - #[test] - fn deserialize_rejects_every_state_the_sketch_cannot_represent() { - #[derive(Serialize)] - struct Forged { - cells: Vector3D, - params: MicroParams, - clock: SubWindowClock, - rounding: u64, - } - - fn cells_for(rows: usize, cols: usize, depth: usize) -> Vector3D { - let mut cells: Vector3D = Vector3D::unfilled(rows, cols, depth); - cells.fill(0); - cells - } - fn attempt(cells: Vector3D, params: MicroParams, clock: SubWindowClock) -> String { - let forged = Forged { - cells, - params, - clock, - rounding: 1, - }; - let bytes = rmp_serde::to_vec_named(&forged).expect("encode"); - MicroCm::::deserialize_from_bytes(&bytes) - .err() - .map(|e| e.to_string()) - .unwrap_or_default() - } - - let good = MicroParams::new(4, 2); - let depth = MicroLayout::new(good).depth(); - let clock = SubWindowClock::count_based(100); - - // A sane payload decodes, so each case below fails for its own - // reason rather than because the shape was wrong to begin with. - assert_eq!( - attempt(cells_for(2, 16, depth), good, clock), - "", - "the control payload must decode" - ); - - // A record whose zoom exponent is past the cap: `c^Z` overflows. - let mut zoomed = cells_for(2, 16, depth); - zoomed.as_mut_slice()[MicroLayout::new(good).pixels()] = 200; - // A record whose shutter is not below one unit: the merge carry it - // bounds can then overflow the pixel sum. - let mut loose = cells_for(2, 16, depth); - { - let layout = MicroLayout::new(good); - let slice = loose.as_mut_slice(); - slice[layout.pixels()] = 3; // Z = 3, so one unit is 8 - slice[layout.pixels() + 1] = 99; // shutter low byte - } - - for (label, err, want) in [ - ( - "t past the cap", - attempt( - cells_for(2, 16, depth), - MicroParams { - t: usize::MAX, - c: 2, - }, - clock, - ), - "must be in 1..=", - ), - ( - "zoom base below 2", - attempt(cells_for(2, 16, depth), MicroParams { t: 4, c: 1 }, clock), - "zoom base c must be in", - ), - ( - "zoom base past a byte pixel", - attempt( - cells_for(2, 16, depth), - MicroParams { t: 4, c: 1_000 }, - clock, - ), - "zoom base c must be in", - ), - ( - "bucket_len the layout disagrees with", - attempt(cells_for(2, 16, depth + 4), good, clock), - "does not match", - ), - ( - "cols not a power of two", - attempt(cells_for(2, 17, depth), good, clock), - "power of two", - ), - ( - "single column", - attempt(cells_for(2, 1, depth), good, clock), - "at least 2", - ), - ( - // 22 rows x 6 column bits = 132 > 128. - "rows past the packed hash", - attempt(cells_for(22, 64, depth), good, clock), - "exceeds the", - ), - ( - "zero-length sub-window", - attempt( - cells_for(2, 16, depth), - good, - SubWindowClock::CountBased { - items_per_sub_window: 0, - seen: 0, - }, - ), - "must be non-zero", - ), - ( - "zero-length time sub-window", - attempt( - cells_for(2, 16, depth), - good, - SubWindowClock::TimeBased { - sub_window_len: 0, - epoch: 0, - now: 0, - }, - ), - "must be non-zero", - ), - ( - "zoom exponent past the cap", - attempt(zoomed, good, clock), - "exceeds the maximum", - ), - ( - "shutter at or above one unit", - attempt(loose, good, clock), - "not below one unit", - ), - ] { - assert!( - err.contains(want), - "{label}: expected an error containing {want:?}, got {err:?}" - ); - } - } } diff --git a/src/sketches/microscope/cell.rs b/src/sketches/microscope/cell.rs index e8830856..ba10e166 100644 --- a/src/sketches/microscope/cell.rs +++ b/src/sketches/microscope/cell.rs @@ -36,7 +36,6 @@ //! per byte halves the record and is a later change; the layout type already //! hides every offset from callers, so it is contained here. -use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::cell::Cell; /// The largest value an 8-bit pixel can hold. @@ -76,7 +75,7 @@ pub enum DeltaStrategy { } /// Configuration of a MicroscopeSketch cell. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct MicroParams { /// Number of sub-windows the sliding window is split into. pub t: usize, @@ -107,7 +106,7 @@ impl MicroParams { Self::checked(t, c).unwrap_or_else(|detail| panic!("{detail}")) } - /// [`Self::new`] as a `Result`, for validating deserialized values. + /// [`Self::new`] as a `Result`. pub fn checked(t: usize, c: u32) -> Result { if t == 0 || t > MAX_SUB_WINDOWS { return Err(format!( @@ -218,32 +217,6 @@ pub fn zoom(cell: &[u8], layout: &MicroLayout) -> u8 { cell[layout.zoom_at] } -/// Checks the two invariants a cell's own bytes must satisfy, for use on -/// deserialized records. -/// -/// Neither is expressible in the layout, so a crafted payload can carry a -/// record that decodes cleanly and then misbehaves on first use: -/// -/// - `Z` above the cap makes `c^Z` overflow, which panics in a debug build -/// and silently wraps to a nonsense unit in a release one. -/// - a shutter at or above `c^Z` breaks the assumption that a shutter holds -/// less than one unit, which is what bounds the carry a merge computes. -pub fn validate(cell: &[u8], layout: &MicroLayout) -> Result<(), String> { - let z = cell[layout.zoom_at]; - let cap = layout.max_zoom(); - if z > cap { - return Err(format!("zoom exponent {z} exceeds the maximum {cap}")); - } - let unit = (layout.params.c as u64).pow(z as u32); - let s = shutter(cell, layout) as u64; - if s >= unit { - return Err(format!( - "shutter {s} is not below one unit (c^Z = {unit}) at zoom {z}" - )); - } - Ok(()) -} - /// A cell's shutter. /// /// Read and written through `to_le_bytes`/`from_le_bytes` rather than a cast, @@ -272,25 +245,12 @@ pub fn scaled_pixel(cell: &[u8], layout: &MicroLayout, n: u64) -> f64 { /// /// It advances through a `Cell`, so a `&Rounding` can be captured by a /// non-`mut` closure — which is what lets a cell insert run inside -/// `Vector3D::fast_insert`, whose closure is `Fn`. The state is a plain -/// `u64`, so a sketch holding one can serialize it or reseed on load. +/// `Vector3D::fast_insert`, whose closure is `Fn`. #[derive(Clone, Debug)] pub struct Rounding { state: Cell, } -impl Serialize for Rounding { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_u64(self.state.get()) - } -} - -impl<'de> Deserialize<'de> for Rounding { - fn deserialize>(deserializer: D) -> Result { - Ok(Self::new(u64::deserialize(deserializer)?)) - } -} - impl Rounding { /// Seeds the generator. A zero seed is replaced, since xorshift is stuck /// at zero. @@ -304,7 +264,7 @@ impl Rounding { } } - /// The current state, for serialization. + /// The current state. #[inline(always)] pub fn state(&self) -> u64 { self.state.get() diff --git a/src/sketches/mode.rs b/src/sketches/mode.rs index 22078b3b..9c3d2538 100644 --- a/src/sketches/mode.rs +++ b/src/sketches/mode.rs @@ -1,8 +1,7 @@ -use serde::{Deserialize, Serialize}; /// Hashing mode of a matrix-backed sketch in which row `r` hashes the key on /// its own, at seed-table index `r`. ASAPv1 `mode` `"regular"`. -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default)] pub struct RegularPath; /// Hashing mode of a matrix-backed sketch in which the key is hashed once, at @@ -10,5 +9,5 @@ pub struct RegularPath; /// from that one matrix hash. The hash is one packed 64- or 128-bit value when /// the rows' bits fit, else one hash per row (`MatrixHashLayout`). ASAPv1 /// `mode` `"fast"`. -#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default)] pub struct FastPath; diff --git a/src/sketches/space_saving.rs b/src/sketches/space_saving.rs index c24a816c..fa4a5b7a 100644 --- a/src/sketches/space_saving.rs +++ b/src/sketches/space_saving.rs @@ -28,7 +28,6 @@ use crate::common::DigestBuildHasher; use crate::{DataInput, DefaultXxHasher, OwnedDataInput, SketchHasher}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; use smallvec::SmallVec; use std::collections::HashMap; use std::marker::PhantomData; @@ -555,17 +554,8 @@ fn key_order(key: &OwnedDataInput) -> (u8, u128, &[u8]) { } } -/// Serialized form: the summary as the triples it answers with, since the -/// bucket and counter lists and the key index all follow from them. -#[derive(Serialize)] -struct SpaceSavingRef<'a> { - capacity: usize, - total: u64, - discarded_max: u64, - entries: Vec<(&'a OwnedDataInput, u64, u64)>, -} - -#[derive(Deserialize)] +/// A summary as the triples it answers with; the bucket and counter lists and +/// the key index all follow from them. struct SpaceSavingState { capacity: usize, total: u64, @@ -573,26 +563,20 @@ struct SpaceSavingState { entries: Vec<(OwnedDataInput, u64, u64)>, } -impl Serialize for SpaceSaving { - fn serialize(&self, serializer: S) -> Result { - SpaceSavingRef { +#[cfg(test)] +impl SpaceSaving { + /// Rebuilds a copy from the triples it answers with, as a decode does. + fn rebuilt(&self) -> Result { + Self::rebuild(SpaceSavingState { capacity: self.capacity, total: self.total, discarded_max: self.discarded_max, entries: self .monitored .iter() - .map(|c| (&c.key, self.buckets[c.bucket].count, c.error)) + .map(|c| (c.key.clone(), self.buckets[c.bucket].count, c.error)) .collect(), - } - .serialize(serializer) - } -} - -impl<'de, H: SketchHasher> Deserialize<'de> for SpaceSaving { - fn deserialize>(deserializer: D) -> Result { - let state = SpaceSavingState::deserialize(deserializer)?; - Self::rebuild(state).map_err(serde::de::Error::custom) + }) } } @@ -1179,10 +1163,10 @@ mod tests { } /// The ceiling a merge leaves behind lives only in `discarded_max`; an under-full - /// summary reads it from no counter, so a serde round trip that dropped it + /// summary reads it from no counter, so a rebuild that dropped it /// would answer below the truth. #[test] - fn a_serde_round_trip_carries_a_ceiling_no_counter_holds() { + fn a_rebuild_carries_a_ceiling_no_counter_holds() { let mut left: SpaceSaving = SpaceSaving::with_capacity(33); let mut right: SpaceSaving = SpaceSaving::with_capacity(1); for _ in 0..10 { @@ -1195,8 +1179,7 @@ mod tests { assert!(left.len() < left.capacity(), "the merge left room to spare"); assert_eq!(left.min_count(), 30); - let bytes = rmp_serde::to_vec(&left).expect("serialize"); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("deserialize"); + let decoded: SpaceSaving = left.rebuilt().expect("rebuild"); decoded.validate().expect("decoded summary"); assert_eq!( @@ -1496,8 +1479,7 @@ mod tests { #[test] fn a_decoded_summary_rebuilds_both_link_directions() { let (summary, truth) = fuzzed(48, 200, 3_000, 77); - let bytes = rmp_serde::to_vec(&summary).expect("serialize"); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("deserialize"); + let decoded: SpaceSaving = summary.rebuilt().expect("rebuild"); decoded.validate().expect("decoded summary"); assert_eq!(decoded.len(), summary.len()); @@ -1709,17 +1691,16 @@ mod tests { assert_eq!(left.estimate(&DataInput::Bytes(&[0x02])), 7); } - /// A serde round trip carries the raw bytes, and the decoded summary + /// A rebuild carries the raw bytes, and the rebuilt summary /// answers the same `DataInput::Bytes` the original did — which it can only /// do if the rebuild hashed the stored key to the digest a query reaches. #[test] - fn a_serde_round_trip_keeps_a_byte_key() { + fn a_rebuild_keeps_a_byte_key() { let mut summary: SpaceSaving = SpaceSaving::with_capacity(4); summary.update_weighted(&DataInput::Bytes(RAW), 9); summary.update_weighted(&DataInput::Bytes(&[0x00; 5]), 4); - let bytes = rmp_serde::to_vec(&summary).expect("serialize"); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("deserialize"); + let decoded: SpaceSaving = summary.rebuilt().expect("rebuild"); decoded.validate().expect("decoded summary"); assert_eq!(bytes_of(&decoded), bytes_of(&summary)); @@ -1879,8 +1860,7 @@ mod collisions { summary.update_weighted(&DataInput::I64(key), weight); } - let bytes = rmp_serde::to_vec(&summary).expect("serialize"); - let decoded: Colliding = rmp_serde::from_slice(&bytes).expect("deserialize"); + let decoded: Colliding = summary.rebuilt().expect("rebuild"); decoded.validate().expect("decoded summary"); assert_eq!(decoded.len(), 3); diff --git a/src/sketches/space_saving/wire.rs b/src/sketches/space_saving/wire.rs index 2bc2e449..ab3915b5 100644 --- a/src/sketches/space_saving/wire.rs +++ b/src/sketches/space_saving/wire.rs @@ -1113,4 +1113,40 @@ mod tests { ); assert_eq!(decoded.top_k(8).len(), 2); } + + /// Known triples, seated out of emitted order, serialize to the + /// `asapv1_golden/` bytes. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + + let summary = |capacity, total, discarded_max, emitted: Vec<(OwnedDataInput, u64, u64)>| { + let mut entries = emitted; + entries.reverse(); + SpaceSaving::::rebuild(SpaceSavingState { + capacity, + total, + discarded_max, + entries, + }) + .expect("valid state") + }; + + let i64_keys = [ + (-1, 70000, 0), + (300, 9, 2), + (0, 5, 0), + (1, 5, 1), + (i64::MIN, 5, 4), + (-129, 5, 0), + ] + .map(|(key, count, error)| (OwnedDataInput::I64(key), count, error)); + let golden = hex(include_str!("../../../asapv1_golden/space_saving_i64_c8.hex")); + assert_eq!(summary(8, 70030, 4, i64_keys.to_vec()).to_asapv1().unwrap(), golden); + + let byte_keys = [(&b"\xff\x00"[..], 9, 0), (b"", 4, 1), (b"ab", 4, 0)] + .map(|(key, count, error)| (OwnedDataInput::Bytes(key.to_vec()), count, error)); + let golden = hex(include_str!("../../../asapv1_golden/space_saving_bytes_c3.hex")); + assert_eq!(summary(3, 20, 3, byte_keys.to_vec()).to_asapv1().unwrap(), golden); + } } diff --git a/src/sketches/uniform.rs b/src/sketches/uniform.rs index b3350dd9..2f898021 100644 --- a/src/sketches/uniform.rs +++ b/src/sketches/uniform.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use crate::DataInput; @@ -9,7 +8,7 @@ const DEFAULT_SEED: u64 = GOLDEN_GAMMA; const MIX_MULTIPLIER_1: u64 = 0xBF58_476D_1CE4_E5B9; const MIX_MULTIPLIER_2: u64 = 0x94D0_49BB_1331_11EB; -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] struct SampleEntry { priority: u64, value: f64, @@ -21,7 +20,7 @@ impl SampleEntry { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct UniformSampling { sample_rate: f64, total_seen: u64, diff --git a/src/sketches/uniform/wire.rs b/src/sketches/uniform/wire.rs index 43bc7b9a..cbde75a9 100644 --- a/src/sketches/uniform/wire.rs +++ b/src/sketches/uniform/wire.rs @@ -481,4 +481,27 @@ mod tests { "an over-full sampler must not serialize" ); } + + /// Known entries and running scalars serialize to the `asapv1_golden/` + /// bytes. + #[test] + fn known_state_matches_golden() { + use crate::asapv1::envelope::hex; + + let sampler = UniformSampling { + sample_rate: 0.25, + total_seen: 20, + rng_state: 0x0123_4567_89ab_cdef, + entries: [ + (3u64, -0.0f64), + (3, 0.0), + (1 << 63, 1e300), + (u64::MAX, f64::NAN), + ] + .map(|(priority, value)| SampleEntry::new(priority, value)) + .to_vec(), + }; + let golden = hex(include_str!("../../../asapv1_golden/uniform_sampling_r0p25.hex")); + assert_eq!(sampler.to_asapv1().unwrap(), golden); + } } diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 67c38d1e..d6aeb9bd 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -2,7 +2,9 @@ //! //! Each fixture is built from a fixed, KNOWN raw sketch state (register bytes / //! matrix values set directly, never hashed) and asserted to serialize to the -//! exact bytes in `asapv1_golden/*.hex`. That directory is a git submodule of +//! exact bytes in `asapv1_golden/*.hex`. A state the public API cannot build is +//! checked here by decoding to it, and its encode in the sketch's own +//! `wire.rs`. That directory is a git submodule of //! , the fixtures every ASAPv1 implementation //! must reproduce. //! @@ -1028,39 +1030,37 @@ fn kll_dynamic_i64_k200_matches_golden() { } // --------------------------------------------------------------------------- -// DDSketch and its delta: set the stores and scalars directly -> serialize == -// golden, golden round-trips, and the delta merged into the signed state gives -// the expected stores and scalars. +// DDSketch and its delta: golden decodes to the known stores and scalars and +// round-trips, and the delta merged into the signed state gives the expected +// stores and scalars. Encoding the known state is checked in +// `src/sketches/ddsketch/wire.rs`, which can build it. // --------------------------------------------------------------------------- const DD_ALPHA: f64 = 0.01; -/// An α=0.01 DDSketch with its stores, zero count and scalars set directly, -/// built through `DdSketch`'s serde form. -fn dd_state( +/// Decodes `golden`, checks the decoded stores, zero count and scalars, and +/// re-serializes it. +fn assert_dd_golden( + golden: &str, positive: (&[u64], i32), negative: (&[u64], i32), zero_count: u64, - scalars: (f64, f64, f64), -) -> DdSketch { - let (sum, min, max) = scalars; - let store = |(counts, offset): (&[u64], i32)| ((counts.to_vec(),), offset); - let state = ( - DD_ALPHA, - store(positive), - sum, - min, - max, - store(negative), - zero_count, - ); - rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state") -} - -fn assert_dd_golden(golden: &str, sketch: &DdSketch) { + (sum, min, max): (f64, f64, f64), +) { let want = decode_hex(golden); - assert_eq!(sketch.to_asapv1().expect("serialize"), want); let decoded = DdSketch::from_asapv1(&want).expect("decode"); + assert_eq!( + (decoded.positive_store_counts(), decoded.positive_store_offset()), + positive + ); + assert_eq!( + (decoded.negative_store_counts(), decoded.negative_store_offset()), + negative + ); + assert_eq!(decoded.zero_count(), zero_count); + assert_eq!(decoded.sum().to_bits(), sum.to_bits()); + assert_eq!(decoded.min().map(f64::to_bits), Some(min.to_bits())); + assert_eq!(decoded.max().map(f64::to_bits), Some(max.to_bits())); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } @@ -1076,20 +1076,21 @@ fn dd_buckets(counts: &[u64], offset: i32) -> Vec<(i32, u64)> { fn ddsketch_positive_a001_matches_golden() { let counts = [1, 0, 127, 128, 300, 65536, 4294967296]; let scalars = (2181071000.0, 0.453125, 0.5078125); - let sketch = dd_state((&counts, -40), (&[], 0), 0, scalars); - assert_dd_golden(GOLDEN_DD_POSITIVE, &sketch); + assert_dd_golden(GOLDEN_DD_POSITIVE, (&counts, -40), (&[], 0), 0, scalars); } #[test] fn ddsketch_signed_a001_matches_golden() { let scalars = (2523.90625, -0.016, 515.0); - let sketch = dd_state((&[3, 0, 2], 310), (&[5, 1], -208), 7, scalars); - assert_dd_golden(GOLDEN_DD_SIGNED, &sketch); + assert_dd_golden(GOLDEN_DD_SIGNED, (&[3, 0, 2], 310), (&[5, 1], -208), 7, scalars); } #[test] fn ddsketch_empty_a001_matches_golden() { - assert_dd_golden(GOLDEN_DD_EMPTY, &DdSketch::new(DD_ALPHA)); + let want = decode_hex(GOLDEN_DD_EMPTY); + assert_eq!(DdSketch::new(DD_ALPHA).to_asapv1().expect("serialize"), want); + let decoded = DdSketch::from_asapv1(&want).expect("decode"); + assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } #[test] @@ -1470,13 +1471,11 @@ fn univmon_empty_l3_2x4_h5_matches_golden() { } // --------------------------------------------------------------------------- -// CountSketchWithL2Norm: counts, l2 accumulators and seed index set through the sketch's -// serde form -> serialize == golden, round-trips. +// CountSketchWithL2Norm: golden decodes to the known counts and round-trips. +// Encoding the known state is checked in `src/sketches/countsketch_l2_norm/wire.rs`. // --------------------------------------------------------------------------- const L2HH_COUNTS: [[i64; 4]; 2] = [[127, 128, 65535, -32768], [-32, -33, -2147483648, i64::MIN]]; -/// Set independently of the counts. -const L2HH_L2: [i64; 2] = [65536, i64::MAX]; const L2HH_SEED_INDEX: usize = 7; #[test] @@ -1484,12 +1483,6 @@ fn count_l2hh_2x4_seed7_matches_golden() { let want = decode_hex(GOLDEN_L2HH); let flat: Vec = L2HH_COUNTS.iter().flatten().copied().collect(); - let state = ((&flat, 2, 4), (L2HH_L2,), 2, 4, L2HH_SEED_INDEX); - let sketch: CountSketchWithL2Norm = - rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state"); - let got = sketch.to_asapv1().expect("serialize"); - assert_eq!(got, want, "CountSketchWithL2Norm bytes diverge from golden"); - let decoded: CountSketchWithL2Norm = CountSketchWithL2Norm::from_asapv1(&want).expect("decode"); assert_eq!(decoded.as_storage().as_slice(), flat.as_slice()); assert_eq!((decoded.rows(), decoded.cols()), (2, 4)); @@ -1749,23 +1742,18 @@ fn kmv_k8_matches_golden() { } // --------------------------------------------------------------------------- -// Bloom: the bit grid set through the filter's serde form -> serialize == -// golden, round-trips. Column `c` of row `r` is bit `c % 64` of word +// Bloom: golden decodes to the known bit grid and round-trips. Encoding the +// known grid is checked in `src/sketches/bloom/wire.rs`. Column `c` of row `r` is bit `c % 64` of word // `r * ceil(cols / 64) + c / 64`. // --------------------------------------------------------------------------- fn assert_bloom_golden( golden: &str, - (rows, cols, mode): (usize, usize, &str), + (rows, cols): (usize, usize), words: &[u64], inserted: u64, ) { let want = decode_hex(golden); - let state = ((words, rows, cols), inserted, mode); - let sketch: Bloom = - rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state"); - assert_eq!(sketch.to_asapv1().expect("serialize"), want); - let decoded = Bloom::::from_asapv1(&want).expect("decode"); assert_eq!((decoded.rows(), decoded.cols()), (rows, cols)); assert_eq!(decoded.inserted(), inserted); @@ -1782,36 +1770,28 @@ fn assert_bloom_golden( #[test] fn bloom_regular_3x128_matches_golden() { let words = [1, 1 << 63, 0, 0xff, u64::MAX, 0x10000]; - assert_bloom_golden::(GOLDEN_BLOOM_REGULAR, (3, 128, "regular"), &words, 300); + assert_bloom_golden::(GOLDEN_BLOOM_REGULAR, (3, 128), &words, 300); } #[test] fn bloom_fast_5x32_matches_golden() { let words = [1, 0x8000_0000, 0, 0xffff_ffff, 0x10000]; - assert_bloom_golden::(GOLDEN_BLOOM_FAST, (5, 32, "fast"), &words, 9); + assert_bloom_golden::(GOLDEN_BLOOM_FAST, (5, 32), &words, 9); } // --------------------------------------------------------------------------- -// Space-Saving: the triples set through the summary's serde form, seated out -// of emitted order -> serialize == golden, round-trips. +// Space-Saving: golden decodes to the known triples and round-trips. Encoding +// the known triples is checked in `src/sketches/space_saving/wire.rs`. // --------------------------------------------------------------------------- fn assert_space_saving_golden( golden: &str, capacity: usize, total: u64, - discarded_max: u64, emitted: &[(OwnedDataInput, u64, u64)], min_count: u64, ) { let want = decode_hex(golden); - let mut seated = emitted.to_vec(); - seated.reverse(); - let state = (capacity, total, discarded_max, seated); - let sketch: SpaceSaving = - rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state"); - assert_eq!(sketch.to_asapv1().expect("serialize"), want); - let decoded: SpaceSaving = SpaceSaving::from_asapv1(&want).expect("decode"); assert_eq!((decoded.capacity(), decoded.total()), (capacity, total)); assert_eq!(decoded.min_count(), min_count); @@ -1835,7 +1815,7 @@ fn space_saving_i64_c8_matches_golden() { (-129, 5, 0), ] .map(|(key, count, error)| (OwnedDataInput::I64(key), count, error)); - assert_space_saving_golden(GOLDEN_SPACE_SAVING_I64, 8, 70030, 4, &emitted, 4); + assert_space_saving_golden(GOLDEN_SPACE_SAVING_I64, 8, 70030, &emitted, 4); } /// A full summary: `min_count` is its smallest count. `b""` precedes `b"ab"`. @@ -1843,12 +1823,12 @@ fn space_saving_i64_c8_matches_golden() { fn space_saving_bytes_c3_matches_golden() { let emitted = [(&b"\xff\x00"[..], 9, 0), (b"", 4, 1), (b"ab", 4, 0)] .map(|(key, count, error)| (OwnedDataInput::Bytes(key.to_vec()), count, error)); - assert_space_saving_golden(GOLDEN_SPACE_SAVING_BYTES, 3, 20, 3, &emitted, 4); + assert_space_saving_golden(GOLDEN_SPACE_SAVING_BYTES, 3, 20, &emitted, 4); } // --------------------------------------------------------------------------- -// UniformSampling: entries and running scalars set through the sampler's serde -// form -> serialize == golden, round-trips. +// UniformSampling: golden decodes to the known samples and round-trips. +// Encoding the known state is checked in `src/sketches/uniform/wire.rs`. // --------------------------------------------------------------------------- /// The priority-3 tie orders `-0.0` before `0.0`; a `NaN` sample is legal. @@ -1862,11 +1842,6 @@ fn uniform_sampling_r0p25_matches_golden() { (1 << 63, 1e300), (u64::MAX, f64::NAN), ]; - let state = (0.25f64, 20u64, 0x0123_4567_89ab_cdefu64, entries); - let sketch: UniformSampling = - rmp_serde::from_slice(&rmp_serde::to_vec(&state).unwrap()).expect("valid state"); - assert_eq!(sketch.to_asapv1().expect("serialize"), want); - let decoded = UniformSampling::from_asapv1(&want).expect("decode"); assert_eq!((decoded.sample_rate(), decoded.total_seen()), (0.25, 20)); let bits: Vec = decoded.samples().iter().map(|v| v.to_bits()).collect(); diff --git a/tests/ddsketch_signed.rs b/tests/ddsketch_signed.rs index 9413fe0a..22855f57 100644 --- a/tests/ddsketch_signed.rs +++ b/tests/ddsketch_signed.rs @@ -76,7 +76,7 @@ fn zero_only_special_values_and_octo_flush() { } #[test] -fn tiny_values_are_counted_in_zero_bucket_and_signed_serde_survives() { +fn tiny_values_are_counted_in_zero_bucket_and_survive_asapv1() { let mut core = DdSketch::new(0.01); let tiny = f64::MIN_POSITIVE / 2.; for v in [-tiny, tiny, 0.] { @@ -85,7 +85,7 @@ fn tiny_values_are_counted_in_zero_bucket_and_signed_serde_survives() { assert_eq!(core.get_count(), 3); assert_eq!(core.zero_count(), 3); core.try_add(&-3.).unwrap(); - let decoded: DdSketch = rmp_serde::from_slice(&rmp_serde::to_vec(&core).unwrap()).unwrap(); + let decoded = DdSketch::from_asapv1(&core.to_asapv1().unwrap()).unwrap(); assert_eq!(decoded.get_count(), 4); assert_eq!(decoded.sum(), core.sum()); assert_eq!(decoded.min(), core.min()); diff --git a/tests/e2e/heavy_hitters.rs b/tests/e2e/heavy_hitters.rs index 1b2dee15..76f6f22d 100644 --- a/tests/e2e/heavy_hitters.rs +++ b/tests/e2e/heavy_hitters.rs @@ -358,16 +358,16 @@ fn a_merge_keeps_the_total_and_never_reads_low() { ); } -/// A round-trip through serde preserves every answer, including the key index -/// the summary looks up on. +/// An ASAPv1 round trip preserves every answer, including the key index the +/// summary looks up on. #[test] -fn a_serde_round_trip_preserves_every_answer() { +fn an_asapv1_round_trip_preserves_every_answer() { let stream = stream(); let truth = truth_of(&stream); let summary = filled(256, &stream); - let bytes = rmp_serde::to_vec(&summary).expect("serialize"); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("deserialize"); + let bytes = summary.to_asapv1().expect("serialize"); + let decoded: SpaceSaving = SpaceSaving::from_asapv1(&bytes).expect("deserialize"); assert_eq!(decoded.len(), summary.len()); assert_eq!(decoded.capacity(), summary.capacity()); @@ -610,8 +610,8 @@ fn a_round_trip_carries_the_merged_ceiling() { let mut merged = filled(32, &stream[..stream.len() / 2]); merged.merge(&filled(4, &stream[stream.len() / 2..])); - let bytes = rmp_serde::to_vec(&merged).expect("serialize"); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("deserialize"); + let bytes = merged.to_asapv1().expect("serialize"); + let decoded: SpaceSaving = SpaceSaving::from_asapv1(&bytes).expect("deserialize"); assert_eq!(decoded.min_count(), merged.min_count()); assert_eq!(decoded.total(), merged.total()); @@ -627,98 +627,6 @@ fn a_round_trip_carries_the_merged_ceiling() { } } -/// Serialized state that no run of the algorithm could have produced is -/// refused, rather than decoded into a summary that panics or loops. -#[test] -fn crafted_state_fails_closed() { - #[derive(serde::Serialize)] - struct CraftedState { - capacity: usize, - total: u64, - discarded_max: u64, - entries: Vec<(OwnedDataInput, u64, u64)>, - } - - let crafted = |capacity: usize, - total: u64, - discarded_max: u64, - entries: Vec<(OwnedDataInput, u64, u64)>| { - rmp_serde::to_vec(&CraftedState { - capacity, - total, - discarded_max, - entries, - }) - .expect("serialize") - }; - - let refused = [ - (crafted(0, 100, 0, Vec::new()), "capacity is zero"), - ( - crafted( - 1, - 100, - 0, - vec![ - (OwnedDataInput::I64(1), 2, 0), - (OwnedDataInput::I64(2), 2, 0), - ], - ), - "over a capacity of 1", - ), - ( - crafted(4, 100, 0, vec![(OwnedDataInput::I64(1), 0, 0)]), - "counter at zero", - ), - ( - crafted(4, 100, 0, vec![(OwnedDataInput::I64(1), 3, 9)]), - "error of 9 against a count of 3", - ), - ( - crafted( - 4, - 100, - 0, - vec![ - (OwnedDataInput::I64(1), 3, 0), - (OwnedDataInput::I64(1), 2, 0), - ], - ), - "same key twice", - ), - ( - crafted(4, 100, 200, vec![(OwnedDataInput::I64(1), 3, 0)]), - "ceiling of 200 above its lowest count of 3", - ), - ( - crafted(4, 2, 0, vec![(OwnedDataInput::I64(1), 9, 0)]), - "total of 2 under the 9", - ), - ]; - for (bytes, expected) in &refused { - let problem = rmp_serde::from_slice::(bytes) - .expect_err("a crafted state must be refused, not decoded") - .to_string(); - assert!( - problem.contains(expected), - "expected a complaint about {expected}, got {problem}" - ); - } - - let bytes = crafted( - 4, - 100, - 0, - vec![ - (OwnedDataInput::I64(1), 3, 1), - (OwnedDataInput::I64(2), 9, 2), - ], - ); - let decoded: SpaceSaving = rmp_serde::from_slice(&bytes).expect("a well formed state"); - assert_eq!(decoded.top_k(8).len(), 2); - assert_eq!(decoded.estimate(&DataInput::I64(2)), 9); -} - /// `SpaceSaving::default()` is the documented counter budget, and it fills. #[test] fn the_default_summary_holds_the_default_capacity() { diff --git a/tests/e2e/membership.rs b/tests/e2e/membership.rs index 6047aa80..634818a3 100644 --- a/tests/e2e/membership.rs +++ b/tests/e2e/membership.rs @@ -482,13 +482,12 @@ fn clearing_restores_an_empty_filter() { } } -/// A round-trip through serde preserves every answer, members and probes -/// alike. +/// An ASAPv1 round trip preserves every answer, members and probes alike. #[test] -fn a_serde_round_trip_preserves_every_answer() { +fn an_asapv1_round_trip_preserves_every_answer() { let filter = filled_regular(0.01); - let bytes = rmp_serde::to_vec(&filter).expect("serialize"); - let decoded: Bloom = rmp_serde::from_slice(&bytes).expect("deserialize"); + let bytes = filter.to_asapv1().expect("serialize"); + let decoded = Bloom::::from_asapv1(&bytes).expect("deserialize"); assert_eq!(decoded.rows(), filter.rows()); assert_eq!(decoded.cols(), filter.cols()); @@ -585,7 +584,7 @@ fn the_two_hash_paths_agree_only_where_the_geometry_gives_each_row_its_own_hash( ); } -/// The serialized form carries which path built the filter, so bytes cannot be +/// The ASAPv1 metadata carries which path built the filter, so bytes cannot be /// decoded into the other one. Without the tag this succeeds and the filter /// then reports almost every one of its own members absent. #[test] @@ -596,21 +595,19 @@ fn a_filter_cannot_be_decoded_into_the_other_hash_path() { fast.update(&DataInput::I64(key)); } - let regular_bytes = rmp_serde::to_vec(®ular).expect("serialize"); - let fast_bytes = rmp_serde::to_vec(&fast).expect("serialize"); + let regular_bytes = regular.to_asapv1().expect("serialize"); + let fast_bytes = fast.to_asapv1().expect("serialize"); - let err = rmp_serde::from_slice::>(®ular_bytes) - .expect_err("regular bytes decoded as fast path"); assert!( - err.to_string().contains("regular") && err.to_string().contains("fast"), - "unexpected error: {err}" + Bloom::::from_asapv1(®ular_bytes).is_err(), + "regular bytes decoded as fast path" ); - assert!(rmp_serde::from_slice::>(&fast_bytes).is_err()); + assert!(Bloom::::from_asapv1(&fast_bytes).is_err()); // `RegularPath` is the default parameter, so the unannotated form is the // one a caller reaches for and it must fail too. - assert!(rmp_serde::from_slice::(&fast_bytes).is_err()); + assert!(::from_asapv1(&fast_bytes).is_err()); - let decoded: Bloom = rmp_serde::from_slice(&fast_bytes).expect("same path decodes"); + let decoded = Bloom::::from_asapv1(&fast_bytes).expect("same path decodes"); for key in members() { assert!(decoded.contains(&DataInput::I64(key))); } @@ -618,13 +615,13 @@ fn a_filter_cannot_be_decoded_into_the_other_hash_path() { /// The fast path round-trips like the regular one, tag and all. #[test] -fn a_fast_path_serde_round_trip_preserves_every_answer() { +fn a_fast_path_asapv1_round_trip_preserves_every_answer() { let mut filter = Bloom::::with_capacity(MEMBERS as usize, 0.01); for key in members() { filter.update(&DataInput::I64(key)); } - let bytes = rmp_serde::to_vec(&filter).expect("serialize"); - let decoded: Bloom = rmp_serde::from_slice(&bytes).expect("deserialize"); + let bytes = filter.to_asapv1().expect("serialize"); + let decoded = Bloom::::from_asapv1(&bytes).expect("deserialize"); assert_eq!( (decoded.rows(), decoded.cols()), diff --git a/tests/e2e/nitro.rs b/tests/e2e/nitro.rs index ca9123d7..62a593f4 100644 --- a/tests/e2e/nitro.rs +++ b/tests/e2e/nitro.rs @@ -670,20 +670,13 @@ fn nitro_saturates_oversized_weights_instead_of_wrapping() { } // --------------------------------------------------------------------------- -// Continuation across a serde round trip and across a context snapshot +// Continuation across a context snapshot // --------------------------------------------------------------------------- /// Rates whose reciprocal is not an integer, so the stochastic-rounding stream /// is live and its state actually has to survive. const CONTINUATION_RATES: [f64; 2] = [0.3, 0.07]; -fn round_trip( - cm: &CountMinSketch, FastPath>, -) -> CountMinSketch, FastPath> { - let bytes = rmp_serde::to_vec_named(cm).expect("serialize a Nitro-enabled CountMinSketch"); - rmp_serde::from_slice(&bytes).expect("deserialize a Nitro-enabled CountMinSketch") -} - fn nitro_enabled(rate: f64, seed: u64) -> CountMinSketch, FastPath> { let mut cm = CountMinSketch::, FastPath>::with_dimensions(ROWS, COLS); cm.enable_nitro_with_seed(rate, seed); @@ -698,152 +691,6 @@ fn cells(cm: &CountMinSketch, FastPath>) -> Vec { .collect() } -/// A sketch that is encoded and decoded after every update must end up -/// identical to one that was never interrupted. -/// -/// This is the property the sampling state has to be *complete* for. The -/// rounding stream is part of it: it is serialized, so a decoded sketch -/// continues the Bernoulli sequence rather than restarting it from a constant, -/// which matters at any rate whose reciprocal is not an integer. -#[test] -fn a_serde_round_trip_after_every_update_reproduces_the_uninterrupted_run() { - const UPDATES: usize = 600; - const TAIL: usize = 200; - let key = DataInput::U64(SOLO_KEY); - - for &rate in &CONTINUATION_RATES { - let seed = trial_seed(0); - let mut continuous = nitro_enabled(rate, seed); - let mut interrupted = nitro_enabled(rate, seed); - - for _ in 0..UPDATES { - continuous.nitro_update(&key); - interrupted.nitro_update(&key); - interrupted = round_trip(&interrupted); - } - - assert_eq!( - cells(&interrupted), - cells(&continuous), - "rate={rate}: a sketch round-tripped after every one of {UPDATES} updates \ - must hold the same counters, cell for cell, as one that was never \ - serialized" - ); - assert_eq!( - interrupted.nitro_estimate(&key), - continuous.nitro_estimate(&key), - "rate={rate}: the estimates must agree exactly" - ); - - // And the *future* has to agree too: a decoded sketch that resumed - // from a reset rounding stream can hold the right counters and still - // diverge on the next update. - for _ in 0..TAIL { - continuous.nitro_update(&key); - interrupted.nitro_update(&key); - } - assert_eq!( - cells(&interrupted), - cells(&continuous), - "rate={rate}: {TAIL} further updates after the last round trip must follow \ - the same admission and weight sequence" - ); - } -} - -/// A payload written before `Nitro::rounding_state` existed must still decode. -/// -/// `rounding_state` is a trailing serialized field carrying -/// `#[serde(default)]`, so a map-encoded payload that simply lacks the key -/// decodes with the default stream. This builds exactly that payload from a -/// mirror of the field layout without it — there is no other way to produce -/// one once the field exists — and checks that the counters survive and the -/// decoded sketch is usable. -/// -/// `mask_bits` and `mask` are present because such payloads carry them; -/// `Vector2D`'s decoder recomputes both from `cols` and ignores what the -/// payload says, so the values below only have to be well-formed. -#[test] -fn a_payload_written_before_the_rounding_field_existed_still_decodes() { - #[derive(serde::Serialize)] - struct LegacyNitro { - is_nitro_mode: bool, - sampling_rate: f64, - to_skip: usize, - inv_ln_one_minus_p: f64, - delta: u64, - idx: usize, - mask: usize, - } - #[derive(serde::Serialize)] - struct LegacyVector2D { - data: Vec, - rows: usize, - cols: usize, - mask_bits: u32, - mask: u128, - nitro: LegacyNitro, - } - #[derive(serde::Serialize)] - struct LegacyCountMin { - counts: LegacyVector2D, - rows: usize, - cols: usize, - } - - const RATE: f64 = 0.3; - // Row 0 column 17 holds 4; nothing else is set. Any recognisable pattern - // does, since the point is that the counters survive the decode. - let mut data = vec![0i32; ROWS * COLS]; - data[17] = 4; - - let legacy = LegacyCountMin { - counts: LegacyVector2D { - data: data.clone(), - rows: ROWS, - cols: COLS, - mask_bits: COLS.trailing_zeros(), - mask: (COLS - 1) as u128, - nitro: LegacyNitro { - is_nitro_mode: true, - sampling_rate: RATE, - to_skip: 2, - inv_ln_one_minus_p: 1.0 / (1.0 - RATE).ln(), - delta: (1.0 / RATE).floor() as u64, - idx: 9, - mask: 0x1_0000, - }, - }, - rows: ROWS, - cols: COLS, - }; - - let bytes = rmp_serde::to_vec_named(&legacy).expect("encode the legacy shape"); - let mut decoded: CountMinSketch, FastPath> = - rmp_serde::from_slice(&bytes).expect("a payload without `rounding_state` must decode"); - - assert_eq!( - cells(&decoded), - data, - "the counters must survive a decode of the pre-`rounding_state` shape" - ); - - // And the decoded sketch keeps sampling: the rounding stream starts from - // its default rather than failing to decode. - let key = DataInput::U64(SOLO_KEY); - for _ in 0..2_000 { - decoded.nitro_update(&key); - } - let spec = SamplingConfidenceSpec::new(RATE, Z); - let mass = signed_row_masses(decoded.as_storage())[0] - 4.0; - assert!( - spec.check(mass, 2_000.0, 0.0).is_ok(), - "a decoded legacy sketch must keep sampling at its stored rate: row-0 mass \ - {mass} for 2000 updates at p={RATE}, band +-{:.1}", - spec.half_width(2_000.0) - ); -} - /// A context snapshot taken mid-stream and restored onto a fresh sketch must /// replay the rest of the stream exactly. /// diff --git a/tests/hll_custom_precision.rs b/tests/hll_custom_precision.rs index 26bb3867..697d4691 100644 --- a/tests/hll_custom_precision.rs +++ b/tests/hll_custom_precision.rs @@ -1,9 +1,8 @@ //! Exercises `impl_hll_registers!` from outside the crate. //! //! This is an integration test, so it compiles as a separate crate: it only -//! sees `asap_sketchlib`'s public surface, and it never imports `serde`, -//! `serde-big-array`, or any of the other names the macro expansion needs. -//! That makes it a real check that the exported macro is usable downstream. +//! sees `asap_sketchlib`'s public surface. That makes it a real check that the +//! exported macro is usable downstream. //! //! The precisions here are deliberately outside the built-in //! {12, 14, 16} set, including `lg_k = 18` (needed by sketch-bench's @@ -148,21 +147,6 @@ fn custom_precision_indexing_and_iteration() { assert_eq!(sum, 7 + 1 + 2 + 3); } -#[test] -fn custom_precision_round_trips_through_serde() { - let mut sketch = HyperLogLogImpl::::new(); - for i in 0..5_000 { - sketch.update(&DataInput::String(item(i))); - } - - let bytes = rmp_serde::to_vec(&sketch).expect("serialize"); - let restored: HyperLogLogImpl = - rmp_serde::from_slice(&bytes).expect("deserialize"); - - assert_eq!(restored.registers_as_slice(), sketch.registers_as_slice()); - assert_eq!(restored.estimate(), sketch.estimate()); -} - /// Regression: register storage must be allocated on the heap, never built as /// an `[u8; N]` value and copied into the box. Test threads get a 2 MiB stack /// and lg_k=22 is a 4 MiB array, so a stack temporary overflows in debug @@ -181,15 +165,14 @@ fn large_precision_allocates_on_the_heap() { for i in 0..1_000 { big.update(&DataInput::String(item(i))); } - let bytes = rmp_serde::to_vec(&big).expect("serialize"); - let restored: HyperLogLogImpl = - rmp_serde::from_slice(&bytes).expect("deserialize"); + let bytes = big.to_asapv1().expect("serialize"); + let restored = HyperLogLogImpl::::from_asapv1(&bytes) + .expect("deserialize"); assert_eq!(restored.registers_as_slice(), big.registers_as_slice()); } -/// The ASAPv1 envelope is the cross-language format, and it is a different -/// code path from the derive-based serde impls above: it writes `precision` -/// into the metadata and validates the register length on the way back in. +/// The ASAPv1 envelope writes `precision` into the metadata and validates the +/// register length on the way back in. /// Covers all three variants, including HIP's extra running scalars. #[test] fn custom_precision_round_trips_through_the_asapv1_wire_format() { diff --git a/tests/pbt/count_min_hll.rs b/tests/pbt/count_min_hll.rs index 57113bf7..811b5e8a 100644 --- a/tests/pbt/count_min_hll.rs +++ b/tests/pbt/count_min_hll.rs @@ -299,22 +299,6 @@ macro_rules! shape_laws { prop_assert_eq!(with_empty.as_storage().as_slice(), &before[..]); } - #[test] - fn a_round_trip_preserves_the_grid_and_its_answers(groups in groups()) { - let sketch = sketch_of(&groups); - let probed: Vec = groups.iter().map(|(key, _)| *key).collect(); - - round_trip!( - @raw_serde Sketch, - sketch, - |s: &Sketch| s.as_storage().as_slice().to_vec(), - |s: &Sketch| (s.rows(), s.cols(), s.precision()), - |s: &Sketch| probed - .iter() - .map(|k| s.estimate(&DataInput::U64(*k)).to_bits()) - .collect::>(), - ); - } } } }; diff --git a/tests/pbt/microscope.rs b/tests/pbt/microscope.rs index 728bda42..4d1bfd90 100644 --- a/tests/pbt/microscope.rs +++ b/tests/pbt/microscope.rs @@ -23,7 +23,6 @@ use asap_sketchlib::sketches::microscope::{ }; use asap_sketchlib::{DataInput, DefaultXxHasher, MatrixFastHash, MicroCm, SketchHasher, Vector3D}; use proptest::prelude::*; -use serde::Serialize; use std::collections::{HashMap, VecDeque}; /// Key domain the streams draw from. Narrow enough that a handful of columns @@ -418,12 +417,11 @@ fn the_rounding_seed_reaches_the_zoom() { proptest! { #![proptest_config(ProptestConfig::with_cases(96))] - // ===== A live cell holds what the wire refuses to decode without ===== + // ===== A live cell never holds a whole unit in its shutter ===== // - // `deserialize` rejects a record whose shutter has reached one `c^Z` - // unit, because that is the assumption bounding the carry a merge - // computes. A payload cannot be the only way in: the running sketch has - // to keep it too, through every zoom-out, boundary and reclaim. + // A shutter below one `c^Z` unit is the assumption bounding the carry a + // merge computes. The running sketch keeps it through every zoom-out, + // boundary and reclaim. #[test] fn a_live_cell_never_holds_a_whole_unit_in_its_shutter(run in zooming_run()) { @@ -808,81 +806,4 @@ proptest! { } } - // ===== A grid that was sized but never filled ===== - // - // `Vector3D::unfilled` leaves the storage empty and every bucket accessor - // panics until `fill` has run, so a `MicroCm` around one would panic on - // its first insert or query. No public constructor produces that state, - // and the wire must not either: the geometry a payload declares has to - // match the bytes it carries. - - #[test] - fn a_grid_that_was_sized_but_never_filled_is_refused_on_the_wire( - rows in 1usize..4, - cols in prop_oneof![Just(2usize), Just(4), Just(8)], - t in 1usize..5, - c in prop_oneof![Just(2u32), Just(3), Just(4)], - per_sub_window in 1u64..1000, - ) { - #[derive(Serialize)] - struct Forged { - cells: Vector3D, - params: MicroParams, - clock: SubWindowClock, - rounding: u64, - } - - let params = MicroParams::new(t, c); - let depth = Micro::with_dimensions( - rows, cols, params, SubWindowClock::count_based(per_sub_window), 0, - ).cell_bytes(); - - let forge = |cells: Vector3D| { - let bytes = rmp_serde::to_vec_named(&Forged { - cells, - params, - clock: SubWindowClock::count_based(per_sub_window), - rounding: 1, - }).expect("encode"); - Micro::deserialize_from_bytes(&bytes).map_err(|e| e.to_string()) - }; - - // The same geometry, filled, is the control: the refusal below is - // about the missing bytes and nothing else. - let mut filled = Vector3D::unfilled(rows, cols, depth); - filled.fill(0u8); - prop_assert!(forge(filled).is_ok(), "the control payload must decode"); - - let refused = forge(Vector3D::unfilled(rows, cols, depth)); - prop_assert!(refused.is_err(), "a grid with no bytes must be refused"); - let err = refused.err().unwrap_or_default(); - prop_assert!( - err.contains("does not match rows * cols * bucket_len"), - "unexpected refusal for an unfilled {}x{}x{} grid: {}", - rows, cols, depth, err - ); - } - - // ===== The wire ===== - - #[test] - fn a_zoomed_sketch_round_trips_to_the_same_answers(run in zooming_run()) { - let mut sketch = run.empty(); - let peak = run.drive(&mut sketch, &mut Window::new(run.t)); - prop_assert!(peak > 0, "the run never zoomed: {:?}", run); - - let bytes = sketch.serialize_to_bytes().expect("encode"); - let decoded = Micro::deserialize_from_bytes(&bytes).expect("decode"); - prop_assert_eq!(decoded.max_zoom(), sketch.max_zoom()); - prop_assert_eq!(decoded.sub_window(), sketch.sub_window()); - for k in 0..DOMAIN { - prop_assert!( - close( - decoded.estimate_with(&key(k), DeltaStrategy::Over), - sketch.estimate_with(&key(k), DeltaStrategy::Over), - ), - "key {} answers differently after a round trip", k - ); - } - } } diff --git a/tests/pbt/support.rs b/tests/pbt/support.rs index 6d80e5fd..9bdd289a 100644 --- a/tests/pbt/support.rs +++ b/tests/pbt/support.rs @@ -14,22 +14,16 @@ use proptest::prelude::*; /// cannot represent every state their constructor accepts. Silent corruption /// is not allowed. macro_rules! round_trip { - ($ty:ty, $sketch:expr, $($probe:expr),+ $(,)?) => { - round_trip!(@codec to_asapv1, from_asapv1, $ty, $sketch, $($probe),+) - }; - (@raw_serde $ty:ty, $sketch:expr, $($probe:expr),+ $(,)?) => { - round_trip!(@codec serialize_to_bytes, deserialize_from_bytes, $ty, $sketch, $($probe),+) - }; - (@codec $encode:ident, $decode:ident, $ty:ty, $sketch:expr, $($probe:expr),+) => {{ + ($ty:ty, $sketch:expr, $($probe:expr),+ $(,)?) => {{ let original = &$sketch; - if let Ok(bytes) = original.$encode() { - let decoded = <$ty>::$decode(&bytes) + if let Ok(bytes) = original.to_asapv1() { + let decoded = <$ty>::from_asapv1(&bytes) .expect("bytes the encoder produced must decode"); $( prop_assert_eq!($probe(&decoded), $probe(original)); )+ prop_assert_eq!( - decoded.$encode().expect("a decoded sketch must re-encode"), + decoded.to_asapv1().expect("a decoded sketch must re-encode"), bytes, "the re-encode is not byte-identical" ); From 763c7c6ff91d530c7b152fdad39c81a556c190be Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 20 Oct 2026 06:45:36 +0000 Subject: [PATCH 4/4] fix: build fixes for the format removals (#206, #207, #208) Drop the Count-Min and Count Sketch `rows`/`cols` fields only serde read, the HLL wire re-exports only the portable path used, and imports the removals left unused; gate KMV's heap constructor on `experimental`; build the DDSketch admission golden from the decoded signed state; refresh the lockfile and the PBT law counts. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 69 ------------------- src/asapv1/admission_rate.rs | 2 +- src/asapv1/wire_key.rs | 2 +- src/common/heap.rs | 1 - src/common/structures/bit_matrix.rs | 1 - src/common/structures/heap.rs | 1 + src/common/structures/vector3d.rs | 2 - src/lib.rs | 2 +- src/sketch_framework/hydra.rs | 1 - src/sketch_framework/hydra/wire.rs | 4 +- src/sketches/bloom/wire.rs | 11 ++- src/sketches/coco/wire.rs | 2 +- src/sketches/count_min_fold.rs | 1 - src/sketches/count_sketch_fold.rs | 1 - src/sketches/countminsketch.rs | 10 +-- src/sketches/countminsketch/wire.rs | 2 +- src/sketches/countminsketch_heap/heap_wire.rs | 2 +- src/sketches/countminsketch_hll.rs | 2 - src/sketches/countsketch.rs | 10 +-- src/sketches/countsketch/wire.rs | 2 +- src/sketches/countsketch_l2_norm/wire.rs | 9 ++- src/sketches/ddsketch.rs | 3 +- src/sketches/ddsketch/wire.rs | 10 ++- src/sketches/elastic/wire.rs | 2 +- src/sketches/hll.rs | 1 - src/sketches/hll/wire.rs | 20 ++++-- src/sketches/kll/wire.rs | 2 +- src/sketches/kll_dynamic.rs | 1 - src/sketches/microscope.rs | 2 - src/sketches/mod.rs | 2 +- src/sketches/mode.rs | 1 - src/sketches/space_saving/wire.rs | 18 +++-- src/sketches/uniform.rs | 1 - src/sketches/uniform/wire.rs | 4 +- tests/PBT_COVERAGE.md | 8 +-- tests/asapv1_golden.rs | 40 +++++++++-- tests/e2e/cardinality.rs | 5 +- tests/e2e/composition.rs | 2 +- tests/hll_custom_precision.rs | 4 +- tests/pbt/count_min.rs | 4 +- tests/pbt/count_sketch.rs | 4 +- tests/pbt/microscope.rs | 3 +- 42 files changed, 116 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4f6f24d..73e03e7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,29 +2,19 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - [[package]] name = "asap_sketchlib" version = "0.3.0" dependencies = [ - "bytes", "core_affinity", "crossbeam-channel", "proptest", - "prost", "rand", "rmp-serde", "serde", - "serde-big-array", "serde_bytes", "smallvec", "twox-hash", - "xxhash-rust", ] [[package]] @@ -60,12 +50,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - [[package]] name = "cfg-if" version = "1.0.3" @@ -98,12 +82,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "errno" version = "0.3.14" @@ -144,15 +122,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "libc" version = "0.2.177" @@ -233,29 +202,6 @@ dependencies = [ "unarray", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -378,15 +324,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-big-array" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" -dependencies = [ - "serde", -] - [[package]] name = "serde_bytes" version = "0.11.19" @@ -538,12 +475,6 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" -[[package]] -name = "xxhash-rust" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" - [[package]] name = "zerocopy" version = "0.8.27" diff --git a/src/asapv1/admission_rate.rs b/src/asapv1/admission_rate.rs index ae6d9c34..23f5bcee 100644 --- a/src/asapv1/admission_rate.rs +++ b/src/asapv1/admission_rate.rs @@ -60,8 +60,8 @@ pub(crate) fn deserialize<'de, D: Deserializer<'de>>( mod tests { use std::panic::{AssertUnwindSafe, catch_unwind}; - use crate::input::HydraCounter; use crate::asapv1::envelope; + use crate::input::HydraCounter; use crate::sketch_framework::eh_bucket_sketch::wire::{SketchState, rebuild_sketch}; use crate::sketches::CountSketchDelta; use crate::*; diff --git a/src/asapv1/wire_key.rs b/src/asapv1/wire_key.rs index 8ea3dc54..154e104b 100644 --- a/src/asapv1/wire_key.rs +++ b/src/asapv1/wire_key.rs @@ -252,8 +252,8 @@ impl<'de, V: Visitor<'de>> Visitor<'de> for Float32Only { #[cfg(test)] mod tests { use super::*; - use crate::input::HydraCounter; use crate::asapv1::envelope::{self, hex}; + use crate::input::HydraCounter; use crate::*; fn float64(value: f64) -> Vec { diff --git a/src/common/heap.rs b/src/common/heap.rs index b6f9aa19..60ff8036 100644 --- a/src/common/heap.rs +++ b/src/common/heap.rs @@ -965,7 +965,6 @@ mod index_invariants { heap.assert_index_consistent("refilled"); assert_eq!(heap.len(), 8); } - } #[cfg(test)] diff --git a/src/common/structures/bit_matrix.rs b/src/common/structures/bit_matrix.rs index ac346696..5e232596 100644 --- a/src/common/structures/bit_matrix.rs +++ b/src/common/structures/bit_matrix.rs @@ -534,5 +534,4 @@ mod tests { "unexpected error: {err}" ); } - } diff --git a/src/common/structures/heap.rs b/src/common/structures/heap.rs index d8aecab7..f77b2f8d 100644 --- a/src/common/structures/heap.rs +++ b/src/common/structures/heap.rs @@ -75,6 +75,7 @@ impl> CommonHeap { } /// Wraps `data` that already satisfies the heap invariant for `order`. + #[cfg(feature = "experimental")] pub(crate) fn from_heap_vec(data: Vec, capacity: usize, order: O) -> Self { Self { data, diff --git a/src/common/structures/vector3d.rs b/src/common/structures/vector3d.rs index 5ab5cc1f..35d7c80e 100644 --- a/src/common/structures/vector3d.rs +++ b/src/common/structures/vector3d.rs @@ -1,4 +1,3 @@ - use super::matrix_storage::cols_mask_bits; use crate::MatrixFastHash; @@ -342,5 +341,4 @@ mod tests { // bucket instead of panicking. let _ = v.bucket_slice(0, 5); } - } diff --git a/src/lib.rs b/src/lib.rs index dd11f5b7..26b6d165 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ pub mod sketches; #[cfg(test)] pub mod test_utils; -pub use common::*; pub use asapv1::envelope::peek_kind_id; +pub use common::*; pub use sketch_framework::*; pub use sketches::*; diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 4a153775..8f0124f0 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -20,7 +20,6 @@ //! - Manousis et al., VLDB 2022. //! - use crate::Vector2D; use crate::input::{HydraCounter, HydraQuery}; use crate::{DataInput, HYDRA_SEED_INDEX, hash_for_matrix_seeded}; diff --git a/src/sketch_framework/hydra/wire.rs b/src/sketch_framework/hydra/wire.rs index 373689dc..b2d41160 100644 --- a/src/sketch_framework/hydra/wire.rs +++ b/src/sketch_framework/hydra/wire.rs @@ -44,8 +44,6 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::de::IgnoredAny; use serde::{Deserialize, Serialize}; -use crate::common::hash::check_matrix_rows; -use crate::input::HydraCounter; use crate::asapv1::admission_rate; use crate::asapv1::envelope; use crate::asapv1::kind; @@ -53,6 +51,8 @@ use crate::asapv1::kind::{ HYDRA_COUNT_MIN, HYDRA_COUNT_SKETCH, HYDRA_HLL, HYDRA_KLL, HYDRA_UNIVMON, }; use crate::asapv1::wire_key::exact_vec; +use crate::common::hash::check_matrix_rows; +use crate::input::HydraCounter; use crate::sketch_framework::univmon::wire::{PyramidPayload, UnivMonMetadata, univmon_metadata}; use crate::sketches::countminsketch_heap::heap_wire::{EMPTY_KEY_TYPE, check_empty_key_type}; use crate::{ diff --git a/src/sketches/bloom/wire.rs b/src/sketches/bloom/wire.rs index fa792ea1..803d7cf2 100644 --- a/src/sketches/bloom/wire.rs +++ b/src/sketches/bloom/wire.rs @@ -627,7 +627,12 @@ mod tests { use crate::asapv1::envelope::hex; use std::marker::PhantomData; - fn bloom(rows: usize, cols: usize, words: &[u64], inserted: u64) -> Bloom { + fn bloom( + rows: usize, + cols: usize, + words: &[u64], + inserted: u64, + ) -> Bloom { Bloom { bits: BitMatrix::from_words(rows, cols, words.to_vec()).expect("valid grid"), inserted, @@ -637,7 +642,9 @@ mod tests { } let regular = bloom::(3, 128, &[1, 1 << 63, 0, 0xff, u64::MAX, 0x10000], 300); - let golden = hex(include_str!("../../../asapv1_golden/bloom_regular_3x128.hex")); + let golden = hex(include_str!( + "../../../asapv1_golden/bloom_regular_3x128.hex" + )); assert_eq!(regular.to_asapv1().unwrap(), golden); let fast = bloom::(5, 32, &[1, 0x8000_0000, 0, 0xffff_ffff, 0x10000], 9); diff --git a/src/sketches/coco/wire.rs b/src/sketches/coco/wire.rs index 34658ab3..a28bdbb4 100644 --- a/src/sketches/coco/wire.rs +++ b/src/sketches/coco/wire.rs @@ -39,10 +39,10 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; -use crate::common::hash::check_matrix_rows; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::COCO; +use crate::common::hash::check_matrix_rows; use crate::{DataInput, HashProfile, SketchHasher, Vector2D}; use super::{CocoBucket, CocoRng, CocoSketch}; diff --git a/src/sketches/count_min_fold.rs b/src/sketches/count_min_fold.rs index c1edb6b2..74d4d716 100644 --- a/src/sketches/count_min_fold.rs +++ b/src/sketches/count_min_fold.rs @@ -15,7 +15,6 @@ //! additional approximation error — the accuracy is identical to a full-width //! CMS with W columns. - use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap}; use std::marker::PhantomData; diff --git a/src/sketches/count_sketch_fold.rs b/src/sketches/count_sketch_fold.rs index e2b976d1..4ec43449 100644 --- a/src/sketches/count_sketch_fold.rs +++ b/src/sketches/count_sketch_fold.rs @@ -14,7 +14,6 @@ //! additional approximation error — the accuracy is identical to a full-width //! Count Sketch with W columns. - use crate::count_min_fold::FoldCell; use crate::{DataInput, DefaultXxHasher, SketchHasher, TopKHeap, compute_median_inline_f64}; use std::marker::PhantomData; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index a4ff220e..9f01e63c 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -10,8 +10,8 @@ use std::marker::PhantomData; -use crate::common::structure_utils::AdmittedRows; use crate::asapv1::admission_rate::admit_all; +use crate::common::structure_utils::AdmittedRows; use crate::octo_delta::{ CountMinSketchCellDelta, KeyedCountMinSketchCellDelta, OCTO_COUNT_MIN_DEFAULT_THRESHOLD, OCTO_MAX_PROMOTION_THRESHOLD, @@ -39,8 +39,6 @@ pub struct CountMinSketch< H: SketchHasher = DefaultXxHasher, > { counts: S, - rows: usize, - cols: usize, _mode: PhantomData, _hasher: PhantomData, admission_rate: f64, @@ -194,8 +192,6 @@ where pub fn with_dimensions(rows: usize, cols: usize) -> Self { CountMinSketch { counts: Vector2D::filled(rows, cols, T::default()), - rows, - cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), @@ -207,12 +203,8 @@ where impl CountMinSketch { /// Creates a sketch from an existing matrix storage instance. pub fn from_storage(counts: S) -> Self { - let rows = counts.rows(); - let cols = counts.cols(); Self { counts, - rows, - cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), diff --git a/src/sketches/countminsketch/wire.rs b/src/sketches/countminsketch/wire.rs index 60b02da2..c8437f9e 100644 --- a/src/sketches/countminsketch/wire.rs +++ b/src/sketches/countminsketch/wire.rs @@ -20,13 +20,13 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::common::hash::check_matrix_rows; use crate::asapv1::admission_rate; use crate::asapv1::delta::DeltaCounter; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::COUNT_MIN; use crate::asapv1::wire_key::exact_vec; +use crate::common::hash::check_matrix_rows; use crate::{FastPath, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::CountMinSketch; diff --git a/src/sketches/countminsketch_heap/heap_wire.rs b/src/sketches/countminsketch_heap/heap_wire.rs index f851cd5f..9760943d 100644 --- a/src/sketches/countminsketch_heap/heap_wire.rs +++ b/src/sketches/countminsketch_heap/heap_wire.rs @@ -49,9 +49,9 @@ use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashSet; -use crate::common::hash::check_matrix_rows; use crate::asapv1::envelope; use crate::asapv1::wire_key::{WireBytes, WireString, exact_vec}; +use crate::common::hash::check_matrix_rows; use crate::{HashProfile, HeapCount, OwnedDataInput, TopKHeap}; /// Metadata `key_type` of a heap that holds nothing. A heap with no entries has diff --git a/src/sketches/countminsketch_hll.rs b/src/sketches/countminsketch_hll.rs index 9ca80d57..ece6ddcf 100644 --- a/src/sketches/countminsketch_hll.rs +++ b/src/sketches/countminsketch_hll.rs @@ -237,7 +237,6 @@ impl CountMinHll { } Ok(()) } - } #[cfg(test)] @@ -464,5 +463,4 @@ mod tests { let sk = CountMinHll::::with_dimensions(21, 64, 6); assert_eq!(sk.rows(), 21); } - } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 645ef81b..157a8271 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -7,8 +7,8 @@ //! - Charikar, Chen & Farach-Colton, "Finding Frequent Items in Data Streams," //! ICALP 2002. -use crate::common::structure_utils::AdmittedRows; use crate::asapv1::admission_rate::admit_all; +use crate::common::structure_utils::AdmittedRows; use crate::{ DataInput, DefaultMatrixI32, DefaultMatrixI64, DefaultMatrixI128, DefaultXxHasher, FastPath, FastPathHasher, FixedMatrix, MatrixFastHash, MatrixStorage, NitroTarget, QuickMatrixI64, @@ -33,8 +33,6 @@ pub struct CountSketch< H: SketchHasher = DefaultXxHasher, > { counts: S, - rows: usize, - cols: usize, _mode: PhantomData, _hasher: PhantomData, admission_rate: f64, @@ -209,8 +207,6 @@ where pub fn with_dimensions(rows: usize, cols: usize) -> Self { CountSketch { counts: Vector2D::filled(rows, cols, T::from(0)), - rows, - cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), @@ -226,12 +222,8 @@ where { /// Wraps an existing matrix storage as a Count Sketch. pub fn from_storage(counts: S) -> Self { - let rows = counts.rows(); - let cols = counts.cols(); Self { counts, - rows, - cols, _mode: PhantomData, _hasher: PhantomData, admission_rate: admit_all(), diff --git a/src/sketches/countsketch/wire.rs b/src/sketches/countsketch/wire.rs index 5152075a..53ec5338 100644 --- a/src/sketches/countsketch/wire.rs +++ b/src/sketches/countsketch/wire.rs @@ -25,13 +25,13 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::common::hash::check_matrix_rows; use crate::asapv1::admission_rate; use crate::asapv1::delta::DeltaCounter; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::COUNT_SKETCH; use crate::asapv1::wire_key::exact_vec; +use crate::common::hash::check_matrix_rows; use crate::{FastPath, HashProfile, RegularPath, SketchHasher, Vector2D}; use super::{CountSketch, CountSketchCounter}; diff --git a/src/sketches/countsketch_l2_norm/wire.rs b/src/sketches/countsketch_l2_norm/wire.rs index b3b549c2..a354b7fc 100644 --- a/src/sketches/countsketch_l2_norm/wire.rs +++ b/src/sketches/countsketch_l2_norm/wire.rs @@ -31,10 +31,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use crate::common::hash::check_matrix_rows; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::COUNT_SKETCH_WITH_L2_NORM; +use crate::common::hash::check_matrix_rows; use crate::{HashProfile, SketchHasher, Vector1D, Vector2D}; use super::CountSketchWithL2Norm; @@ -709,7 +709,8 @@ mod tests { use crate::asapv1::envelope::hex; use std::marker::PhantomData; - const COUNTS: [[i64; 4]; 2] = [[127, 128, 65535, -32768], [-32, -33, -2147483648, i64::MIN]]; + const COUNTS: [[i64; 4]; 2] = + [[127, 128, 65535, -32768], [-32, -33, -2147483648, i64::MIN]]; let sketch: CountSketchWithL2Norm = CountSketchWithL2Norm { counts: Vector2D::from_fn(2, 4, |r, c| COUNTS[r][c]), row_l2_squared: Vector1D::from_vec(vec![65536, i64::MAX]), @@ -718,7 +719,9 @@ mod tests { seed_index: 7, _hasher: PhantomData, }; - let golden = hex(include_str!("../../../asapv1_golden/count_l2hh_2x4_seed7.hex")); + let golden = hex(include_str!( + "../../../asapv1_golden/count_l2hh_2x4_seed7.hex" + )); assert_eq!(sketch.to_asapv1().unwrap(), golden); } } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 7095af9b..89f35641 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -16,10 +16,10 @@ //! use crate::DataInput; +use crate::asapv1::admission_rate::{self, admit_all}; use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; use crate::common::structures::Vector1D; -use crate::asapv1::admission_rate::{self, admit_all}; use crate::octo_delta::{DdSign, DdSketchBucketDelta}; /// The ASAPv1 DDSketch delta (kind_id `0x05 0x80`). @@ -789,5 +789,4 @@ mod tests { d.add(&(max_indexable / 2.0)); assert_eq!(d.get_count(), count_before + 2); } - } diff --git a/src/sketches/ddsketch/wire.rs b/src/sketches/ddsketch/wire.rs index c4d82cf4..83bac86b 100644 --- a/src/sketches/ddsketch/wire.rs +++ b/src/sketches/ddsketch/wire.rs @@ -29,12 +29,12 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::common::structures::Vector1D; use crate::asapv1::admission_rate; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::DDSKETCH; use crate::asapv1::wire_key::exact; +use crate::common::structures::Vector1D; use super::{Buckets, DdSketch, DdSketchState}; @@ -696,7 +696,9 @@ mod tests { 0, (2181071000.0, 0.453125, 0.5078125), ); - let golden = hex(include_str!("../../../asapv1_golden/ddsketch_positive_a001.hex")); + let golden = hex(include_str!( + "../../../asapv1_golden/ddsketch_positive_a001.hex" + )); assert_eq!(positive.to_asapv1().unwrap(), golden); let signed = state( @@ -705,7 +707,9 @@ mod tests { 7, (2523.90625, -0.016, 515.0), ); - let golden = hex(include_str!("../../../asapv1_golden/ddsketch_signed_a001.hex")); + let golden = hex(include_str!( + "../../../asapv1_golden/ddsketch_signed_a001.hex" + )); assert_eq!(signed.to_asapv1().unwrap(), golden); } } diff --git a/src/sketches/elastic/wire.rs b/src/sketches/elastic/wire.rs index c6a004c5..ecb08dd8 100644 --- a/src/sketches/elastic/wire.rs +++ b/src/sketches/elastic/wire.rs @@ -36,11 +36,11 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError use serde::{Deserialize, Serialize}; use std::marker::PhantomData; -use crate::common::hash::check_matrix_rows; use crate::asapv1::admission_rate; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::kind::ELASTIC; +use crate::common::hash::check_matrix_rows; use crate::{CountMinSketch, HashProfile, SketchHasher, Vector2D}; use super::{ElasticSketch, HeavyBucket}; diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 2353c00e..007fffe5 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -46,7 +46,6 @@ mod delta; mod wire; pub use delta::HyperLogLogDelta; pub use wire::HllWireVariant; -pub(crate) use wire::{HllMetadata, HllPayloadHip, HllPayloadPlain, standard_hll_metadata}; /// Generic HyperLogLog sketch parameterized by estimation variant, register storage, and hasher. #[derive(Clone, Debug)] diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs index f2cf0bfd..4e1156fd 100644 --- a/src/sketches/hll/wire.rs +++ b/src/sketches/hll/wire.rs @@ -109,7 +109,8 @@ pub(crate) fn hll_metadata( } /// The standard ProjectASAP profile metadata (the [`DefaultXxHasher`] profile). -pub(crate) fn standard_hll_metadata(precision: u32) -> HllMetadata { +#[cfg(test)] +fn standard_hll_metadata(precision: u32) -> HllMetadata { hll_metadata::(precision, None) } @@ -281,9 +282,9 @@ impl HyperLogLogHipImpl { #[cfg(test)] mod tests { use super::*; + use crate::DataInput; use crate::sketches::hll::{HyperLogLog, HyperLogLogHip, HyperLogLogHipP12, HyperLogLogP12}; use crate::structures::fixed_structure::{HllRegistersP12, HllRegistersP14}; - use crate::{DataInput, HllRegisters}; const ERROR_TOLERANCE: f64 = 0.02; const SERDE_SAMPLE: usize = 100_000; @@ -455,7 +456,12 @@ mod tests { admission_rate: 1.0, } } - fn hip(regs: &[u8], kxq0: f64, kxq1: f64, est: f64) -> HyperLogLogHipImpl { + fn hip( + regs: &[u8], + kxq0: f64, + kxq1: f64, + est: f64, + ) -> HyperLogLogHipImpl { HyperLogLogHipImpl { registers: registers_from_bytes(regs).expect("registers"), kxq0, @@ -500,9 +506,13 @@ mod tests { ] { let (kxq0, kxq1, est) = state; let encoded = if regs.len() == 4096 { - hip::(regs, kxq0, kxq1, est).to_asapv1().unwrap() + hip::(regs, kxq0, kxq1, est) + .to_asapv1() + .unwrap() } else { - hip::(regs, kxq0, kxq1, est).to_asapv1().unwrap() + hip::(regs, kxq0, kxq1, est) + .to_asapv1() + .unwrap() }; assert_eq!(encoded, golden); } diff --git a/src/sketches/kll/wire.rs b/src/sketches/kll/wire.rs index abf4cc1f..6bfb8ec9 100644 --- a/src/sketches/kll/wire.rs +++ b/src/sketches/kll/wire.rs @@ -29,10 +29,10 @@ use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError}; use serde::{Deserialize, Serialize}; -use crate::common::numerical::NumericalValue; use crate::asapv1::envelope; use crate::asapv1::kind; use crate::asapv1::wire_key::exact_vec; +use crate::common::numerical::NumericalValue; use super::{ CAPACITY_CACHE_LEN, Coin, Kll, MAX_K, MAX_LEVELS, checked_weighted_count, compute_max_capacity, diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 4a5fa1da..0ed8b658 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -12,7 +12,6 @@ //! //! - - use crate::common::input::data_input_to_f64; use crate::common::numerical::NumericalValue; use crate::{DataInput, Vector1D}; diff --git a/src/sketches/microscope.rs b/src/sketches/microscope.rs index 8cec824c..2a130bac 100644 --- a/src/sketches/microscope.rs +++ b/src/sketches/microscope.rs @@ -586,7 +586,6 @@ impl MicroCm { } Ok(()) } - } #[cfg(test)] @@ -921,5 +920,4 @@ mod tests { assert!(err.contains("different sub-windows"), "unexpected: {err}"); assert_eq!(a.as_storage().as_slice(), before.as_slice()); } - } diff --git a/src/sketches/mod.rs b/src/sketches/mod.rs index 8539d027..53cdd64f 100644 --- a/src/sketches/mod.rs +++ b/src/sketches/mod.rs @@ -81,8 +81,8 @@ pub use elastic::ElasticSketch; /// An exact set of string keys and its delta. pub mod set_aggregator; -pub use set_aggregator::{SetAggregator, SetAggregatorDelta}; pub use elastic::HeavyBucket; +pub use set_aggregator::{SetAggregator, SetAggregatorDelta}; /// HyperLogLog implementations and aliases. pub mod hll; diff --git a/src/sketches/mode.rs b/src/sketches/mode.rs index 9c3d2538..6fc35223 100644 --- a/src/sketches/mode.rs +++ b/src/sketches/mode.rs @@ -1,4 +1,3 @@ - /// Hashing mode of a matrix-backed sketch in which row `r` hashes the key on /// its own, at seed-table index `r`. ASAPv1 `mode` `"regular"`. #[derive(Clone, Copy, Debug, Default)] diff --git a/src/sketches/space_saving/wire.rs b/src/sketches/space_saving/wire.rs index ab3915b5..cb08bb5d 100644 --- a/src/sketches/space_saving/wire.rs +++ b/src/sketches/space_saving/wire.rs @@ -1141,12 +1141,22 @@ mod tests { (-129, 5, 0), ] .map(|(key, count, error)| (OwnedDataInput::I64(key), count, error)); - let golden = hex(include_str!("../../../asapv1_golden/space_saving_i64_c8.hex")); - assert_eq!(summary(8, 70030, 4, i64_keys.to_vec()).to_asapv1().unwrap(), golden); + let golden = hex(include_str!( + "../../../asapv1_golden/space_saving_i64_c8.hex" + )); + assert_eq!( + summary(8, 70030, 4, i64_keys.to_vec()).to_asapv1().unwrap(), + golden + ); let byte_keys = [(&b"\xff\x00"[..], 9, 0), (b"", 4, 1), (b"ab", 4, 0)] .map(|(key, count, error)| (OwnedDataInput::Bytes(key.to_vec()), count, error)); - let golden = hex(include_str!("../../../asapv1_golden/space_saving_bytes_c3.hex")); - assert_eq!(summary(3, 20, 3, byte_keys.to_vec()).to_asapv1().unwrap(), golden); + let golden = hex(include_str!( + "../../../asapv1_golden/space_saving_bytes_c3.hex" + )); + assert_eq!( + summary(3, 20, 3, byte_keys.to_vec()).to_asapv1().unwrap(), + golden + ); } } diff --git a/src/sketches/uniform.rs b/src/sketches/uniform.rs index 2f898021..bb12c4db 100644 --- a/src/sketches/uniform.rs +++ b/src/sketches/uniform.rs @@ -1,4 +1,3 @@ - use crate::DataInput; mod wire; diff --git a/src/sketches/uniform/wire.rs b/src/sketches/uniform/wire.rs index cbde75a9..92ec09d8 100644 --- a/src/sketches/uniform/wire.rs +++ b/src/sketches/uniform/wire.rs @@ -501,7 +501,9 @@ mod tests { .map(|(priority, value)| SampleEntry::new(priority, value)) .to_vec(), }; - let golden = hex(include_str!("../../../asapv1_golden/uniform_sampling_r0p25.hex")); + let golden = hex(include_str!( + "../../../asapv1_golden/uniform_sampling_r0p25.hex" + )); assert_eq!(sampler.to_asapv1().unwrap(), golden); } } diff --git a/tests/PBT_COVERAGE.md b/tests/PBT_COVERAGE.md index 8d6f34a5..61e49791 100644 --- a/tests/PBT_COVERAGE.md +++ b/tests/PBT_COVERAGE.md @@ -2,8 +2,8 @@ One `proptest` law per line. Each compares the sketch against an answer computed without it; what that answer is comes first in every section. One -file per sketch under `tests/pbt/`. `cargo test --test pbt` runs 367 laws; -with `--all-features`, 435, adding the four **experimental** modules. Every +file per sketch under `tests/pbt/`. `cargo test --test pbt` runs 342 laws; +with `--all-features`, 407, adding the four **experimental** modules. Every law was mutation-checked: break the implementation, the law goes red. Shared generators, `grid` and `round_trip!` live in `support.rs`. @@ -461,7 +461,7 @@ The u64 key encoding must be injective and order preserving over every f64. Comp - wire - ASAPv1 keeps count, extrema, CDF breakpoints, F2; both counter widths, even and odd halving period, an overrun ordered sample -## count_min_hll.rs — 25 (experimental) +## count_min_hll.rs — 24 (experimental) A Count-Min grid of HLL registers. Compared against: a standalone `HyperLogLogImpl` on the same value hashes, byte-equal. Column routing read from a one-key probe sketch. 9 laws, 8 expanded over 3 shapes: 4 x 512 at p4 (rows mostly private), 4 x 32 at p6 and 3 x 32 at p8 (collisions). @@ -498,7 +498,7 @@ Retained set == the `k` smallest distinct hashes, so laws are mostly equalities. - wire - ASAPv1 keeps retained set, `k`, estimate; empty included -## microscope.rs — 15 (experimental) +## microscope.rs — 13 (experimental) Sliding-window frequency with adaptive zoom. A cell holds `t + 2` byte **pixels** (a ring indexed by sub-window) sharing one **zoom** exponent `Z`; a pixel is worth `c^Z` items and the cell rescales when a pixel would overflow. The **shutter** is the current sub-window's carry, `< c^Z`, rounded into its pixel at the boundary. A **reclaim** lowers `Z` at a boundary when load has fallen. Compared against: an exact `VecDeque` window tagged by sub-window. diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index d6aeb9bd..3053efdd 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -1050,11 +1050,17 @@ fn assert_dd_golden( let want = decode_hex(golden); let decoded = DdSketch::from_asapv1(&want).expect("decode"); assert_eq!( - (decoded.positive_store_counts(), decoded.positive_store_offset()), + ( + decoded.positive_store_counts(), + decoded.positive_store_offset() + ), positive ); assert_eq!( - (decoded.negative_store_counts(), decoded.negative_store_offset()), + ( + decoded.negative_store_counts(), + decoded.negative_store_offset() + ), negative ); assert_eq!(decoded.zero_count(), zero_count); @@ -1082,13 +1088,22 @@ fn ddsketch_positive_a001_matches_golden() { #[test] fn ddsketch_signed_a001_matches_golden() { let scalars = (2523.90625, -0.016, 515.0); - assert_dd_golden(GOLDEN_DD_SIGNED, (&[3, 0, 2], 310), (&[5, 1], -208), 7, scalars); + assert_dd_golden( + GOLDEN_DD_SIGNED, + (&[3, 0, 2], 310), + (&[5, 1], -208), + 7, + scalars, + ); } #[test] fn ddsketch_empty_a001_matches_golden() { let want = decode_hex(GOLDEN_DD_EMPTY); - assert_eq!(DdSketch::new(DD_ALPHA).to_asapv1().expect("serialize"), want); + assert_eq!( + DdSketch::new(DD_ALPHA).to_asapv1().expect("serialize"), + want + ); let decoded = DdSketch::from_asapv1(&want).expect("decode"); assert_eq!(decoded.to_asapv1().expect("re-serialize"), want); } @@ -1711,8 +1726,21 @@ fn cs_i64_regular_2x4_admission_matches_golden() { #[test] fn ddsketch_signed_a001_admission_matches_golden() { let scalars = (2523.90625, -0.016, 515.0); - let sketch = dd_state((&[3, 0, 2], 310), (&[5, 1], -208), 7, scalars); - assert_dd_golden(GOLDEN_DD_ADMISSION, &sketch.with_admission_rate(0.125)); + let signed = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_SIGNED)).expect("decode"); + assert_eq!( + signed + .with_admission_rate(0.125) + .to_asapv1() + .expect("serialize"), + decode_hex(GOLDEN_DD_ADMISSION) + ); + assert_dd_golden( + GOLDEN_DD_ADMISSION, + (&[3, 0, 2], 310), + (&[5, 1], -208), + 7, + scalars, + ); let decoded = DdSketch::from_asapv1(&decode_hex(GOLDEN_DD_ADMISSION)).expect("decode"); assert_eq!(decoded.admission_rate(), 0.125); assert_eq!((decoded.get_count(), decoded.sum()), (18, 2523.90625)); diff --git a/tests/e2e/cardinality.rs b/tests/e2e/cardinality.rs index 73fd0656..84e36d16 100644 --- a/tests/e2e/cardinality.rs +++ b/tests/e2e/cardinality.rs @@ -746,9 +746,8 @@ fn a_set_aggregator_delta_describes_the_change_and_survives_the_wire() { added: HashSet::new(), removed: HashSet::new(), }; - let round_tripped = - SetAggregatorDelta::from_asapv1(&empty.to_asapv1().expect("encode empty")) - .expect("decode empty"); + let round_tripped = SetAggregatorDelta::from_asapv1(&empty.to_asapv1().expect("encode empty")) + .expect("decode empty"); assert!( round_tripped.added.is_empty() && round_tripped.removed.is_empty(), "an empty delta must stay empty across the wire" diff --git a/tests/e2e/composition.rs b/tests/e2e/composition.rs index f1b05cae..97a6f8ee 100644 --- a/tests/e2e/composition.rs +++ b/tests/e2e/composition.rs @@ -12,7 +12,7 @@ use crate::common; -use common::specs::{CardinalityConfidenceSpec, CountMinSpec, CountSketchSpec, KllRankSpec, Tally}; +use common::specs::{CardinalityConfidenceSpec, CountMinSpec, CountSketchSpec, Tally}; use common::{FreqTruth, NumericTruth, uniform_u64, zipf_u64}; use asap_sketchlib::{ diff --git a/tests/hll_custom_precision.rs b/tests/hll_custom_precision.rs index 697d4691..778b8fc0 100644 --- a/tests/hll_custom_precision.rs +++ b/tests/hll_custom_precision.rs @@ -166,8 +166,8 @@ fn large_precision_allocates_on_the_heap() { big.update(&DataInput::String(item(i))); } let bytes = big.to_asapv1().expect("serialize"); - let restored = HyperLogLogImpl::::from_asapv1(&bytes) - .expect("deserialize"); + let restored = + HyperLogLogImpl::::from_asapv1(&bytes).expect("deserialize"); assert_eq!(restored.registers_as_slice(), big.registers_as_slice()); } diff --git a/tests/pbt/count_min.rs b/tests/pbt/count_min.rs index c5fb821d..b4612dd7 100644 --- a/tests/pbt/count_min.rs +++ b/tests/pbt/count_min.rs @@ -8,9 +8,7 @@ //! The bound laws use an exact `HashMap` of the generated stream as the //! oracle, which is independent of the sketch's own bookkeeping. -use crate::support::{ - PROBE_KEYS, edge_dimension, edge_rows, grid, keys, -}; +use crate::support::{PROBE_KEYS, edge_dimension, edge_rows, grid, keys}; use asap_sketchlib::{ CountMinSketch, DataInput, DefaultMatrixI32, FastPath, MatrixFastHash, MatrixStorage, QuickMatrixI64, RegularPath, Vector2D, hash_for_matrix, diff --git a/tests/pbt/count_sketch.rs b/tests/pbt/count_sketch.rs index 0222bef6..cba6e754 100644 --- a/tests/pbt/count_sketch.rs +++ b/tests/pbt/count_sketch.rs @@ -2,9 +2,7 @@ //! //! Charikar, Chen, Farach-Colton, ICALP '02. -use crate::support::{ - PROBE_KEYS, edge_dimension, edge_rows, grid, keys, -}; +use crate::support::{PROBE_KEYS, edge_dimension, edge_rows, grid, keys}; use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; use proptest::prelude::*; diff --git a/tests/pbt/microscope.rs b/tests/pbt/microscope.rs index 4d1bfd90..be624220 100644 --- a/tests/pbt/microscope.rs +++ b/tests/pbt/microscope.rs @@ -17,11 +17,10 @@ //! `a_hot_key_saturates_a_pixel_and_forces_a_zoom` pins the generator to a //! load that zooms, and every law below re-asserts it on the run it drew. -use crate::support::close; use asap_sketchlib::sketches::microscope::{ DeltaStrategy, MicroLayout, MicroParams, SubWindowClock, }; -use asap_sketchlib::{DataInput, DefaultXxHasher, MatrixFastHash, MicroCm, SketchHasher, Vector3D}; +use asap_sketchlib::{DataInput, DefaultXxHasher, MatrixFastHash, MicroCm, SketchHasher}; use proptest::prelude::*; use std::collections::{HashMap, VecDeque};