From b033ba2b6f6a874d372e442a07464126718df976 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:59:39 -0600 Subject: [PATCH 1/3] refactor(coco)!: drop the proto format Delete proto/cocosketch/cocosketch.proto (CocoSketchState), reserve the SketchEnvelope oneof field 17 ("coco"), regenerate, and remove the Coco section of the cross-language proto consumer. Coco serializes only as ASAPv1. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +++ docs/features.md | 2 +- docs/tests.md | 2 +- proto/cocosketch/cocosketch.proto | 35 ------------- proto/sketchlib.proto | 5 +- src/proto/generated/sketchlib.v1.rs | 27 +--------- tests/xtest_consumer.rs | 80 ----------------------------- tools/gen-proto/src/main.rs | 1 - 8 files changed, 14 insertions(+), 146 deletions(-) delete mode 100644 proto/cocosketch/cocosketch.proto diff --git a/CHANGELOG.md b/CHANGELOG.md index 80d85f98..f35c5ce5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,14 @@ signals a backwards-compatible change. as an empty window like any other, and that window takes a place in `max_windows`. +### Removed + +- **Coco's protobuf format.** `proto/cocosketch/cocosketch.proto` and the + generated `asap_sketchlib::proto::sketchlib::CocoSketchState` are gone, and + the `SketchEnvelope` oneof field `coco` (17) is reserved, so + `sketch_envelope::SketchState::Coco` is gone too. Coco serializes only as + ASAPv1 (`Coco::serialize_to_bytes` / `deserialize_from_bytes`). + ### Fixed - `KLLDynamic` panicked on every quantile query once a NaN had entered the diff --git a/docs/features.md b/docs/features.md index 64f296a3..1d87ce7a 100644 --- a/docs/features.md +++ b/docs/features.md @@ -112,7 +112,7 @@ MessagePack (`rmp-serde`) support. **serde support** means the type derives `Ser | EHSketchList | Yes | Yes | | ExponentialHistogram | No | Yes | -Protobuf (prost): `.proto` definitions exist for CountMin, Count, HLL, DDSketch, KLL, Elastic, Coco, Hydra, and UnivMon. Rust conversion code is in progress. +Protobuf (prost): `.proto` definitions exist for CountMin, Count, HLL, DDSketch, KLL, Elastic, Hydra, and UnivMon. Rust conversion code is in progress. ### API Stability diff --git a/docs/tests.md b/docs/tests.md index ccb1571c..e577a012 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -2202,7 +2202,7 @@ Test file: [`tests/xtest_consumer.rs`](../tests/xtest_consumer.rs) | test_name | test_description | what_is_tested | | --- | --- | --- | -| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes nine `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `hll.pb` an ErtlMLE cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `coco.pb` `coco:hot` at least `500`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | +| `cross_language_proto` | Protobuf envelopes read from the cross-language fixture directory answer their point queries in band. | Asserts nothing unless fixtures are available: with no `XTEST_DIR` and no discoverable producer directory holding `tests/cross_language/xtest_producer_test.go`, it prints a skip line and returns. With fixtures it decodes eight `SketchEnvelope` files and records a pass or failure for each — `countmin.pb` min-frequency for `item:42` at least `101`, `kll.pb` p50 within 5% of `5000` and p99 within 5% of `9900`, `ddsketch.pb` the same two quantiles within 2%, `hll.pb` an ErtlMLE cardinality inside `[40_000, 65_000]`, `countsketch.pb` `cs:hot` at least `200`, `elastic.pb` `elephant` at least `900`, `univmon.pb` a cardinality inside `[1_000, 15_000]`, `hydra.pb` `hydra:42` at least `51` — using the hash seeds `0xcafe3553`, `0x6a09e667` and `0xbb67ae85`, and, when `countmin_sampled.pb` and `hll_sampled.pb` exist, requires `effective_sample_p` of `0.1` and a `rescale_count` by `1/p` within 5% of `100_000` and 6% of `200_000`. A single `assert!` at the end fails if any check failed; a missing or mistyped envelope panics on the spot. | ### HLL Custom Precision diff --git a/proto/cocosketch/cocosketch.proto b/proto/cocosketch/cocosketch.proto deleted file mode 100644 index 041b3c04..00000000 --- a/proto/cocosketch/cocosketch.proto +++ /dev/null @@ -1,35 +0,0 @@ -// cocosketch.proto — Portable wire format for CocoSketch flow-size estimator state. - -syntax = "proto3"; - -package sketchlib.v1; - -option go_package = "github.com/ProjectASAP/sketchlib-go/proto/cocosketch;cocosketchpb"; - -// ============================================================================ -// CocoSketch -// ============================================================================ - -// CocoSketchState is the portable state of a CocoSketch flow-size estimator. -// -// The bucket table is stored flat in row-major order (d rows × width cols). -// Element [r][c] is at index r * width + c. -message CocoSketchState { - // Number of hash rows. - uint32 d = 1; - - // Number of buckets per row. - uint32 width = 2; - - // Full hash stored in each bucket, length = d * width. - // Zero means the bucket is empty (has_keys[i] == false). - repeated uint64 hashes = 3 [packed = true]; - - // Cumulative count value per bucket, length = d * width. - repeated uint64 vals = 4 [packed = true]; - - // Whether each bucket holds a valid key entry, length = d * width. - repeated bool has_keys = 5 [packed = true]; - - reserved 6 to 15; -} diff --git a/proto/sketchlib.proto b/proto/sketchlib.proto index 0a6d8193..3bc37e2d 100644 --- a/proto/sketchlib.proto +++ b/proto/sketchlib.proto @@ -36,7 +36,6 @@ import "kll/kll.proto"; import "ddsketch/ddsketch.proto"; import "univmon/univmon.proto"; import "hydra/hydra.proto"; -import "cocosketch/cocosketch.proto"; import "elasticsketch/elasticsketch.proto"; option go_package = "github.com/ProjectASAP/sketchlib-go/proto/sketch_envelope;sketch_envelopepb"; @@ -73,10 +72,12 @@ message SketchEnvelope { DDSketchState ddsketch = 14; UnivMonState univmon = 15; HydraState hydra = 16; - CocoSketchState coco = 17; ElasticState elastic = 18; } + reserved 17; + reserved "coco"; + reserved 19 to 31; // Reserved for future sketch types. // Geometric / NitroSketch sampling probability for the producing sketch. diff --git a/src/proto/generated/sketchlib.v1.rs b/src/proto/generated/sketchlib.v1.rs index 2c098d3f..c1503e2f 100644 --- a/src/proto/generated/sketchlib.v1.rs +++ b/src/proto/generated/sketchlib.v1.rs @@ -800,29 +800,6 @@ impl HydraCounterType { } } } -/// CocoSketchState is the portable state of a CocoSketch flow-size estimator. -/// -/// The bucket table is stored flat in row-major order (d rows × width cols). -/// Element [r][c] is at index r * width + c. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CocoSketchState { - /// Number of hash rows. - #[prost(uint32, tag = "1")] - pub d: u32, - /// Number of buckets per row. - #[prost(uint32, tag = "2")] - pub width: u32, - /// Full hash stored in each bucket, length = d * width. - /// Zero means the bucket is empty (has_keys\[i\] == false). - #[prost(uint64, repeated, tag = "3")] - pub hashes: ::prost::alloc::vec::Vec, - /// Cumulative count value per bucket, length = d * width. - #[prost(uint64, repeated, tag = "4")] - pub vals: ::prost::alloc::vec::Vec, - /// Whether each bucket holds a valid key entry, length = d * width. - #[prost(bool, repeated, tag = "5")] - pub has_keys: ::prost::alloc::vec::Vec, -} /// ElasticState is the portable state of an Elastic Sketch. /// /// The Elastic Sketch has a heavy part (one bucket per index) and a light @@ -892,7 +869,7 @@ pub struct SketchEnvelope { /// The sketch payload. Exactly one field must be set. #[prost( oneof = "sketch_envelope::SketchState", - tags = "10, 11, 12, 13, 14, 15, 16, 17, 18" + tags = "10, 11, 12, 13, 14, 15, 16, 18" )] pub sketch_state: ::core::option::Option, } @@ -915,8 +892,6 @@ pub mod sketch_envelope { Univmon(super::UnivMonState), #[prost(message, tag = "16")] Hydra(super::HydraState), - #[prost(message, tag = "17")] - Coco(super::CocoSketchState), #[prost(message, tag = "18")] Elastic(super::ElasticState), } diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs index 0113ff45..c702eb3f 100644 --- a/tests/xtest_consumer.rs +++ b/tests/xtest_consumer.rs @@ -10,7 +10,6 @@ //! ddsketch.pb — DDSketchState with alpha + bucket array //! hll.pb — HyperLogLogState (ErtlMLE variant) //! countsketch.pb — CountSketchState with float64 signed counters -//! coco.pb — CocoSketchState (hash+val+hasKey buckets) //! elastic.pb — ElasticState (heavy buckets + light CountMin) //! univmon.pb — UnivMonState (layered CountSketch + TopK heaps) //! hydra.pb — HydraState (CM-cell grid) @@ -283,45 +282,6 @@ fn cross_language_proto() { all_ok = false; } - // ----------------------------------------------------------------------- - // CocoSketch - // ----------------------------------------------------------------------- - println!(); - println!("[CocoSketch] Step 1/3 — Read coco.pb"); - let bytes = read_file(in_dir.join("coco.pb")); - let env = SketchEnvelope::decode(bytes.as_slice()).expect("decode coco envelope"); - - println!( - "[CocoSketch] Step 2/3 — Validate envelope (format_version={}, producer={})", - env.format_version, - env.producer.as_ref().map_or("?", |p| &p.library) - ); - - let coco_state = match env.sketch_state { - Some(sketch_envelope::SketchState::Coco(ref s)) => s.clone(), - other => panic!("expected CocoSketch sketch_state, got {other:?}"), - }; - - println!( - "[CocoSketch] d={} width={} buckets={}", - coco_state.d, - coco_state.width, - coco_state.hashes.len() - ); - - // Query "coco:hot" — inserted with val=500. - // Go uses: hash = common.Hash64([]byte("coco:hot")) = xxh3_64_seeded(seedList[0], ...) - // DeriveIndex(hash, row, width): col = (hash >> (row * maskBitsForWidth(width))) & (width-1) - let coco_hash = xxh3_64_seeded(SEED_0, b"coco:hot"); - let coco_est = coco_estimate(&coco_state, coco_hash); - println!("[CocoSketch] Step 3/3 — 'coco:hot' est = {coco_est} (expect ≥ 500)"); - if coco_est >= 500 { - println!("[CocoSketch] PASS"); - } else { - eprintln!("[CocoSketch] FAIL: estimate {coco_est} < 500"); - all_ok = false; - } - // ----------------------------------------------------------------------- // ElasticSketch // ----------------------------------------------------------------------- @@ -698,21 +658,6 @@ fn col_bits(cols: usize) -> u64 { width.trailing_zeros() as u64 } -/// maskBitsForWidth — mirrors Go's common.maskBitsForWidth. -/// Returns the number of bits needed to represent (width-1). -fn mask_bits_for_width(width: usize) -> u64 { - if width <= 1 { - return 1; - } - let mut u = width - 1; - let mut bits = 0u64; - while u > 0 { - bits += 1; - u >>= 1; - } - bits -} - /// XXH3-64 with explicit seed, matching Go's `hash64_seeded(seed, key)`. fn xxh3_64_seeded(seed: u64, data: &[u8]) -> u64 { XxHash3_64::oneshot_with_seed(seed, data) @@ -835,31 +780,6 @@ fn median_f64(v: &mut [f64]) -> f64 { } } -// --------------------------------------------------------------------------- -// CocoSketch estimate -// --------------------------------------------------------------------------- -// Mirrors Go's CocoSketch.EstimateHash: -// for row i: col = DeriveIndex(hash, i, width); if b.HasKey && b.Hash==hash: total += b.Val -// DeriveIndex(hash, row, width) = (hash >> (row * maskBitsForWidth(width))) & (width-1) - -fn coco_estimate(state: &CocoSketchState, hash: u64) -> u64 { - let d = state.d as usize; - let width = state.width as usize; - let mbw = mask_bits_for_width(width); - let mask = (width as u64) - 1; - let mut total = 0u64; - - for i in 0..d { - let shift = (i as u64) * mbw; - let col = ((hash >> shift) & mask) as usize; - let idx = i * width + col; - if state.has_keys[idx] && state.hashes[idx] == hash { - total += state.vals[idx]; - } - } - total -} - // --------------------------------------------------------------------------- // ElasticSketch query // --------------------------------------------------------------------------- diff --git a/tools/gen-proto/src/main.rs b/tools/gen-proto/src/main.rs index 1de6ad15..7fb4d9b3 100644 --- a/tools/gen-proto/src/main.rs +++ b/tools/gen-proto/src/main.rs @@ -35,7 +35,6 @@ const PROTO_FILES: &[&str] = &[ "ddsketch/ddsketch.proto", "univmon/univmon.proto", "hydra/hydra.proto", - "cocosketch/cocosketch.proto", "elasticsketch/elasticsketch.proto", "sketchlib.proto", ]; From fde0256836f34ec500cd823c9c61a9a31ff0d61a Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 00:59:39 -0600 Subject: [PATCH 2/3] test(coco): add an ASAPv1 golden fixture Bump asapv1_golden to the commit adding coco_3x8.hex, a 3x8 bucket table set directly: values at both ends of every msgpack uint width, empty, 31-byte, 32-byte and multi-byte UTF-8 keys, and unoccupied buckets. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 ++ asapv1_golden | 2 +- docs/asapv1_wire_format.md | 2 +- docs/message_pack_format.md | 4 +-- docs/tests.md | 1 + tests/asapv1_golden.rs | 64 +++++++++++++++++++++++++++++++++++-- 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f35c5ce5..898d5544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,8 @@ signals a backwards-compatible change. worker's seed and the aggregator's from one seed. The unseeded constructors draw from the thread generator. The seed is not serialized, and the ASAPv1 payload is unchanged. +- **ASAPv1 golden fixture for Coco** (`0c 00`): `asapv1_golden/coco_3x8.hex`, + a 3x8 bucket table set directly, checked by `tests/asapv1_golden.rs`. ### Changed diff --git a/asapv1_golden b/asapv1_golden index 8287a4ae..f9bbc3d7 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit 8287a4ae04e6cfbfb3ed33dafbf1dac77d8f7fc1 +Subproject commit f9bbc3d7f7a093e046d5389f9d3afc26b8c5285d diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index fd14e70a..7f1853c7 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -1341,7 +1341,7 @@ Good direction (more compact, higher fidelity, less Rust-internal duplication), 2. **Golden byte-vector fixtures** in one shared repo, [`sketchlib-golden-bytes`](https://github.com/ProjectASAP/sketchlib-golden-bytes), which each implementation mounts at `asapv1_golden/`; both languages decode and re-encode them byte-identically. These replace the `portable`-as-oracle round-trip test. 3. **This registry**, mirrored, never independently allocated. -Fixtures exist for six `kind_id`s — HLL's three estimators, Count-Min, Count Sketch and compact KLL — and `sketchlib-go` mirrors those. Every other kind here has a payload and **no fixture**, so this document is its only contract; `asapv1_golden/README.md` lists the gap. `portable` carries its own Go goldens. +Fixtures exist for seven `kind_id`s — HLL's three estimators, Count-Min, Count Sketch, compact KLL and Coco. Every other kind here has a payload and **no fixture**, so this document is its only contract; `asapv1_golden/README.md` lists the gap. `portable` carries its own Go goldens. **Hash profile on the Go side.** Rust derives the hash spec from a generic `HashProfile` bound on the hasher type; Go has no generic hasher type, so there is nothing to derive from. diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index 12a2b91c..ae238e7e 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -169,8 +169,8 @@ decode → re-encode them byte-identically. The `kind_id` registry is mirrored verbatim with Go's `wire/asapmsgpack/magic_ids.go`, never independently allocated. -That proof covers the six `kind_id`s a fixture exists for — HLL's three -estimators, Count-Min, Count Sketch and compact KLL. Every other implemented +That proof covers the seven `kind_id`s a fixture exists for — HLL's three +estimators, Count-Min, Count Sketch, compact KLL and Coco. Every other implemented kind has **no ASAPv1 golden and therefore no ASAPv1 drift guard**; [its README](https://github.com/ProjectASAP/sketchlib-golden-bytes#coverage) lists what is covered. The older `portable` path is guarded separately, by the `sketchlib-go` goldens diff --git a/docs/tests.md b/docs/tests.md index e577a012..26dbd0de 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -2123,6 +2123,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | --- | --- | --- | | `cms_f64_fast_2x3_matches_golden` | A known f64 fast-path Count-Min matrix serializes to its checked-in golden bytes. | Verifies `CountMin::, FastPath>::from_storage` over the `2x3` matrix `[[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]` serializes to `asapv1_golden/cms_f64_fast_2x3.hex`, and that decoding that fixture recovers the six cells in row-major order and re-serializes to the same bytes. | | `cms_i64_regular_2x3_matches_golden` | A known i64 regular-path Count-Min matrix serializes to its checked-in golden bytes. | Verifies `CountMin::, RegularPath>::from_storage` over the `2x3` matrix `[[0, 1, 127], [128, 300, 65536]]` serializes to `asapv1_golden/cms_i64_regular_2x3.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `3`, the six cells in row-major order, and a byte-identical re-serialization. | +| `coco_3x8_matches_golden` | A known Coco bucket table serializes to its checked-in golden bytes. | Sets 14 of the 24 buckets of `Coco::init_with_size(8, 3)` directly — each key in the column its row hashes it to, values spanning positive fixint through uint64 at both ends of each width, keys including `""`, a 31-byte and a 32-byte key and a multi-byte UTF-8 key, and `"zero"` holding `0` — verifies it serializes to `asapv1_golden/coco_3x8.hex`, and that decoding the fixture reports `d` `3`, `w` `8`, all 24 buckets in row-major order with the other ten unoccupied, and a byte-identical re-serialization. | | `cs_i32_regular_2x4_matches_golden` | The i32 counter type reaches the metadata, and an i32 golden is not an i64 sketch. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` cast to `i32` serializes to `asapv1_golden/cs_i32_regular_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, that the i32 and i64 regular fixtures have equal length yet differ, and that the i32 fixture fails to decode as `Count, RegularPath>`. | | `cs_i64_fast_2x4_matches_golden` | The fast-path mode reaches the bytes, so the same matrix encodes differently per path. | Verifies `Count::, FastPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` serializes to `asapv1_golden/cs_i64_fast_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, and that the fixture differs from `cs_i64_regular_2x4.hex` built from the same matrix. | | `cs_i64_regular_2x4_matches_golden` | A known signed Count-Sketch matrix serializes to its checked-in golden bytes. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` — sweeping the msgpack integer widths in both directions — serializes to `asapv1_golden/cs_i64_regular_2x4.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `4`, the eight cells, and a byte-identical re-serialization. | diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 30b41651..69f9e307 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -9,8 +9,8 @@ //! See `asapv1_golden/README.md`. use asap_sketchlib::{ - Classic, Count, CountMin, ErtlMLE, FastPath, HllSketch, HllVariant, HyperLogLogHIPP12, - HyperLogLogP12, KLL, MessagePackCodec, RegularPath, Vector2D, + Classic, Coco, CocoBucket, Count, CountMin, ErtlMLE, FastPath, HllSketch, HllVariant, + HyperLogLogHIPP12, HyperLogLogP12, KLL, MessagePackCodec, RegularPath, Vector2D, }; fn decode_hex(s: &str) -> Vec { @@ -32,6 +32,7 @@ const GOLDEN_CS_FAST: &str = include_str!("../asapv1_golden/cs_i64_fast_2x4.hex" const GOLDEN_CS_I32: &str = include_str!("../asapv1_golden/cs_i32_regular_2x4.hex"); const GOLDEN_KLL_F64: &str = include_str!("../asapv1_golden/kll_f64_k200.hex"); const GOLDEN_KLL_I64: &str = include_str!("../asapv1_golden/kll_i64_k200.hex"); +const GOLDEN_COCO: &str = include_str!("../asapv1_golden/coco_3x8.hex"); /// The known P12 register pattern shared by all three HLL fixtures. fn p12_registers() -> Vec { @@ -287,3 +288,62 @@ fn kll_i64_k200_matches_golden() { assert_eq!(decoded.quantile(0.0), 1.0); assert_eq!(decoded.quantile(1.0), 50.0); } + +// --------------------------------------------------------------------------- +// Coco: build a known bucket table -> serialize == golden, round-trips. +// --------------------------------------------------------------------------- + +/// The occupied buckets of the 3x8 Coco fixture as `(row, col, key, value)`; +/// every other bucket is unoccupied. Each key sits in the column its row hashes +/// it to, which the codec checks on both sides. +const COCO_CELLS: [(usize, usize, &str, u64); 14] = [ + (0, 0, "uint16-max", 65535), + (0, 1, "fixint-max", 127), + (0, 2, "clé-ünïcode-流量", 4), + (0, 4, "fixstr-max-31-bytes-0123456789a", 2), + (0, 5, "uint8-min", 128), + (0, 6, "uint64-max", u64::MAX), + (0, 7, "", 1), + (1, 0, "uint64-min", 4294967296), + (1, 1, "uint32-min", 65536), + (1, 3, "str8-min-32-bytes-0123456789abcd", 3), + (1, 5, "zero", 0), + (1, 6, "uint32-max", 4294967295), + (1, 7, "uint8-max", 255), + (2, 5, "uint16-min", 256), +]; + +/// The fixture's 24 buckets in row-major order. +fn coco_buckets() -> Vec<(Option, u64)> { + let mut buckets = vec![(None, 0); 24]; + for (r, c, key, val) in COCO_CELLS { + buckets[r * 8 + c] = (Some(key.to_string()), val); + } + buckets +} + +#[test] +fn coco_3x8_matches_golden() { + let want = decode_hex(GOLDEN_COCO); + + let mut sketch: Coco = Coco::init_with_size(8, 3); + for (r, c, key, val) in COCO_CELLS { + sketch.table[r][c] = CocoBucket { + full_key: Some(key.to_string()), + val, + }; + } + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "Coco bytes diverge from golden"); + + let decoded: Coco = Coco::deserialize_from_bytes(&want).expect("decode"); + assert_eq!((decoded.d, decoded.w), (3, 8)); + let cells: Vec<(Option, u64)> = decoded + .table + .as_slice() + .iter() + .map(|b| (b.full_key.clone(), b.val)) + .collect(); + assert_eq!(cells, coco_buckets()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} From 1d5886de04652e463bd33eb36231cd6752937b68 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Fri, 2 Oct 2026 01:06:28 -0600 Subject: [PATCH 3/3] test(coco): replace the golden with a 3x7 table; defer fixture counts to the golden README A width that is not a power of two separates the spec's % cols fold from a mask fold. The fixture adds a 4-byte UTF-8 key. Docs no longer state how many kind_ids have fixtures; they point at the golden README's Coverage section. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +-- asapv1_golden | 2 +- docs/asapv1_wire_format.md | 4 +-- docs/message_pack_format.md | 7 +++--- docs/tests.md | 2 +- tests/asapv1_golden.rs | 49 +++++++++++++++++++------------------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 898d5544..e084db81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,8 +66,8 @@ signals a backwards-compatible change. worker's seed and the aggregator's from one seed. The unseeded constructors draw from the thread generator. The seed is not serialized, and the ASAPv1 payload is unchanged. -- **ASAPv1 golden fixture for Coco** (`0c 00`): `asapv1_golden/coco_3x8.hex`, - a 3x8 bucket table set directly, checked by `tests/asapv1_golden.rs`. +- **ASAPv1 golden fixture for Coco** (`0c 00`): `asapv1_golden/coco_3x7.hex`, + a 3x7 bucket table set directly, checked by `tests/asapv1_golden.rs`. ### Changed diff --git a/asapv1_golden b/asapv1_golden index f9bbc3d7..e60f34b1 160000 --- a/asapv1_golden +++ b/asapv1_golden @@ -1 +1 @@ -Subproject commit f9bbc3d7f7a093e046d5389f9d3afc26b8c5285d +Subproject commit e60f34b1521a9b6550f9062275f3a60877329318 diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md index 7f1853c7..62d45d5a 100644 --- a/docs/asapv1_wire_format.md +++ b/docs/asapv1_wire_format.md @@ -1341,14 +1341,14 @@ Good direction (more compact, higher fidelity, less Rust-internal duplication), 2. **Golden byte-vector fixtures** in one shared repo, [`sketchlib-golden-bytes`](https://github.com/ProjectASAP/sketchlib-golden-bytes), which each implementation mounts at `asapv1_golden/`; both languages decode and re-encode them byte-identically. These replace the `portable`-as-oracle round-trip test. 3. **This registry**, mirrored, never independently allocated. -Fixtures exist for seven `kind_id`s — HLL's three estimators, Count-Min, Count Sketch, compact KLL and Coco. Every other kind here has a payload and **no fixture**, so this document is its only contract; `asapv1_golden/README.md` lists the gap. `portable` carries its own Go goldens. +`asapv1_golden/README.md`'s Coverage section lists which `kind_id`s have fixtures; for every other kind this document is the only contract. `portable` carries its own Go goldens. **Hash profile on the Go side.** Rust derives the hash spec from a generic `HashProfile` bound on the hasher type; Go has no generic hasher type, so there is nothing to derive from. On the Go side the profile is simply **written into** the metadata on encode and **read from** it on decode. Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketch is only mergeable/queryable if its `hash_profile_id` + seeds match the profile Go is prepared to reproduce. -Sequencing: (2) covers six `kind_id`s, and the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. +Sequencing: (2) covers the `kind_id`s `asapv1_golden/README.md`'s Coverage section lists, and the one `native bytes == portable bytes` test is HLL-only, since no other `portable` type emits ASAPv1 bytes. Retire `portable` once the fixtures cover the rest. --- diff --git a/docs/message_pack_format.md b/docs/message_pack_format.md index ae238e7e..108f133f 100644 --- a/docs/message_pack_format.md +++ b/docs/message_pack_format.md @@ -169,10 +169,9 @@ decode → re-encode them byte-identically. The `kind_id` registry is mirrored verbatim with Go's `wire/asapmsgpack/magic_ids.go`, never independently allocated. -That proof covers the seven `kind_id`s a fixture exists for — HLL's three -estimators, Count-Min, Count Sketch, compact KLL and Coco. Every other implemented -kind has **no ASAPv1 golden and therefore no ASAPv1 drift guard**; -[its README](https://github.com/ProjectASAP/sketchlib-golden-bytes#coverage) lists what is covered. +That proof covers only the `kind_id`s that have a fixture; +[its README's Coverage section](https://github.com/ProjectASAP/sketchlib-golden-bytes#coverage) +lists which. A kind without one has **no ASAPv1 drift guard**. The older `portable` path is guarded separately, by the `sketchlib-go` goldens of its own listed above. diff --git a/docs/tests.md b/docs/tests.md index 26dbd0de..88ac8982 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -2123,7 +2123,7 @@ Test file: [`tests/asapv1_golden.rs`](../tests/asapv1_golden.rs) | --- | --- | --- | | `cms_f64_fast_2x3_matches_golden` | A known f64 fast-path Count-Min matrix serializes to its checked-in golden bytes. | Verifies `CountMin::, FastPath>::from_storage` over the `2x3` matrix `[[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]` serializes to `asapv1_golden/cms_f64_fast_2x3.hex`, and that decoding that fixture recovers the six cells in row-major order and re-serializes to the same bytes. | | `cms_i64_regular_2x3_matches_golden` | A known i64 regular-path Count-Min matrix serializes to its checked-in golden bytes. | Verifies `CountMin::, RegularPath>::from_storage` over the `2x3` matrix `[[0, 1, 127], [128, 300, 65536]]` serializes to `asapv1_golden/cms_i64_regular_2x3.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `3`, the six cells in row-major order, and a byte-identical re-serialization. | -| `coco_3x8_matches_golden` | A known Coco bucket table serializes to its checked-in golden bytes. | Sets 14 of the 24 buckets of `Coco::init_with_size(8, 3)` directly — each key in the column its row hashes it to, values spanning positive fixint through uint64 at both ends of each width, keys including `""`, a 31-byte and a 32-byte key and a multi-byte UTF-8 key, and `"zero"` holding `0` — verifies it serializes to `asapv1_golden/coco_3x8.hex`, and that decoding the fixture reports `d` `3`, `w` `8`, all 24 buckets in row-major order with the other ten unoccupied, and a byte-identical re-serialization. | +| `coco_3x7_matches_golden` | A known Coco bucket table serializes to its checked-in golden bytes. | Sets 15 of the 21 buckets of `Coco::init_with_size(7, 3)` directly — each key in the column its row hashes it to folded `% 7`, a width that is not a power of two so a mask fold misplaces the keys; values spanning positive fixint through uint64 at both ends of each width; keys including `""`, a 31-byte and a 32-byte key and UTF-8 code points of two, three and four bytes; and `"zero"` holding `0` — verifies it serializes to `asapv1_golden/coco_3x7.hex`, and that decoding the fixture reports `d` `3`, `w` `7`, all 21 buckets in row-major order with the other six unoccupied, and a byte-identical re-serialization. | | `cs_i32_regular_2x4_matches_golden` | The i32 counter type reaches the metadata, and an i32 golden is not an i64 sketch. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` cast to `i32` serializes to `asapv1_golden/cs_i32_regular_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, that the i32 and i64 regular fixtures have equal length yet differ, and that the i32 fixture fails to decode as `Count, RegularPath>`. | | `cs_i64_fast_2x4_matches_golden` | The fast-path mode reaches the bytes, so the same matrix encodes differently per path. | Verifies `Count::, FastPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` serializes to `asapv1_golden/cs_i64_fast_2x4.hex`, that decoding it recovers the eight cells and re-serializes identically, and that the fixture differs from `cs_i64_regular_2x4.hex` built from the same matrix. | | `cs_i64_regular_2x4_matches_golden` | A known signed Count-Sketch matrix serializes to its checked-in golden bytes. | Verifies `Count::, RegularPath>::from_storage` over `[[0, 127, 128, 65536], [-1, -33, -32768, -2147483648]]` — sweeping the msgpack integer widths in both directions — serializes to `asapv1_golden/cs_i64_regular_2x4.hex`, and that decoding the fixture reports `rows()` `2`, `cols()` `4`, the eight cells, and a byte-identical re-serialization. | diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs index 69f9e307..31e3cb36 100644 --- a/tests/asapv1_golden.rs +++ b/tests/asapv1_golden.rs @@ -32,7 +32,7 @@ const GOLDEN_CS_FAST: &str = include_str!("../asapv1_golden/cs_i64_fast_2x4.hex" const GOLDEN_CS_I32: &str = include_str!("../asapv1_golden/cs_i32_regular_2x4.hex"); const GOLDEN_KLL_F64: &str = include_str!("../asapv1_golden/kll_f64_k200.hex"); const GOLDEN_KLL_I64: &str = include_str!("../asapv1_golden/kll_i64_k200.hex"); -const GOLDEN_COCO: &str = include_str!("../asapv1_golden/coco_3x8.hex"); +const GOLDEN_COCO: &str = include_str!("../asapv1_golden/coco_3x7.hex"); /// The known P12 register pattern shared by all three HLL fixtures. fn p12_registers() -> Vec { @@ -293,40 +293,41 @@ fn kll_i64_k200_matches_golden() { // Coco: build a known bucket table -> serialize == golden, round-trips. // --------------------------------------------------------------------------- -/// The occupied buckets of the 3x8 Coco fixture as `(row, col, key, value)`; +/// The occupied buckets of the 3x7 Coco fixture as `(row, col, key, value)`; /// every other bucket is unoccupied. Each key sits in the column its row hashes -/// it to, which the codec checks on both sides. -const COCO_CELLS: [(usize, usize, &str, u64); 14] = [ - (0, 0, "uint16-max", 65535), - (0, 1, "fixint-max", 127), - (0, 2, "clé-ünïcode-流量", 4), - (0, 4, "fixstr-max-31-bytes-0123456789a", 2), - (0, 5, "uint8-min", 128), - (0, 6, "uint64-max", u64::MAX), - (0, 7, "", 1), - (1, 0, "uint64-min", 4294967296), - (1, 1, "uint32-min", 65536), - (1, 3, "str8-min-32-bytes-0123456789abcd", 3), - (1, 5, "zero", 0), - (1, 6, "uint32-max", 4294967295), - (1, 7, "uint8-max", 255), - (2, 5, "uint16-min", 256), +/// it to, folded `% 7`, which the codec checks on both sides. +const COCO_CELLS: [(usize, usize, &str, u64); 15] = [ + (0, 0, "uint32-min", 65536), + (0, 1, "uint16-max", 65535), + (0, 2, "fixint-max", 127), + (0, 3, "", 1), + (0, 4, "uint8-max", 255), + (0, 5, "emoji-😀", 5), + (1, 0, "uint16-min", 256), + (1, 1, "str8-min-32-bytes-0123456789abcd", 3), + (1, 2, "uint8-min", 128), + (1, 3, "clé-ünïcode-流量", 4), + (1, 4, "fixstr-max-31-bytes-0123456789a", 2), + (1, 5, "uint64-min", 4294967296), + (1, 6, "zero", 0), + (2, 1, "uint64-max", u64::MAX), + (2, 5, "uint32-max", 4294967295), ]; -/// The fixture's 24 buckets in row-major order. +/// The fixture's 21 buckets in row-major order. fn coco_buckets() -> Vec<(Option, u64)> { - let mut buckets = vec![(None, 0); 24]; + let mut buckets = vec![(None, 0); 21]; for (r, c, key, val) in COCO_CELLS { - buckets[r * 8 + c] = (Some(key.to_string()), val); + buckets[r * 7 + c] = (Some(key.to_string()), val); } buckets } #[test] -fn coco_3x8_matches_golden() { +fn coco_3x7_matches_golden() { let want = decode_hex(GOLDEN_COCO); - let mut sketch: Coco = Coco::init_with_size(8, 3); + let mut sketch: Coco = Coco::init_with_size(7, 3); for (r, c, key, val) in COCO_CELLS { sketch.table[r][c] = CocoBucket { full_key: Some(key.to_string()), @@ -337,7 +338,7 @@ fn coco_3x8_matches_golden() { assert_eq!(got, want, "Coco bytes diverge from golden"); let decoded: Coco = Coco::deserialize_from_bytes(&want).expect("decode"); - assert_eq!((decoded.d, decoded.w), (3, 8)); + assert_eq!((decoded.d, decoded.w), (3, 7)); let cells: Vec<(Option, u64)> = decoded .table .as_slice()