diff --git a/apps/web/app/benchmark.tsx b/apps/web/app/benchmark.tsx index 5b6baea..6eae8ea 100644 --- a/apps/web/app/benchmark.tsx +++ b/apps/web/app/benchmark.tsx @@ -25,9 +25,9 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 2503, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 2512, kind: "generic" }, { label: "Protobuf", bytes: 7190, kind: "binary" }, - { label: "Hyperfly", bytes: 2109, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 2054, kind: "profile" }, - { label: "Hyperfly Profiled", bytes: 823, kind: "full" }, + { label: "Hyperfly", bytes: 2083, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 2030, kind: "profile" }, + { label: "Hyperfly Profiled", bytes: 795, kind: "full" }, ], note: "An audit log repeats itself across requests, not within one: the same user agents, the same actor emails, the same resource ids, request after request. A compressor only ever sees one response and has to rediscover them every time — which is why Brotli takes 55 bytes off and the profile takes 1 231. It learned 692 values once, and pays for itself after ten requests.", }, @@ -39,9 +39,9 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 1473, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 1422, kind: "generic" }, { label: "Protobuf", bytes: 2007, kind: "binary" }, - { label: "Hyperfly", bytes: 896, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 818, kind: "profile" }, - { label: "Hyperfly Profiled", bytes: 638, kind: "full" }, + { label: "Hyperfly", bytes: 807, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 753, kind: "profile" }, + { label: "Hyperfly Profiled", bytes: 576, kind: "full" }, ], note: "An enum with six members is an index, not a string. Bounded integers ship as offsets from their declared minimum and booleans pack into bitmaps — that is the first row, before anything has been compressed or learned. The profile then learns the fleet: the device ids that recur on every page.", }, @@ -55,7 +55,7 @@ const PAYLOADS: Payload[] = [ { label: "Protobuf", bytes: 388, kind: "binary" }, { label: "Hyperfly", bytes: 271, kind: "hyperfly" }, { label: "Hyperfly + Brotli", bytes: 273, kind: "profile" }, - { label: "Hyperfly Profiled", bytes: 188, kind: "full" }, + { label: "Hyperfly Profiled", bytes: 187, kind: "full" }, ], note: "The single-entity response, and the case a general compressor handles worst: under a kilobyte there is nothing yet to build a window from. Brotli actually costs two bytes here rather than saving any — at this size its framing outweighs what it finds. What does work is knowing the catalogue in advance.", }, @@ -67,9 +67,9 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 2307, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 2294, kind: "generic" }, { label: "Protobuf", bytes: 4396, kind: "binary" }, - { label: "Hyperfly", bytes: 1908, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 1902, kind: "profile" }, - { label: "Hyperfly Profiled", bytes: 1535, kind: "full" }, + { label: "Hyperfly", bytes: 1882, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 1877, kind: "profile" }, + { label: "Hyperfly Profiled", bytes: 1511, kind: "full" }, ], note: "Prose is the hard case: the bodies are genuinely new every time and nothing can invent redundancy that is not there. What does recur are the authors, so that is what the profile takes. This is the narrowest margin on the page, and it is the honest one to look at first.", }, @@ -81,11 +81,11 @@ const PAYLOADS: Payload[] = [ { label: "JSON + gzip", bytes: 928, kind: "generic" }, { label: "JSON + Brotli — edge q4", bytes: 842, kind: "generic" }, { label: "Protobuf", bytes: 2034, kind: "binary" }, - { label: "Hyperfly", bytes: 496, kind: "hyperfly" }, - { label: "Hyperfly + Brotli", bytes: 372, kind: "profile" }, - { label: "Hyperfly Profiled", bytes: 372, kind: "full" }, + { label: "Hyperfly", bytes: 384, kind: "hyperfly" }, + { label: "Hyperfly + Brotli", bytes: 362, kind: "profile" }, + { label: "Hyperfly Profiled", bytes: 362, kind: "full" }, ], - note: "Timestamps become deltas and exact-decimal prices travel as integer mantissas rather than eight raw bytes. The last two rows are identical to the byte, and the row is left in to show it: this route's only string sits outside the array, so there is no column for a dictionary to key on and training buys nothing at all.", + note: "Timestamps arrive at a constant stride, so the differences between them are identical and pack to a width of zero bits — the column carries its first value and nothing else. Exact-decimal prices travel as integer mantissas bit-packed to the span actually present. The last two rows are identical to the byte, and left in to show it: this route's only string sits outside the array, so training buys nothing at all.", }, ]; diff --git a/packages/hyperfly/src/canonical.ts b/packages/hyperfly/src/canonical.ts index 21e1e0e..0e5dc30 100644 --- a/packages/hyperfly/src/canonical.ts +++ b/packages/hyperfly/src/canonical.ts @@ -60,7 +60,7 @@ export function serializeNode(node: IRNode): string { export type PlanLayout = "row" | "columnar"; -const PLAN_VERSION: Record = { row: 1, columnar: 3 }; +const PLAN_VERSION: Record = { row: 1, columnar: 4 }; export function serializeShared(shared: SharedProfile): string { const columns = shared.columns.map( diff --git a/packages/hyperfly/src/columnar.ts b/packages/hyperfly/src/columnar.ts index 1169fae..0624642 100644 --- a/packages/hyperfly/src/columnar.ts +++ b/packages/hyperfly/src/columnar.ts @@ -59,6 +59,67 @@ function bitsToFloat(bits: bigint): number { const NEG_ZERO_BITS = 0x8000000000000000n; +const MAX_WIDTH = 56; + +/** Bits needed for an unsigned value; zero for zero, so a constant column packs to nothing. */ +function bitWidth(max: bigint): number { + let w = 0; + let v = max; + while (v > 0n) { + v >>= 1n; + w++; + } + return w; +} + +function packedBytes(count: number, width: number): number { + return Math.ceil((count * width) / 8); +} + +/** Spec §3.1: little-endian bit stream, value i at bits [i*w, (i+1)*w). */ +function packBits(w: Writer, values: readonly bigint[], width: number): void { + if (width === 0) return; + let acc = 0n; + let bits = 0; + for (const value of values) { + acc |= value << BigInt(bits); + bits += width; + while (bits >= 8) { + w.u8(Number(acc & 0xffn)); + acc >>= 8n; + bits -= 8; + } + } + if (bits > 0) w.u8(Number(acc & 0xffn)); +} + +function unpackBits(r: Reader, count: number, width: number, path: string): bigint[] { + if (width > MAX_WIDTH) { + throw new DecodeError("marker", `${path}: bit width ${width} exceeds ${MAX_WIDTH}`); + } + if (width === 0) return new Array(count).fill(0n); + const bytes = r.bytes(packedBytes(count, width)); + const mask = (1n << BigInt(width)) - 1n; + const out: bigint[] = new Array(count); + let acc = 0n; + let bits = 0; + let index = 0; + for (let i = 0; i < count; i++) { + while (bits < width) { + acc |= BigInt(bytes[index++] ?? 0) << BigInt(bits); + bits += 8; + } + out[i] = acc & mask; + acc >>= BigInt(width); + bits -= width; + } + // leftover bits are padding and must be zero, or one value would have two encodings + if (acc !== 0n) { + throw new DecodeError("bitmap", `${path}: nonzero bit-packing padding`); + } + return out; +} + function intForm(node: IntNode, value: number): bigint { return node.min !== undefined ? BigInt(value) - BigInt(node.min) : zigzag(BigInt(value)); } @@ -81,26 +142,64 @@ function encodeIntColumn(w: Writer, node: IntNode, values: number[]): void { w.u8(0); return; } + const exact = values.map((v) => BigInt(v)); const forms = values.map((v) => intForm(node, v)); const diffs: bigint[] = []; - for (let i = 1; i < values.length; i++) { - diffs.push(zigzag(BigInt(values[i]!) - BigInt(values[i - 1]!))); - } + for (let i = 1; i < exact.length; i++) diffs.push(exact[i]! - exact[i - 1]!); + const rawCost = forms.reduce((n, f) => n + ulebLen(f), 0); - const deltaCost = ulebLen(forms[0]!) + diffs.reduce((n, d) => n + ulebLen(d), 0); - if (deltaCost < rawCost) { - w.u8(1); - writeUleb(w, forms[0]!); - for (const d of diffs) writeUleb(w, d); - } else { - w.u8(0); + const deltaCost = ulebLen(forms[0]!) + diffs.reduce((n, d) => n + ulebLen(zigzag(d)), 0); + + // frame of reference: subtract the column minimum, then spend only the bits the + // remaining span needs rather than a whole number of bytes per value + const forBase = exact.reduce((m, v) => (v < m ? v : m), exact[0]!); + const forWidth = bitWidth(exact.reduce((m, v) => (v - forBase > m ? v - forBase : m), 0n)); + const forCost = + forWidth > MAX_WIDTH + ? Infinity + : ulebLen(zigzag(forBase)) + 1 + packedBytes(exact.length, forWidth); + + let deltaForCost = Infinity; + let deltaBase = 0n; + let deltaWidth = 0; + if (diffs.length > 0) { + deltaBase = diffs.reduce((m, d) => (d < m ? d : m), diffs[0]!); + deltaWidth = bitWidth(diffs.reduce((m, d) => (d - deltaBase > m ? d - deltaBase : m), 0n)); + if (deltaWidth <= MAX_WIDTH) { + deltaForCost = + ulebLen(forms[0]!) + ulebLen(zigzag(deltaBase)) + 1 + packedBytes(diffs.length, deltaWidth); + } + } + + const best = Math.min(rawCost, deltaCost, forCost, deltaForCost); + if (best === rawCost) { + w.u8(0x00); for (const f of forms) writeUleb(w, f); + return; + } + if (best === deltaCost) { + w.u8(0x01); + writeUleb(w, forms[0]!); + for (const d of diffs) writeUleb(w, zigzag(d)); + return; + } + if (best === forCost) { + w.u8(0x02); + writeUleb(w, zigzag(forBase)); + w.u8(forWidth); + packBits(w, exact.map((v) => v - forBase), forWidth); + return; } + w.u8(0x03); + writeUleb(w, forms[0]!); + writeUleb(w, zigzag(deltaBase)); + w.u8(deltaWidth); + packBits(w, diffs.map((d) => d - deltaBase), deltaWidth); } function decodeIntColumn(r: Reader, node: IntNode, count: number, path: string): number[] { const mode = r.u8(); - if (mode > 1) throw new DecodeError("marker", `${path}: invalid int column mode 0x${mode.toString(16)}`); + if (mode > 3) throw new DecodeError("marker", `${path}: invalid int column mode 0x${mode.toString(16)}`); const out: number[] = new Array(count); if (count === 0) { if (mode !== 0) throw new DecodeError("marker", `${path}: empty column must use mode 0x00`); @@ -124,10 +223,33 @@ function decodeIntColumn(r: Reader, node: IntNode, count: number, path: string): return num; }; - if (mode === 0) { + if (mode === 0x00) { for (let i = 0; i < count; i++) out[i] = validate(fromForm(readUleb(r)), i); return out; } + + if (mode === 0x02) { + const base = unzigzag(readUleb(r)); + const width = r.u8(); + const packed = unpackBits(r, count, width, path); + for (let i = 0; i < count; i++) out[i] = validate(base + packed[i]!, i); + return out; + } + + if (mode === 0x03) { + const first = fromForm(readUleb(r)); + const base = unzigzag(readUleb(r)); + const width = r.u8(); + const packed = unpackBits(r, Math.max(0, count - 1), width, path); + let running = first; + out[0] = validate(running, 0); + for (let i = 1; i < count; i++) { + running = running + base + packed[i - 1]!; + out[i] = validate(running, i); + } + return out; + } + let prev = fromForm(readUleb(r)); out[0] = validate(prev, 0); for (let i = 1; i < count; i++) { diff --git a/packages/hyperfly/test/zod.test.ts b/packages/hyperfly/test/zod.test.ts index 957496a..31933b0 100644 --- a/packages/hyperfly/test/zod.test.ts +++ b/packages/hyperfly/test/zod.test.ts @@ -156,7 +156,7 @@ describe("cross-adapter parity (retro)", () => { }); // identical string is asserted in python/tests/test_cross_adapter.py expect(serializeArtifact(toIR(Row), "columnar")).toBe( - '{"wire":1,"plan":{"layout":"columnar","version":3},"ir":' + + '{"wire":1,"plan":{"layout":"columnar","version":4},"ir":' + '{"kind":"struct","fields":[' + '{"name":"id","type":{"kind":"string"}},' + '{"name":"kind","type":{"kind":"literal","value":"a"}},' + diff --git a/python/src/hyperfly/_codec.py b/python/src/hyperfly/_codec.py index cf33520..53d2f1b 100644 --- a/python/src/hyperfly/_codec.py +++ b/python/src/hyperfly/_codec.py @@ -105,6 +105,60 @@ def _literal_matches(lit: Any, v: Any) -> bool: return type(v) is str and v == lit +_MAX_WIDTH = 56 + + +def _bit_width(max_value: int) -> int: + """Bits needed for an unsigned value; zero for zero, so a constant column packs to nothing.""" + return max_value.bit_length() + + +def _packed_bytes(count: int, width: int) -> int: + return -(-count * width // 8) if width else 0 + + +def _pack_bits(out: bytearray, values: list[int], width: int) -> None: + """Spec 3.1: little-endian bit stream, value i at bits [i*w, (i+1)*w).""" + if width == 0: + return + acc = 0 + bits = 0 + for value in values: + acc |= value << bits + bits += width + while bits >= 8: + out.append(acc & 0xFF) + acc >>= 8 + bits -= 8 + if bits: + out.append(acc & 0xFF) + + +def _unpack_bits(r: Reader, count: int, width: int, path: str) -> list[int]: + if width > _MAX_WIDTH: + _dfail("marker", path, f"bit width {width} exceeds {_MAX_WIDTH}") + if width == 0: + return [0] * count + data = r.take(_packed_bytes(count, width)) + mask = (1 << width) - 1 + out: list[int] = [] + acc = 0 + bits = 0 + index = 0 + for _ in range(count): + while bits < width: + acc |= (data[index] if index < len(data) else 0) << bits + index += 1 + bits += 8 + out.append(acc & mask) + acc >>= width + bits -= width + # leftover bits are padding and must be zero, or one value would have two encodings + if acc: + _dfail("bitmap", path, "nonzero bit-packing padding") + return out + + def _int_form(node: dict[str, Any], value: int) -> int: lo = node.get("min") return value - lo if lo is not None else zigzag(value) @@ -310,18 +364,58 @@ def _encode_int_column(out: bytearray, node: dict[str, Any], values: list[int]) out.append(0) return forms = [_int_form(node, v) for v in values] - diffs = [zigzag(values[i] - values[i - 1]) for i in range(1, len(values))] + diffs = [values[i] - values[i - 1] for i in range(1, len(values))] + raw_cost = sum(uleb_len(f) for f in forms) - delta_cost = uleb_len(forms[0]) + sum(uleb_len(d) for d in diffs) - if delta_cost < raw_cost: - out.append(1) - write_uleb(out, forms[0]) - for d in diffs: - write_uleb(out, d) - else: - out.append(0) + delta_cost = uleb_len(forms[0]) + sum(uleb_len(zigzag(d)) for d in diffs) + + # frame of reference: subtract the column minimum, then spend only the bits the + # remaining span needs rather than a whole number of bytes per value + for_base = min(values) + for_width = _bit_width(max(v - for_base for v in values)) + for_cost = ( + math.inf + if for_width > _MAX_WIDTH + else uleb_len(zigzag(for_base)) + 1 + _packed_bytes(len(values), for_width) + ) + + delta_for_cost = math.inf + delta_base = 0 + delta_width = 0 + if diffs: + delta_base = min(diffs) + delta_width = _bit_width(max(d - delta_base for d in diffs)) + if delta_width <= _MAX_WIDTH: + delta_for_cost = ( + uleb_len(forms[0]) + + uleb_len(zigzag(delta_base)) + + 1 + + _packed_bytes(len(diffs), delta_width) + ) + + best = min(raw_cost, delta_cost, for_cost, delta_for_cost) + if best == raw_cost: + out.append(0x00) for f in forms: write_uleb(out, f) + return + if best == delta_cost: + out.append(0x01) + write_uleb(out, forms[0]) + for d in diffs: + write_uleb(out, zigzag(d)) + return + if best == for_cost: + out.append(0x02) + write_uleb(out, zigzag(for_base)) + out.append(for_width) + _pack_bits(out, [v - for_base for v in values], for_width) + return + out.append(0x03) + write_uleb(out, forms[0]) + write_uleb(out, zigzag(delta_base)) + out.append(delta_width) + _pack_bits(out, [d - delta_base for d in diffs], delta_width) def _encode_float_column(out: bytearray, values: list[Any], path: str) -> None: @@ -586,15 +680,31 @@ def _decode_node(r: Reader, node: dict[str, Any], path: str, depth: int, ctx: _C def _decode_int_column(r: Reader, node: dict[str, Any], count: int, path: str) -> list[int]: mode = r.u8() - if mode > 1: + if mode > 3: _dfail("marker", path, f"invalid int column mode 0x{mode:x}") if count == 0: if mode != 0: _dfail("marker", path, "empty column must use mode 0x00") return [] lo = node.get("min") - if mode == 0: + if mode == 0x00: return [_decode_int_value(node, read_uleb(r), f"{path}[{i}]") for i in range(count)] + if mode == 0x02: + base = unzigzag(read_uleb(r)) + width = r.u8() + packed = _unpack_bits(r, count, width, path) + return [_check_decoded(node, base + packed[i], f"{path}[{i}]") for i in range(count)] + if mode == 0x03: + raw_first = read_uleb(r) + running = raw_first + lo if lo is not None else unzigzag(raw_first) + base = unzigzag(read_uleb(r)) + width = r.u8() + packed = _unpack_bits(r, max(0, count - 1), width, path) + acc = [_check_decoded(node, running, f"{path}[0]")] + for i in range(1, count): + running += base + packed[i - 1] + acc.append(_check_decoded(node, running, f"{path}[{i}]")) + return acc out = [] raw = read_uleb(r) prev = raw + lo if lo is not None else unzigzag(raw) diff --git a/python/src/hyperfly/_ir.py b/python/src/hyperfly/_ir.py index c91e924..f98b6e3 100644 --- a/python/src/hyperfly/_ir.py +++ b/python/src/hyperfly/_ir.py @@ -6,7 +6,7 @@ from ._wire import INT_MAX, INT_MIN, HyperflyError LEAF_KINDS = frozenset({"bool", "int", "float64", "string", "bytes", "enum", "literal"}) -_PLAN_VERSION = {"row": 1, "columnar": 3} +_PLAN_VERSION = {"row": 1, "columnar": 4} def _fail(path: str, message: str) -> None: diff --git a/python/tests/test_cross_adapter.py b/python/tests/test_cross_adapter.py index d84bc34..3e7c4e9 100644 --- a/python/tests/test_cross_adapter.py +++ b/python/tests/test_cross_adapter.py @@ -26,7 +26,7 @@ class Row(BaseModel): EXPECTED_ARTIFACT = ( - '{"wire":1,"plan":{"layout":"columnar","version":3},"ir":' + '{"wire":1,"plan":{"layout":"columnar","version":4},"ir":' '{"kind":"struct","fields":[' '{"name":"id","type":{"kind":"string"}},' '{"name":"kind","type":{"kind":"literal","value":"a"}},' diff --git a/rust/src/codec.rs b/rust/src/codec.rs index 0014b8f..6386343 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -857,33 +857,144 @@ impl Codec { } } +const MAX_WIDTH: u8 = 56; + +/// Bits needed for an unsigned value; zero for zero, so a constant column packs to nothing. +fn bit_width(max: u64) -> u8 { + (64 - max.leading_zeros()) as u8 +} + +fn packed_bytes(count: usize, width: u8) -> usize { + if width == 0 { 0 } else { (count * width as usize + 7) / 8 } +} + +/// Spec 3.1: little-endian bit stream, value i at bits [i*w, (i+1)*w). +fn pack_bits(out: &mut Vec, values: &[u64], width: u8) { + if width == 0 { + return; + } + let mut acc: u128 = 0; + let mut bits: u32 = 0; + for value in values { + acc |= (*value as u128) << bits; + bits += width as u32; + while bits >= 8 { + out.push((acc & 0xff) as u8); + acc >>= 8; + bits -= 8; + } + } + if bits > 0 { + out.push((acc & 0xff) as u8); + } +} + +fn unpack_bits(r: &mut Reader, count: usize, width: u8, path: &str) -> Result> { + if width > MAX_WIDTH { + return err(ErrorCode::Marker, format!("{path}: bit width {width} exceeds {MAX_WIDTH}")); + } + if width == 0 { + return Ok(vec![0; count]); + } + let data = r.take(packed_bytes(count, width))?.to_vec(); + let mask: u128 = (1u128 << width) - 1; + let mut out = Vec::with_capacity(count); + let mut acc: u128 = 0; + let mut bits: u32 = 0; + let mut index = 0usize; + for _ in 0..count { + while bits < width as u32 { + let byte = data.get(index).copied().unwrap_or(0); + index += 1; + acc |= (byte as u128) << bits; + bits += 8; + } + out.push((acc & mask) as u64); + acc >>= width; + bits -= width as u32; + } + // leftover bits are padding and must be zero, or one value would have two encodings + if acc != 0 { + return err(ErrorCode::Bitmap, format!("{path}: nonzero bit-packing padding")); + } + Ok(out) +} + fn enc_int_column(out: &mut Vec, min: Option, values: &[i64]) -> Result<()> { if values.is_empty() { out.push(0); return Ok(()); } let forms: Vec = values.iter().map(|v| int_form(min, *v)).collect(); - let diffs: Vec = values.windows(2).map(|w| zigzag(w[1] - w[0])).collect(); + let diffs: Vec = values.windows(2).map(|w| w[1] as i128 - w[0] as i128).collect(); + let raw_cost: usize = forms.iter().map(|f| uleb_len(*f)).sum(); - let delta_cost: usize = uleb_len(forms[0]) + diffs.iter().map(|d| uleb_len(*d)).sum::(); - if delta_cost < raw_cost { - out.push(1); - write_uleb(out, forms[0])?; - for d in diffs { - write_uleb(out, d)?; - } + let delta_cost: usize = + uleb_len(forms[0]) + diffs.iter().map(|d| uleb_len(zigzag(*d as i64))).sum::(); + + // frame of reference: subtract the column minimum, then spend only the bits the + // remaining span needs rather than a whole number of bytes per value + let for_base = *values.iter().min().unwrap(); + let for_span = values.iter().map(|v| (*v as i128 - for_base as i128) as u64).max().unwrap(); + let for_width = bit_width(for_span); + let for_cost = if for_width > MAX_WIDTH { + usize::MAX } else { - out.push(0); + uleb_len(zigzag(for_base)) + 1 + packed_bytes(values.len(), for_width) + }; + + let mut delta_for_cost = usize::MAX; + let mut delta_base: i128 = 0; + let mut delta_width: u8 = 0; + if !diffs.is_empty() { + delta_base = *diffs.iter().min().unwrap(); + let span = diffs.iter().map(|d| (d - delta_base) as u64).max().unwrap(); + delta_width = bit_width(span); + if delta_width <= MAX_WIDTH { + delta_for_cost = uleb_len(forms[0]) + + uleb_len(zigzag(delta_base as i64)) + + 1 + + packed_bytes(diffs.len(), delta_width); + } + } + + let best = raw_cost.min(delta_cost).min(for_cost).min(delta_for_cost); + if best == raw_cost { + out.push(0x00); for f in forms { write_uleb(out, f)?; } + return Ok(()); } + if best == delta_cost { + out.push(0x01); + write_uleb(out, forms[0])?; + for d in &diffs { + write_uleb(out, zigzag(*d as i64))?; + } + return Ok(()); + } + if best == for_cost { + out.push(0x02); + write_uleb(out, zigzag(for_base))?; + out.push(for_width); + let packed: Vec = + values.iter().map(|v| (*v as i128 - for_base as i128) as u64).collect(); + pack_bits(out, &packed, for_width); + return Ok(()); + } + out.push(0x03); + write_uleb(out, forms[0])?; + write_uleb(out, zigzag(delta_base as i64))?; + out.push(delta_width); + let packed: Vec = diffs.iter().map(|d| (d - delta_base) as u64).collect(); + pack_bits(out, &packed, delta_width); Ok(()) } fn dec_int_column(r: &mut Reader, min: Option, max: Option, count: usize, path: &str) -> Result> { let mode = r.u8()?; - if mode > 1 { + if mode > 3 { return err(ErrorCode::Marker, format!("{path}: invalid int column mode {mode:#x}")); } if count == 0 { @@ -899,12 +1010,36 @@ fn dec_int_column(r: &mut Reader, min: Option, max: Option, count: usi } }; let mut out = Vec::with_capacity(count); - if mode == 0 { + if mode == 0x00 { for i in 0..count { out.push(decoded_int(min, max, from_form(read_uleb(r)?), &format!("{path}[{i}]"))?); } return Ok(out); } + if mode == 0x02 { + let base = unzigzag(read_uleb(r)?) as i128; + let width = r.u8()?; + let packed = unpack_bits(r, count, width, path)?; + for (i, p) in packed.iter().enumerate() { + let value = (base + *p as i128).clamp(i64::MIN as i128, i64::MAX as i128) as i64; + out.push(decoded_int(min, max, value, &format!("{path}[{i}]"))?); + } + return Ok(out); + } + if mode == 0x03 { + let first = from_form(read_uleb(r)?); + let base = unzigzag(read_uleb(r)?) as i128; + let width = r.u8()?; + let packed = unpack_bits(r, count.saturating_sub(1), width, path)?; + let mut running = first as i128; + out.push(decoded_int(min, max, first, &format!("{path}[0]"))?); + for i in 1..count { + running += base + packed[i - 1] as i128; + let value = running.clamp(i64::MIN as i128, i64::MAX as i128) as i64; + out.push(decoded_int(min, max, value, &format!("{path}[{i}]"))?); + } + return Ok(out); + } let mut prev = from_form(read_uleb(r)?) as i128; out.push(decoded_int(min, max, prev.clamp(i64::MIN as i128, i64::MAX as i128) as i64, &format!("{path}[0]"))?); for i in 1..count { diff --git a/rust/src/ir.rs b/rust/src/ir.rs index 69ceb86..c3e946a 100644 --- a/rust/src/ir.rs +++ b/rust/src/ir.rs @@ -227,7 +227,7 @@ impl Plan { fn version(self) -> u32 { match self { Plan::Row => 1, - Plan::Columnar => 3, + Plan::Columnar => 4, } } } diff --git a/spec/plan-columnar-v3.md b/spec/plan-columnar-v4.md similarity index 87% rename from spec/plan-columnar-v3.md rename to spec/plan-columnar-v4.md index ee05d07..b6634cc 100644 --- a/spec/plan-columnar-v3.md +++ b/spec/plan-columnar-v4.md @@ -1,12 +1,12 @@ -# Hyperfly plan `columnar` — v3 +# Hyperfly plan `columnar` — v4 Status: draft. Extends `spec/wire-v0.md`; everything there (envelope, varints, bitmaps, scalar encodings, limits, canonical serialization) applies unchanged. The artifact is -`{"wire":1,"plan":{"layout":"columnar","version":3},"ir":…,"profile":…}` — a +`{"wire":1,"plan":{"layout":"columnar","version":4},"ir":…,"profile":…}` — a different fingerprint than the row plan for the same IR, so the two never mix -on the wire. (v1 and v2 were never released; no artifact for either exists in -the wild.) +on the wire. (v1 through v3 were never released; no artifact for any of them +exists in the wild.) ## 1. Scope @@ -33,6 +33,20 @@ always agree. ## 3. Column payloads +### 3.1 Bit packing + +Several column modes pack `k` unsigned values of a fixed width `w` bits. + +- `w` is a single byte, `0 ≤ w ≤ 56`; larger MUST be rejected. The 56-bit + ceiling matches the `uvarint` domain of wire-v0 §3.1. +- `w = 0` encodes no payload bytes at all: every value equals the frame base. + A constant column therefore costs its base and nothing more. +- Value `i` occupies bits `[i·w, (i+1)·w)` of a little-endian bit stream: bit + `n` is bit `n mod 8` of byte `⌊n/8⌋`, counting from the least significant. +- The payload is exactly `⌈k·w/8⌉` bytes. Bits after the last value in the + final byte MUST be zero, and a decoder MUST reject nonzero padding — without + that rule one value would have several encodings. + `k` = participating row count. - **literal** — zero bytes. @@ -68,8 +82,20 @@ always agree. - `0x01` delta: the first value in its wire-v0 form, then `svarint(v[i] - v[i-1])` for each subsequent value. Differences stay within 55 bits for domain-valid values, so the 8-byte uvarint cap holds. + - `0x02` frame of reference: `svarint(base)`, one width byte `w`, then the + values bit-packed (§3.1) as `v[i] - base`. `base` is the column minimum, so + every packed value is non-negative and fits `w` bits. + - `0x03` delta frame of reference: `svarint(v[0])`, `svarint(base)`, one + width byte `w`, then `d[i] - base` bit-packed for `i` in `1..k-1`, where + `d[i] = v[i] - v[i-1]` and `base` is the minimum difference. - Other mode bytes MUST be rejected. Declared bounds are validated per - decoded value, after delta accumulation. + decoded value, after any accumulation. + + A varint spends whole bytes on values that need a fraction of one: a column + ranging over four possible values costs eight bits each under `0x00` and two + under `0x02`. The frame is per-column and self-describing, so it needs no + profile — an untrained codec gets it — and it adapts to the values actually + present rather than the bounds the schema permits. - **float64** — one mode byte, then: - `0x00` raw: each value as 8 bytes LE (wire-v0 §4.5 rules per value). - `0x01` xor: first value as 8 bytes LE, then for each subsequent value a diff --git a/spec/vectors/columnar.json b/spec/vectors/columnar.json index c8d4457..17beb9e 100644 --- a/spec/vectors/columnar.json +++ b/spec/vectors/columnar.json @@ -1,5 +1,5 @@ { - "description": "Golden vectors for plan columnar@3. Hex is body bytes only. All vectors compile with plan: columnar.", + "description": "Golden vectors for plan columnar@4. Hex is body bytes only. All vectors compile with plan: columnar.", "valid": [ { "name": "col-int-raw-on-tie", @@ -31,7 +31,7 @@ "hex": "0300020406" }, { - "name": "col-int-delta", + "name": "col-int-delta-for", "ir": { "kind": "array", "element": { @@ -58,7 +58,8 @@ "t": 100600 } ], - "hex": "0301a08d06d804d804" + "hex": "0303a08d06d80400", + "description": "constant stride: delta frame of reference packs to width 0, so the payload is empty" }, { "name": "col-bool-bitmap", @@ -500,6 +501,110 @@ } ], "hex": "02000204020012" + }, + { + "name": "col-int-delta-still-wins", + "description": "one huge jump among tiny ones makes bit-packing wider than varints, so plain delta wins", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "int" + } + } + ] + } + }, + "value": [ + { + "t": 0 + }, + { + "t": 1 + }, + { + "t": 2 + }, + { + "t": 3 + }, + { + "t": 1099511627776 + } + ], + "hex": "050000020406808080808040" + }, + { + "name": "col-int-for-narrow", + "description": "four possible values cost two bits each, not eight", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "int", + "min": 0, + "max": 30 + } + } + ] + } + }, + "value": [ + { + "t": 1 + }, + { + "t": 2 + }, + { + "t": 3 + }, + { + "t": 4 + } + ], + "hex": "04020202e4" + }, + { + "name": "col-int-for-constant-width-zero", + "description": "a constant column packs to width 0 and carries no payload bytes at all", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "int" + } + } + ] + } + }, + "value": [ + { + "t": 7 + }, + { + "t": 7 + }, + { + "t": 7 + }, + { + "t": 7 + } + ], + "hex": "04020e00" } ], "invalidDecode": [ @@ -635,7 +740,7 @@ ] } }, - "hex": "0102", + "hex": "0104", "error": "marker" }, { @@ -751,6 +856,45 @@ }, "hex": "010200040300dead", "error": "packed" + }, + { + "name": "col-int-for-width-over-56", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "int" + } + } + ] + } + }, + "hex": "01020039", + "error": "marker" + }, + { + "name": "col-int-for-nonzero-padding", + "description": "two 2-bit values leave six padding bits; nonzero padding would give one value two encodings", + "ir": { + "kind": "array", + "element": { + "kind": "struct", + "fields": [ + { + "name": "t", + "type": { + "kind": "int" + } + } + ] + } + }, + "hex": "02020002f4", + "error": "bitmap" } ], "invalidEncode": [ diff --git a/spec/vectors/fingerprints.json b/spec/vectors/fingerprints.json index 5d90040..b2a6b6c 100644 --- a/spec/vectors/fingerprints.json +++ b/spec/vectors/fingerprints.json @@ -125,8 +125,8 @@ "ir": { "kind": "bool" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"bool\"}}", - "fingerprint": "6a7373611b1a316a06e3f056229e4fbb" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"bool\"}}", + "fingerprint": "5942d3d6a6e9113dcc289a30b769825a" }, { "name": "bounded-int@columnar", @@ -136,8 +136,8 @@ "min": 0, "max": 100 }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", - "fingerprint": "e0ff19388faced1d70ac4d8bb1d86b8d" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"int\",\"min\":0,\"max\":100}}", + "fingerprint": "3b850d87c83af17c79aeca8b27e70ca5" }, { "name": "enum@columnar", @@ -151,8 +151,8 @@ "1d" ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", - "fingerprint": "ec9d5aac5141cc277b13c3775bef013b" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"enum\",\"members\":[\"1m\",\"5m\",\"1h\",\"1d\"]}}", + "fingerprint": "2791f80750a05aec10373412baef1c71" }, { "name": "candles-response@columnar", @@ -224,8 +224,8 @@ } ] }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", - "fingerprint": "f000d175875a31831d1093b00a355708" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"route\",\"type\":{\"kind\":\"literal\",\"value\":\"candles\"}},{\"name\":\"candles\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"int\",\"min\":0}},{\"name\":\"o\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"h\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"l\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"c\",\"type\":{\"kind\":\"float64\"}},{\"name\":\"v\",\"type\":{\"kind\":\"float64\"}}]}}},{\"name\":\"cursor\",\"type\":{\"kind\":\"string\"},\"optional\":true}]}}", + "fingerprint": "e5087474cd2e3bfb869729e1178b5d8c" }, { "name": "escaping@columnar", @@ -234,8 +234,8 @@ "kind": "literal", "value": "a\"b\\c" }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", - "fingerprint": "cae5981b91f66217992b7652c780baa6" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"literal\",\"value\":\"a\\\"b\\\\c\"}}", + "fingerprint": "e51d9e46a447e7e7ef200641e4dae75f" }, { "name": "profiled-single", @@ -270,8 +270,8 @@ ] } }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}},\"profile\":{\"columns\":[{\"leaf\":0,\"dict\":[\"a\\\"q\",\"b\\\\s\",\"c\\u0001\",\"🚀\"]}]}}", - "fingerprint": "a65108e20d19c19ff525ddf4789d5ba1" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}},\"profile\":{\"columns\":[{\"leaf\":0,\"dict\":[\"a\\\"q\",\"b\\\\s\",\"c\\u0001\",\"🚀\"]}]}}", + "fingerprint": "c8636152f2da01f1bed5a5e7d20e7339" }, { "name": "profiled-two-arrays", @@ -328,8 +328,8 @@ ] } }, - "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":3},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}}},{\"name\":\"b\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"string\"}}]}}}]},\"profile\":{\"columns\":[{\"leaf\":1,\"dict\":[\"only-second\"]}]}}", - "fingerprint": "123c921f0dccfe8d25f976757766c4e1" + "canonical": "{\"wire\":1,\"plan\":{\"layout\":\"columnar\",\"version\":4},\"ir\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"kind\":\"string\"}}]}}},{\"name\":\"b\",\"type\":{\"kind\":\"array\",\"element\":{\"kind\":\"struct\",\"fields\":[{\"name\":\"t\",\"type\":{\"kind\":\"string\"}}]}}}]},\"profile\":{\"columns\":[{\"leaf\":1,\"dict\":[\"only-second\"]}]}}", + "fingerprint": "c1743c6c3d7f7a13e263f500d149af76" } ] }