From 87991042ab4d87bc064351e854964d0ed4c1726c Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 13:14:15 +0200 Subject: [PATCH 1/8] fix(hist): drop the upper-bound pad, the top clamp covers it --- Architecture.md | 14 ++--- flexviz/cube.py | 6 --- flexviz/engine.py | 15 ++---- flexviz/trace/hist.py | 23 +++----- flexviz/trace/hist2d.py | 7 +-- flexviz/trace/line.py | 3 +- flexviz_polars/src/expressions.rs | 4 +- flexviz_polars/tests/test_plugin_functions.py | 54 +++++++++---------- tests/test_browser.py | 10 ++-- tests/test_browser_cube.py | 21 ++++---- tests/test_cube.py | 41 +------------- tests/test_cube_server.py | 47 ++++++++++++---- tests/test_trace_hist.py | 27 +++++----- 13 files changed, 112 insertions(+), 160 deletions(-) diff --git a/Architecture.md b/Architecture.md index 29c74e16..1d339665 100644 --- a/Architecture.md +++ b/Architecture.md @@ -554,7 +554,7 @@ fig.add_histogram(x="value", bins=20, histnorm="count") date axis, like the line trace) and emits the bin-edge triple in epoch-ms. Mirrors the cube's `_typed_temporal_lit(...).to_physical()` idiom and `temporal_unit` (contract G). -- Bin edges: unzoomed they span the engine-resolved unfiltered domain (see the sibling-domain bullet below), so a cross-filter cannot move them. Zoomed they span the viewport **snapped outward to the lattice of width `(hi - lo) / n`**, which costs at most one extra bin. A pan keeps the span, so the lattice stays fixed and every bar keeps its place instead of sliding under the data. The viewport filter uses the snapped range, so an edge bin is complete. A degenerate span is left alone, and unzoomed domain edges are never snapped. `snap_range` / `snapped_axis` in `trace/bin_grid.py` are the one place a viewport is snapped, shared with the 2-D traces. The 1-D trace then pads `hi` by `_HIST_BIN_EPSILON`, so it bins on `[lo, hi + 1e-10]` and a value on the upper bound lands in the last bin. The 2-D path passes the raw bounds instead: the kernel's top clamp folds a value on the upper bound into the last bin. +- Bin edges: unzoomed they span the engine-resolved unfiltered domain (see the sibling-domain bullet below), so a cross-filter cannot move them. Zoomed they span the viewport **snapped outward to the lattice of width `(hi - lo) / n`**, which costs at most one extra bin. A pan keeps the span, so the lattice stays fixed and every bar keeps its place instead of sliding under the data. The viewport filter uses the snapped range, so an edge bin is complete. A degenerate span is left alone, and unzoomed domain edges are never snapped. `snap_range` / `snapped_axis` in `trace/bin_grid.py` are the one place a viewport is snapped, shared with the 2-D traces. The kernel gets the raw bounds: its top clamp folds a value on the upper bound into the last bin. So the bin range is the row range the viewport mask keeps, and a cube built on the same bounds keeps the same rows. - A zoomed grid can hold one bin more than configured, so `_to_update` reads the grid the trace stored in `get_aggregation_spec` (`Histogram._bin_edges`) and the cube target dim carries the snapped `(domain, bins)`, not the configured `bins`. The 1-D plans emit `count` only and the kernel's `breakpoint` field is never read, so the grid travels beside the result rather than in it (issue #37). The client derives bar centers from those two, so cube-served bars land on the server's bars. - Multiple active histogram traces on the same figure, axes, data axis, and coordinate unit share one no-viewport min/max domain before calling @@ -646,7 +646,7 @@ fig.add_histogram2d(x="x", y="y", histfunc="sum", z="weight", histnorm="percent" - `median` and `n_unique` are intentionally not supported for cartesian `Histogram2D` in this fast-path stage; they can be added back as separate reducers if needed. - The resident viewport path prefilters x/y/z inside the kernel expression. The scan fold applies the viewport filter to the frame instead, so the scan itself rejects the rows. A later viewport-aware kernel can fuse range rejection into the Rust loop. - `histnorm` controls post-aggregation normalization: `None` (no normalization, default), `"percent"`, `"probability"`, `"density"`, `"probability density"`. -- Bin edges: each axis resolves on its own through `axis_edges` in `trace/bin_grid.py`, because the client sends only the axes a zoom moved, so a zoom on x alone re-bins x and leaves y on its full domain. Unzoomed, an axis spans the engine-resolved unfiltered domain of its column (`domain_cols`). Zoomed, it spans the viewport snapped outward to the same fixed lattice the 1-D trace uses, and the mask filters on the snapped rectangle so an edge cell is complete. Both cases pass the raw bounds to the kernel, whose top clamp folds a value at `hi` into the last bin, so the 1-D `_HIST_BIN_EPSILON` has no counterpart here. +- Bin edges: each axis resolves on its own through `axis_edges` in `trace/bin_grid.py`, because the client sends only the axes a zoom moved, so a zoom on x alone re-bins x and leaves y on its full domain. Unzoomed, an axis spans the engine-resolved unfiltered domain of its column (`domain_cols`). Zoomed, it spans the viewport snapped outward to the same fixed lattice the 1-D trace uses, and the mask filters on the snapped rectangle so an edge cell is complete. Both cases pass the raw bounds to the kernel, whose top clamp folds a value at `hi` into the last bin, as in the 1-D trace. - The zoomed grid can hold one more bin per axis than configured, so the trace stores the grid it actually binned on and `_to_update` unpacks `z_flat` with that, not with `x_bins`/`y_bins`. Cube target dims keep the configured bins: a cube hist2d target is full-data only, so it never sees a snapped grid. - Empty bins are emitted as `None`; empty viewports / all-null inputs produce an all-null grid so renderers show gaps instead of zero-count cells. - Public style API: `color_scale`, `color_range` and `color_norm`; defaults are `"Viridis"`, `"auto"` and `"linear"`. @@ -1352,13 +1352,13 @@ them. | trace | target dims | measure | |---|---|---| - | hist (ungrouped) | (binned data col) — display bin edges, `domain_hi + _HIST_BIN_EPSILON`, so a slice reproduces the Rust `fixed_hist` membership bit-exactly; zoomed: snapped `(domain, bins)`, see the Histogram section | count | + | hist (ungrouped) | (binned data col) — display bin edges, so a slice reproduces the Rust `fixed_hist` membership bit-exactly; zoomed: snapped `(domain, bins)`, see the Histogram section | count | | hist (grouped) | (binned data col, *group cols as categorical) — same snap, `get_cube_target_spec` snaps before the grouped branch | count | | bar | (*label cols, *group cols — all categorical) | `count`/`sum`/`mean`/`min`/`max` over `values` | | pie | (*label cols as categorical) | same | | line | (binned x col @ `n_points/2` buckets, *group cols as categorical) — **minmax-only** | `line_env` over `y` | | corr_heatmap | `()` — no grouping dims; the matrix cells are the explicit `columns` pairs; `columns` must be passed explicitly for cube support | `corr` (Pearson only) | - | hist2d | (binned x col, binned y col) — both `bin_variant="hist2d"` (only skips the domain pad; the bin expression is shared with hist1d), **bit-equal to the `fixed_hist2d` kernel**; **full-data only** (declines when either axis is zoomed) | count, or `histfunc` over `z` | + | hist2d | (binned x col, binned y col) — the bin expression is shared with hist1d, **bit-equal to the `fixed_hist2d` kernel**; **full-data only** (declines when either axis is zoomed) | count, or `histfunc` over `z` | | treemap | (*path cols as categorical) — the **leaf** level; the client finalizes leaf cells then **sums** them up every path level (parents = Σ of child finalized values, mirroring `_to_grouped_update`) | `count`/`sum`/`mean`/`min`/`max` over `values` | bar ≡ pie descriptor sharing falls out of content-key dedup: same labels + same measure ⇒ one @@ -1376,9 +1376,9 @@ them. cube. **The line envelope caveat:** the cube ships the *minmax-bucket* envelope so the drag is live every frame, but a line target's commit **always POSTs** (`postRequired`) so the committed delta replaces the approximate envelope — keeping commit ≡ share/restore bit-exact. Both are - x-width envelopes, but their bucket edges can differ: the cube pads its upper bound by - `_HIST_BIN_EPSILON` and divides true, while `bucket_grid` rounds an integer viewport bound - inward and keeps a whole bucket width on integer and temporal x. Issue #24 tracks it. + x-width envelopes, but their bucket edges can differ: the cube divides true, while + `bucket_grid` rounds an integer viewport bound inward and keeps a whole bucket width on + integer and temporal x. Issue #24 tracks it. - **Dtype/name gates** (descriptor methods return `None`; a schema is required for categorical capability): every categorical dim column must be `String`/`Categorical`/`Enum`, with one exception — a **bar/pie label** column may also be **integer- or float-typed**, on **both** the diff --git a/flexviz/cube.py b/flexviz/cube.py index 12b4a6a3..aa8bbb75 100644 --- a/flexviz/cube.py +++ b/flexviz/cube.py @@ -206,11 +206,6 @@ class TargetDimSpec: ``unit`` marks a temporal binned dim (contract G): binning runs on the column's physical representation in that unit and the header dim entry carries it. The engine sets it from the schema dtype. - - ``bin_variant`` marks a binned dim as ``"hist1d"`` (default) or - ``"hist2d"`` (contract K). Both bin bit-equal to their Rust kernels with - ``scale = n/(hi-lo)``. The variant only selects the engine's domain padding - (see ``_resolved_target_dims``). """ column: str @@ -218,7 +213,6 @@ class TargetDimSpec: bins: int | None = None domain: tuple[float, float] | None = None unit: str | None = None - bin_variant: Literal["hist1d", "hist2d"] = "hist1d" def __post_init__(self) -> None: if self.kind == "binned" and self.bins is None: diff --git a/flexviz/engine.py b/flexviz/engine.py index 1d824e6c..74e3c5a5 100644 --- a/flexviz/engine.py +++ b/flexviz/engine.py @@ -31,7 +31,6 @@ _physical_bound_expr, child_uid_from_group_key, ) -from .trace.hist import _HIST_BIN_EPSILON # Event types whose computation is unfiltered and viewport-free. These are the # only events cached in Phase 1: the engine forces them to drop all @@ -754,15 +753,8 @@ def _resolved_target_dims( domains: dict[str, tuple[float, float]], schema: pl.Schema | None = None, ) -> tuple | None: - """Resolve binned target dims and epsilon-pad their upper bounds. - - ``_HIST_BIN_EPSILON`` is added to every ``hist1d``-variant binned dim's - resolved-or-zoomed domain. This mirrors ``_histogram_bounds_exprs``, - which pads in both the unzoomed and zoomed cases, so cube bins align - with display bins. A ``hist2d``-variant dim (contract K) gets no pad: - the ``fixed_hist2d`` kernel and ``_fixed_hist_bin_expr`` both fold a - value at ``hi`` into the top bin through the top clamp. A pad here - would shift bins and break bit-equality with the server delta. + """Resolve binned target dims to the display's bin domain. + Returns ``None`` when a dim's domain cannot be resolved (an all-null column). That target is not served. @@ -784,8 +776,7 @@ def _resolved_target_dims( unit = temporal_unit(dtype) if unit is None: return None # ns/Time gate - pad = 0.0 if d.bin_variant == "hist2d" else _HIST_BIN_EPSILON - dims.append(replace(d, domain=(domain[0], domain[1] + pad), unit=unit)) + dims.append(replace(d, domain=(domain[0], domain[1]), unit=unit)) return tuple(dims) def _active_selections( diff --git a/flexviz/trace/hist.py b/flexviz/trace/hist.py index e9eec8b7..3231bd0f 100644 --- a/flexviz/trace/hist.py +++ b/flexviz/trace/hist.py @@ -58,13 +58,6 @@ # Bin-edge helpers # --------------------------------------------------------------------------- -#: Small offset added to the upper bound (``hi``) so the maximum data point -#: always falls inside the last bin and bin edges are never degenerate. On a -#: zoomed axis it pads the *snapped* ``hi``, so a value sitting exactly on it -#: still lands in the last bin. Distinct from the cube's -#: ``_FIXED_HIST_ROUND_EPS``, which pads the bin index instead of ``hi``. -_HIST_BIN_EPSILON: float = 1e-10 - def _streaming_hist_plan( value_expr: pl.Expr, @@ -279,10 +272,6 @@ def get_cube_target_spec( ``domain`` and ``bins``, so a cube-served bar would otherwise miss the server's bar. ``axis_range`` is already physical here (``FlexEngine._cube_axis_range``), so only the lattice rule applies. - The trace never adds ``_HIST_BIN_EPSILON``: the **engine** - epsilon-pads the upper bound uniformly when resolving domains (both - ``None``-resolved full domains and snapped viewports), mirroring - ``_histogram_bounds_exprs`` so cube bins align with display bins. A **grouped** histogram appends one categorical dim per ``group_by`` column after the binned dim (pinned order — contract C); the client @@ -428,10 +417,14 @@ def _histogram_bounds_exprs( and the mask restricts the rows to that same span, so an edge bin is complete. Snapping costs at most one extra bin, which is why the count comes back too. + + The edges are the bounds themselves, never padded: the kernel's top + clamp puts a value at ``hi`` in the last bin, and the bin range stays + the row range the mask keeps. """ if axis_range is not None: lo, hi, n, mask = snapped_axis(self.data_col, axis_range, self.bins, schema) - return lo, hi + _HIST_BIN_EPSILON, n, mask + return lo, hi, n, mask # The trace's own column must be a resolved key; a missing key means # the caller violated the unzoomed-domains contract. @@ -448,7 +441,7 @@ def _histogram_bounds_exprs( # f64, so the edges are floats from here on. lo = float(min(los)) if los else 0.0 hi = float(max(his)) if his else 1.0 - return lo, hi + _HIST_BIN_EPSILON, self.bins, None + return lo, hi, self.bins, None def _to_update( self, @@ -461,8 +454,8 @@ def _to_update( lo, hi, n_bins = self._bin_edges step = (hi - lo) / n_bins - # hi == lo is the kernel's degenerate span: its epsilon pad vanishes at - # the column's magnitude. Use 1.0 so density norms stay finite. + # hi == lo (a constant column) has no step. Use 1.0 so density norms + # stay finite. bin_width = step if step > 0.0 else 1.0 centers = (pl.int_range(0, n_bins, eager=True) + 0.5) * step + lo diff --git a/flexviz/trace/hist2d.py b/flexviz/trace/hist2d.py index 652f47d9..97bf1408 100644 --- a/flexviz/trace/hist2d.py +++ b/flexviz/trace/hist2d.py @@ -343,9 +343,8 @@ def get_cube_target_spec( """A 2-D histogram is a ``count``/reduce cube target (contract K). Its grouping dims are its own ``(x_col, y_col)`` bin axes (order pinned: - x first, y second) with ``bin_variant="hist2d"`` so the cube bins - bit-equally to the ``fixed_hist2d`` kernel (no span pad, top clamp - only). + x first, y second), binned bit-equally to the ``fixed_hist2d`` kernel + (top clamp, no span pad). The measure is a count when ``z_col`` is ``None``, else the ``histfunc`` reduction over ``z_col``. ``histnorm`` is NOT part of the cube — it is a client-side display normalization applied per-slice (two hist2ds @@ -394,14 +393,12 @@ def get_cube_target_spec( kind="binned", bins=self.x_bins, domain=None, - bin_variant="hist2d", ), TargetDimSpec( column=y_col, kind="binned", bins=self.y_bins, domain=None, - bin_variant="hist2d", ), ), measure=measure, diff --git a/flexviz/trace/line.py b/flexviz/trace/line.py index fe40630e..0cde2849 100644 --- a/flexviz/trace/line.py +++ b/flexviz/trace/line.py @@ -553,8 +553,7 @@ def get_cube_target_spec( ``n_buckets = max(1, n_points // 2)`` matches the legacy minmax bucket count (two extrema points per bucket). ``domain`` is ``axis_range`` - verbatim (``None`` when unzoomed); the engine epsilon-pads it - uniformly when resolving binned domains. + verbatim (``None`` when unzoomed). Caveat (no size guard): a grouped line runs the envelope kernel once per series, so the cube payload multiplies by the group cardinality — diff --git a/flexviz_polars/src/expressions.rs b/flexviz_polars/src/expressions.rs index 8b9118d5..7231df04 100644 --- a/flexviz_polars/src/expressions.rs +++ b/flexviz_polars/src/expressions.rs @@ -758,8 +758,8 @@ where } // Match the old fixed_hist boundary behavior without casting the full column. - // The small epsilon keeps integer-like values on visual bin boundaries from - // falling into the previous bin when the caller has nudged `hi` upward. + // The small epsilon keeps a value on a bin edge from falling into the + // previous bin when float rounding puts it a hair below that edge. let scale = n_bins as f64 / (hi - lo); let max_idx = n_bins - 1; diff --git a/flexviz_polars/tests/test_plugin_functions.py b/flexviz_polars/tests/test_plugin_functions.py index f7acdd28..80673b4f 100644 --- a/flexviz_polars/tests/test_plugin_functions.py +++ b/flexviz_polars/tests/test_plugin_functions.py @@ -377,21 +377,19 @@ def test_length_mismatch_is_rejected(self): # fixed_hist # --------------------------------------------------------------------------- -_EPS = 1e-10 # matches _HIST_BIN_EPSILON in hist.py - class TestFixedHist: # ---- output structure ------------------------------------------------------- def test_struct_field_names(self): s = pl.Series("v", [1.0, 2.0, 3.0], dtype=pl.Float64) - result = _fixed_hist(s, 0.0, 4.0 + _EPS, n_bins=4) + result = _fixed_hist(s, 0.0, 4.0, n_bins=4) assert result.dtype == pl.Struct({"breakpoint": pl.Float64, "count": pl.UInt32}) def test_output_length_equals_n_bins(self): s = pl.Series("v", list(range(100)), dtype=pl.Float64) for n_bins in [1, 5, 10, 20]: - result = _fixed_hist(s, 0.0, 100.0 + _EPS, n_bins=n_bins) + result = _fixed_hist(s, 0.0, 100.0, n_bins=n_bins) assert len(result) == n_bins, f"n_bins={n_bins}" # ---- correctness ------------------------------------------------------------ @@ -400,13 +398,13 @@ def test_uniform_data_equal_counts(self): """100 values spread evenly over 10 bins → each bin has count 10.""" vals = [float(i) for i in range(100)] # 0..99 s = pl.Series("v", vals, dtype=pl.Float64) - lo, hi = 0.0, 100.0 + _EPS + lo, hi = 0.0, 100.0 counts = _fixed_hist_counts(s, lo, hi, n_bins=10) assert counts == [10] * 10 def test_total_count_equals_non_null_len(self): s = pl.Series("v", [1.0, 2.0, None, 4.0, 5.0], dtype=pl.Float64) - counts = _fixed_hist_counts(s, 0.0, 6.0 + _EPS, n_bins=3) + counts = _fixed_hist_counts(s, 0.0, 6.0, n_bins=3) assert sum(counts) == 4 # 4 non-null values def test_breakpoints_formula(self): @@ -428,9 +426,7 @@ def test_counts_match_polars_hist(self): s = pl.Series("v", vals, dtype=pl.Float64) lo, hi = 0.0, 100.0 n_bins = 20 - eps = _EPS - hi_eps = hi + eps - step = (hi_eps - lo) / n_bins + step = (hi - lo) / n_bins edges = [lo + i * step for i in range(n_bins + 1)] polars_counts = ( @@ -439,52 +435,52 @@ def test_counts_match_polars_hist(self): .struct.field("count") .to_list() ) - plugin_counts = _fixed_hist_counts(s, lo, hi_eps, n_bins=n_bins) + plugin_counts = _fixed_hist_counts(s, lo, hi, n_bins=n_bins) assert plugin_counts == polars_counts # ---- dtype support ---------------------------------------------------------- def test_dtype_int32(self): s = pl.Series("v", list(range(50)), dtype=pl.Int32) - counts = _fixed_hist_counts(s, 0.0, 50.0 + _EPS, n_bins=5) + counts = _fixed_hist_counts(s, 0.0, 50.0, n_bins=5) assert sum(counts) == 50 assert counts == [10] * 5 def test_dtype_int64(self): s = pl.Series("v", list(range(50)), dtype=pl.Int64) - counts = _fixed_hist_counts(s, 0.0, 50.0 + _EPS, n_bins=5) + counts = _fixed_hist_counts(s, 0.0, 50.0, n_bins=5) assert counts == [10] * 5 def test_dtype_int8(self): s = pl.Series("v", list(range(-10, 10)), dtype=pl.Int8) - counts = _fixed_hist_counts(s, -10.0, 10.0 + _EPS, n_bins=4) + counts = _fixed_hist_counts(s, -10.0, 10.0, n_bins=4) assert sum(counts) == 20 def test_dtype_uint32(self): s = pl.Series("v", list(range(20)), dtype=pl.UInt32) - counts = _fixed_hist_counts(s, 0.0, 20.0 + _EPS, n_bins=4) + counts = _fixed_hist_counts(s, 0.0, 20.0, n_bins=4) assert counts == [5] * 4 def test_dtype_float32(self): s = pl.Series("v", [float(i) for i in range(40)], dtype=pl.Float32) - counts = _fixed_hist_counts(s, 0.0, 40.0 + _EPS, n_bins=4) + counts = _fixed_hist_counts(s, 0.0, 40.0, n_bins=4) assert counts == [10] * 4 # ---- edge cases ------------------------------------------------------------- def test_empty_series_all_zero_counts(self): s = pl.Series("v", [], dtype=pl.Float64) - counts = _fixed_hist_counts(s, 0.0, 1.0 + _EPS, n_bins=5) + counts = _fixed_hist_counts(s, 0.0, 1.0, n_bins=5) assert counts == [0] * 5 def test_all_nulls_all_zero_counts(self): s = pl.Series("v", [None, None, None], dtype=pl.Float64) - counts = _fixed_hist_counts(s, 0.0, 1.0 + _EPS, n_bins=5) + counts = _fixed_hist_counts(s, 0.0, 1.0, n_bins=5) assert counts == [0] * 5 def test_single_bin(self): s = pl.Series("v", [1.0, 2.0, 3.0], dtype=pl.Float64) - counts = _fixed_hist_counts(s, 0.0, 4.0 + _EPS, n_bins=1) + counts = _fixed_hist_counts(s, 0.0, 4.0, n_bins=1) assert counts == [3] def test_values_at_boundary_clamped(self): @@ -493,7 +489,7 @@ def test_values_at_boundary_clamped(self): n_bins = 5 # lo lands in bin 0, hi lands in bin n_bins-1 (after clamping) s = pl.Series("v", [lo, hi], dtype=pl.Float64) - counts = _fixed_hist_counts(s, lo, hi + _EPS, n_bins=n_bins) + counts = _fixed_hist_counts(s, lo, hi, n_bins=n_bins) assert counts[0] == 1, "value at lo must be in first bin" assert counts[-1] == 1, "value at hi must be in last bin" @@ -544,7 +540,7 @@ def test_dispatched_dtypes(self, dtype): n = _MIN_PAR + 1234 vals = [(i * 7919) % 1000 for i in range(n)] s = pl.Series("v", vals, dtype=dtype) - lo, hi = 0.0, 1000.0 + _EPS + lo, hi = 0.0, 1000.0 assert _fixed_hist_counts(s, lo, hi, 256) == _ref_hist_counts(vals, lo, hi, 256) @pytest.mark.parametrize("dtype", [pl.UInt8, pl.UInt16]) @@ -553,14 +549,14 @@ def test_undispatched_dtypes_fall_back_correctly(self, dtype): n = _MIN_PAR + 7 vals = [i % 200 for i in range(n)] s = pl.Series("v", vals, dtype=dtype) - lo, hi = 0.0, 200.0 + _EPS + lo, hi = 0.0, 200.0 assert _fixed_hist_counts(s, lo, hi, 64) == _ref_hist_counts(vals, lo, hi, 64) def test_nan_is_skipped(self): n = _MIN_PAR + 500 vals = [float("nan") if i % 1000 == 0 else float(i % 997) for i in range(n)] s = pl.Series("v", vals, dtype=pl.Float64) - lo, hi = 0.0, 997.0 + _EPS + lo, hi = 0.0, 997.0 counts = _fixed_hist_counts(s, lo, hi, 128) assert counts == _ref_hist_counts(vals, lo, hi, 128) assert sum(counts) == sum(1 for v in vals if v == v) @@ -570,7 +566,7 @@ def test_nulls_force_the_scalar_fallback_and_stay_exact(self): n = _MIN_PAR + 321 vals = [None if i % 500 == 0 else float(i % 313) for i in range(n)] s = pl.Series("v", vals, dtype=pl.Float64) - lo, hi = 0.0, 313.0 + _EPS + lo, hi = 0.0, 313.0 counts = _fixed_hist_counts(s, lo, hi, 64) assert counts == _ref_hist_counts(vals, lo, hi, 64) assert sum(counts) == sum(1 for v in vals if v is not None) @@ -584,7 +580,7 @@ def test_multi_chunk_matches_single_chunk(self, n_chunks_in): """Concatenated frames are the normal case; they must not fall back.""" n = _MIN_PAR + 999 vals = [float((i * 31) % 500) for i in range(n)] - lo, hi = 0.0, 500.0 + _EPS + lo, hi = 0.0, 500.0 one = pl.Series("v", vals, dtype=pl.Float64) # Uneven cuts, so the work-splitting sees runs of different sizes. cuts = ( @@ -607,7 +603,7 @@ def test_many_small_chunks_stay_exact(self): must not change a single count.""" n = 2 * _MIN_PAR vals = [float((i * 13) % 353) for i in range(n)] - lo, hi = 0.0, 353.0 + _EPS + lo, hi = 0.0, 353.0 step = n // 40 cuts = list(range(0, n, step)) + [n] many = pl.concat( @@ -628,7 +624,7 @@ def test_bins_above_budget_fall_back_and_stay_exact(self): n = _MIN_PAR + 33 n_bins = 9_000_000 vals = [float((i * 7) % 1000) for i in range(n)] - lo, hi = 0.0, 1000.0 + _EPS + lo, hi = 0.0, 1000.0 s = pl.Series("v", vals, dtype=pl.Float64) assert _fixed_hist_counts(s, lo, hi, n_bins) == _ref_hist_counts( vals, lo, hi, n_bins @@ -639,14 +635,14 @@ def test_threshold_boundary(self, n): """Both sides of MIN_PAR must agree — the split is an optimisation.""" vals = [float(i % 251) for i in range(n)] s = pl.Series("v", vals, dtype=pl.Float64) - lo, hi = 0.0, 251.0 + _EPS + lo, hi = 0.0, 251.0 assert _fixed_hist_counts(s, lo, hi, 32) == _ref_hist_counts(vals, lo, hi, 32) def test_values_outside_domain_clamp(self): n = _MIN_PAR + 64 vals = [float(i % 300) - 100.0 for i in range(n)] # spans [-100, 199] s = pl.Series("v", vals, dtype=pl.Float64) - lo, hi = 0.0, 100.0 + _EPS + lo, hi = 0.0, 100.0 counts = _fixed_hist_counts(s, lo, hi, 10) assert counts == _ref_hist_counts(vals, lo, hi, 10) assert sum(counts) == n, "out-of-domain values clamp, they are not dropped" @@ -668,7 +664,7 @@ def test_single_bin_holds_everything(self): n = _MIN_PAR + 17 vals = [float(i % 1000) for i in range(n)] s = pl.Series("v", vals, dtype=pl.Float64) - assert _fixed_hist_counts(s, 0.0, 1000.0 + _EPS, 1) == [n] + assert _fixed_hist_counts(s, 0.0, 1000.0, 1) == [n] # --------------------------------------------------------------------------- diff --git a/tests/test_browser.py b/tests/test_browser.py index c5936af7..93cde2db 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5747,11 +5747,11 @@ def test_hover_lookup_puts_edge_values_in_the_server_bin( from flexviz.server import register_source from flexviz.spec import encode_spec - # Grid [0.1, 0.7] in 3 bins: 0.3 is an inner edge, 0.7 the maximum. - # The 1-D grid pads its top edge, so there 0.3 sits just below the - # edge and only the kernel's round epsilon puts it in bin 1. - vals = [0.1] + [0.3] * 2 + [0.7] * 4 - probes = {0.3: 2, 0.7: 4} + # Grid [0.1, 0.4] in 3 bins: 0.2 is an inner edge, 0.4 the maximum. + # Float rounding puts 0.2 just below the inner edge in the kernel and + # in the hover lookup, so only their round epsilons put it in bin 1. + vals = [0.1] + [0.2] * 2 + [0.4] * 4 + probes = {0.2: 2, 0.4: 4} df = pl.DataFrame({"v": vals, "w": vals}) register_source("_browser_edges_lookup", df) dash = Dashboard(df) diff --git a/tests/test_browser_cube.py b/tests/test_browser_cube.py index 5242acc8..8aa954b1 100644 --- a/tests/test_browser_cube.py +++ b/tests/test_browser_cube.py @@ -88,7 +88,6 @@ encode_fvcube, ) from flexviz.trace.bin_grid import snap_range -from flexviz.trace.hist import _HIST_BIN_EPSILON from tests.test_browser import ( _COUNT_SETTLED_UPDATES, _hold_first_update, @@ -514,7 +513,7 @@ def _reference_slice_counts( column="b", kind="binned", bins=_TGT_BINS, - domain=(b_lo, b_hi + _HIST_BIN_EPSILON), + domain=(b_lo, b_hi), ), ), measure=MeasureSpec(agg="count"), @@ -554,7 +553,7 @@ def _reference_grouped_slice_counts( column="b", kind="binned", bins=_TGT_BINS, - domain=(b_lo, b_hi + _HIST_BIN_EPSILON), + domain=(b_lo, b_hi), ), TargetDimSpec(column=group_col, kind="categorical"), ), @@ -702,7 +701,7 @@ def test_commit_counts_match_reference(self, page: Page, server_port: int): column="b", kind="binned", bins=_TGT_BINS, - domain=(b_lo, b_hi + _HIST_BIN_EPSILON), + domain=(b_lo, b_hi), ), ), measure=MeasureSpec(agg="count"), @@ -1945,7 +1944,7 @@ def _reference_categorical_hist_counts( pl.col("b") .flexviz.fixed_hist( pl.lit(float(b_lo)), - pl.lit(float(b_hi + _HIST_BIN_EPSILON)), + pl.lit(float(b_hi)), n_bins=_TGT_BINS, ) .implode() @@ -2847,7 +2846,7 @@ def test_header_mismatch_demotes_instead_of_storing( column="b", kind="binned", bins=_TGT_BINS, - domain=(b_lo, b_hi + _HIST_BIN_EPSILON), + domain=(b_lo, b_hi), ), ), measure=MeasureSpec(agg="count"), @@ -3396,7 +3395,7 @@ def _hist_counts_ref( pl.col(col) .flexviz.fixed_hist( pl.lit(float(lo)), - pl.lit(float(hi) + _HIST_BIN_EPSILON), + pl.lit(float(hi)), n_bins=bins, ) .implode() @@ -4193,9 +4192,7 @@ def _hist_counts_ref_domain( .filter(filter_expr & pl.col(col).is_between(lo, hi)) .select( pl.col(col) - .flexviz.fixed_hist( - pl.lit(float(lo)), pl.lit(float(hi) + _HIST_BIN_EPSILON), n_bins=bins - ) + .flexviz.fixed_hist(pl.lit(float(lo)), pl.lit(float(hi)), n_bins=bins) .implode() .alias("h") ) @@ -4534,7 +4531,7 @@ def _reference_line_envelope( column="b", kind="binned", bins=_LINE_BUCKETS, - domain=(b_lo, b_hi + _HIST_BIN_EPSILON), + domain=(b_lo, b_hi), ) ] build_df = df @@ -5703,7 +5700,7 @@ def _reference_box2d_hist_counts( column="c", kind="binned", bins=_TGT_BINS, - domain=(c_lo, c_hi + _HIST_BIN_EPSILON), + domain=(c_lo, c_hi), ), ), measure=MeasureSpec(agg="count"), diff --git a/tests/test_cube.py b/tests/test_cube.py index 1c1794cc..1450ac1e 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -3182,14 +3182,12 @@ def _hist2d_target_spec( kind="binned", bins=_HIST2D_NB_X, domain=(x_lo, x_hi), - bin_variant="hist2d", ), TargetDimSpec( column="y", kind="binned", bins=_HIST2D_NB_Y, domain=(y_lo, y_hi), - bin_variant="hist2d", ), ), measure=measure, @@ -3281,7 +3279,7 @@ def test_cube_z_matches_to_update_full_range(self, histfunc, z_col): want = _hist2d_to_update_z(df, histfunc=histfunc, z_col=z_col, histnorm=None) assert got == want - def test_bin_variant_matches_kernel_on_edges(self): + def test_hist2d_bin_expr_matches_kernel_on_edges(self): # The cube hist2d bin expr must bin bit-equal to the fixed_hist2d # kernel, including a value exactly at the domain max, which the top # clamp folds into the top bin. Compare on a single axis. @@ -3410,43 +3408,6 @@ def test_deterministic_bytes(self): assert encode_fvcube(a, "k") == encode_fvcube(b, "k") -class TestHist2dTargetContentKey: - def test_bin_variant_does_not_change_key(self): - # Both variants bin with the same expression, so bin_variant cannot - # separate two dims that agree on column, kind, bins and domain. - df = _hist2d_target_df() - x_lo, x_hi = float(df["x"].min()), float(df["x"].max()) - y_lo, y_hi = float(df["y"].min()), float(df["y"].max()) - free = FreeAxisSpec(column="free", kind="continuous", p=64, domain=(0.0, 1.0)) - dims_2d = ( - TargetDimSpec( - column="x", - kind="binned", - bins=_HIST2D_NB_X, - domain=(x_lo, x_hi), - bin_variant="hist2d", - ), - TargetDimSpec( - column="y", - kind="binned", - bins=_HIST2D_NB_Y, - domain=(y_lo, y_hi), - bin_variant="hist2d", - ), - ) - dims_1d = ( - TargetDimSpec( - column="x", kind="binned", bins=_HIST2D_NB_X, domain=(x_lo, x_hi) - ), - TargetDimSpec( - column="y", kind="binned", bins=_HIST2D_NB_Y, domain=(y_lo, y_hi) - ), - ) - k2 = cube_content_key(CubeSpec(source_name="s", free=free, target_dims=dims_2d)) - k1 = cube_content_key(CubeSpec(source_name="s", free=free, target_dims=dims_1d)) - assert k2 == k1 - - # --------------------------------------------------------------------------- # Treemap target (Step 15): leaf-dims descriptor + finalize-then-sum rollup # --------------------------------------------------------------------------- diff --git a/tests/test_cube_server.py b/tests/test_cube_server.py index 57592e1e..10b00942 100644 --- a/tests/test_cube_server.py +++ b/tests/test_cube_server.py @@ -24,7 +24,6 @@ from flexviz.server import app, register_source from flexviz.spec import AxisRange from flexviz.trace.bin_grid import snap_range -from flexviz.trace.hist import _HIST_BIN_EPSILON pytestmark = pytest.mark.integration @@ -273,7 +272,7 @@ def test_blob_reslice_matches_direct_recompute(self, client, df): (dim,) = header["target_dims"] assert dim["name"] == "b" assert dim["bins"] == 12 - assert dim["domain"] == [b_lo, b_hi + _HIST_BIN_EPSILON] + assert dim["domain"] == [b_lo, b_hi] # Slice the blob over a snapped brush. lo_bin, hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 12.3, 61.7) @@ -301,7 +300,7 @@ def test_blob_reslice_matches_direct_recompute(self, client, df): pl.col("b") .flexviz.fixed_hist( pl.lit(float(b_lo)), - pl.lit(float(b_hi) + _HIST_BIN_EPSILON), + pl.lit(float(b_hi)), n_bins=dim["bins"], ) .implode() @@ -453,8 +452,8 @@ def test_active_source_column_mismatch_returns_empty(self, client, df): assert body["trace_cubes"] == {} def test_zoomed_viewports_resolve_cube_domains(self, client, df): - """Source viewport → free domain verbatim (no epsilon); target viewport - → the SNAPPED display grid + uniform _HIST_BIN_EPSILON.""" + """Source viewport → free domain verbatim; target viewport → the + SNAPPED display grid.""" spec = _two_hist_dashboard(df) src_fig_uid = spec.figures[0].uid tgt_fig_uid = spec.figures[1].uid @@ -469,7 +468,7 @@ def test_zoomed_viewports_resolve_cube_domains(self, client, df): (dim,) = header["target_dims"] lo, hi, n = snap_range(5.0, 60.0, tgt_bins) assert n == tgt_bins + 1 - assert dim["domain"] == [lo, hi + _HIST_BIN_EPSILON] + assert dim["domain"] == [lo, hi] assert dim["bins"] == n def test_reversed_viewports_resolve_like_ascending_ones(self, client, df): @@ -506,6 +505,34 @@ def test_zoomed_hist_cube_bars_land_on_the_display_bars(self, client, df): blob = body["cubes"][body["trace_cubes"][tgt_fig.traces[0].uid]] assert _cube_centers(blob) == pytest.approx(display, abs=1e-9) + def test_zoomed_hist_target_keeps_the_display_rows(self, client): + """A zoomed hist1d target's cube keeps the rows its display bars + count: a row just above the snapped top edge is in neither.""" + df = pl.DataFrame( + { + "a": [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], + "b": [20.0, 50.0, 99.0, 100.0, 100.0 + 5e-11, 110.0], + } + ) + register_source(_SRC, df, cache=True) + dash = Dashboard(df) + dash.add_figure().add_histogram(x="a", bins=16) + dash.add_figure().add_histogram(x="b", bins=20) + spec = dash.to_spec(source_name=_SRC) + src_fig, tgt_fig = spec.figures + spec.state.viewport[f"{tgt_fig.uid}/x"] = AxisRange(min=0.0, max=100.0) + + deltas = client.post("/dashboard/update", json=_init_payload(spec)).json() + display_rows = sum(deltas["figure_deltas"][tgt_fig.uid][0]["updates"]["y"]) + body = _cube_body( + client.post("/dashboard/update", json=_cube_payload(spec, src_fig.uid)) + ) + blob = base64.b64decode( + body["cubes"][body["trace_cubes"][tgt_fig.traces[0].uid]] + ) + cube_rows = sum(_read_u32_col(blob, decode_fvcube_header(blob), "count")) + assert display_rows == cube_rows == 4 + def test_sibling_hist_targets_share_the_display_bin_domain(self, client, df): """Two histograms on one figure bin over their *union* min/max in the legacy delta, so their cubes must resolve the same union — otherwise the @@ -821,7 +848,7 @@ def test_bar_source_serves_hist_and_bar_targets_count_parity( (dim,) = header["target_dims"] assert dim["name"] == "b" assert dim["bins"] == 12 - assert dim["domain"] == [b_lo, b_hi + _HIST_BIN_EPSILON] + assert dim["domain"] == [b_lo, b_hi] # Reslice over the codes for {("alpha",), ("gamma",)}. cats = [tuple(t) for t in header["free"]["categories"]] @@ -1014,7 +1041,7 @@ def test_grouped_hist_target_served_with_per_group_parity(self, cat_client, cat_ "name": "b", "kind": "binned", "bins": 8, - "domain": [b_lo, b_hi + _HIST_BIN_EPSILON], + "domain": [b_lo, b_hi], }, {"name": "sub", "kind": "categorical", "categories": ["s1", "s2", "s3"]}, ] @@ -1283,7 +1310,7 @@ def test_baked_parity_count_and_mean(self, cat_client, cat_df): # domain (+ epsilon) even though the passive filter shrinks the data. (dim,) = hist_header["target_dims"] b_lo, b_hi = cat_df["b"].min(), cat_df["b"].max() - assert dim["domain"] == [b_lo, b_hi + _HIST_BIN_EPSILON] + assert dim["domain"] == [b_lo, b_hi] free_bin = _read_u32_col(hist_blob, hist_header, "free_bin") tgt_bin = _read_u32_col(hist_blob, hist_header, "__bin__b") count = _read_u32_col(hist_blob, hist_header, "count") @@ -1296,7 +1323,7 @@ def test_baked_parity_count_and_mean(self, cat_client, cat_df): passive_expr & active_expr, "b", b_lo, - b_hi + _HIST_BIN_EPSILON, + b_hi, dim["bins"], ) assert 0 < sum(sliced) < cat_df.height diff --git a/tests/test_trace_hist.py b/tests/test_trace_hist.py index 739ea225..2a21794f 100644 --- a/tests/test_trace_hist.py +++ b/tests/test_trace_hist.py @@ -12,7 +12,7 @@ from flexviz.spec import TraceSpec from flexviz.trace import batch_fold as batch_fold_mod from flexviz.trace.bin_grid import snap_range -from flexviz.trace.hist import _HIST_BIN_EPSILON, Histogram, _streaming_hist_plan +from flexviz.trace.hist import Histogram, _streaming_hist_plan # ---- helpers --------------------------------------------------------------- @@ -240,7 +240,7 @@ def test_viewport_range_produces_correct_bin_centers(self, small_df: pl.DataFram centers = list(trace._to_update(df_agg).updates["x"]) lo, hi, n = snap_range(100.0, 400.0, bins) assert n == bins + 1 - step = (hi - lo + _HIST_BIN_EPSILON) / n + step = (hi - lo) / n expected = [lo + (i + 0.5) * step for i in range(n)] assert len(centers) == n for got, want in zip(centers, expected): @@ -375,9 +375,7 @@ def test_counts_equal_the_kernel_at_the_snapped_edges(self): ref = ( inside.select( pl.col("val") - .flexviz.fixed_hist( - pl.lit(lo), pl.lit(hi + _HIST_BIN_EPSILON), n_bins=n - ) + .flexviz.fixed_hist(pl.lit(lo), pl.lit(hi), n_bins=n) .implode() .alias("u") )["u"] @@ -751,7 +749,7 @@ def test_histogram_vertical_has_x_edges(self): assert n == 5, "the triple carries the bin count" assert step > 0 assert lo == pytest.approx(0.0) - assert lo + n * step == pytest.approx(19.0 + _HIST_BIN_EPSILON) + assert lo + n * step == pytest.approx(19.0) assert "y_edges" not in updates def test_histogram_horizontal_has_y_edges(self): @@ -767,7 +765,7 @@ def test_histogram_horizontal_has_y_edges(self): assert "x_edges" not in updates lo, step, n = updates["y_edges"] assert n == 4 - assert lo + n * step == pytest.approx(15.0 + _HIST_BIN_EPSILON) + assert lo + n * step == pytest.approx(15.0) def test_bin_edges_reproduce_the_bin_centers(self): """``lo + (i + 0.5) * step`` is exactly the emitted center of bin i.""" @@ -921,9 +919,9 @@ def test_target_spec_unzoomed_domain_none(self): assert spec is not None assert spec.target_dims[0].domain is None - def test_target_domain_is_the_snapped_range_no_epsilon(self): - # The ENGINE adds _HIST_BIN_EPSILON when resolving domains; the trace - # emits the snapped viewport, the display grid, and nothing else. + def test_target_domain_is_the_snapped_range(self): + # The trace emits the snapped viewport, the display grid, and nothing + # else. trace = Histogram(x="val", bins=10) spec = trace.get_cube_target_spec((100.0, 400.0)) assert spec is not None @@ -939,8 +937,7 @@ def test_target_grid_equals_the_display_grid(self): spec = trace.get_cube_target_spec(axis_range) lo, hi, n_bins, _ = trace._histogram_bounds_exprs(axis_range, None) dim = spec.target_dims[0] - # The engine pads the cube dim's hi exactly like the display path. - assert (dim.domain[0], dim.domain[1] + _HIST_BIN_EPSILON) == (lo, hi) + assert tuple(dim.domain) == (lo, hi) assert dim.bins == n_bins @pytest.mark.parametrize( @@ -964,7 +961,7 @@ def test_temporal_target_grid_equals_the_display_grid(self, viewport): ) dim = trace.get_cube_target_spec(cube_range, schema=schema).target_dims[0] lo, hi, n_bins, _ = trace._histogram_bounds_exprs(viewport, None, schema) - assert (dim.domain[0], dim.domain[1] + _HIST_BIN_EPSILON) == (lo, hi) + assert tuple(dim.domain) == (lo, hi) assert dim.bins == n_bins def test_grouped_hist_target_dims_binned_then_groups(self): @@ -1097,8 +1094,8 @@ def _both_updates( None, "count", ), - # A constant column is the narrowest span the trace can build: the - # engine pads hi by _HIST_BIN_EPSILON, so the span never inverts. + # A constant column has hi == lo: the kernel and the plan both put + # every row in bin 0. ("constant", pl.Series("v", [3.0] * 8), 8, None, "count"), ("viewport", pl.Series("v", _VALUES), 8, (2.0, 6.0), "count"), ("all_below_lo", pl.Series("v", [-5.0, -3.0]), 8, (0.0, 8.0), "count"), From 9d3975846f1c9c21794eb45a13ea1e187e846923 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 13:16:15 +0200 Subject: [PATCH 2/8] fix(hist): draw a Date bar at its bin center --- flexviz/trace/base.py | 9 ++++++++- tests/test_engine.py | 4 ++-- tests/test_trace_hist.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/flexviz/trace/base.py b/flexviz/trace/base.py index 865dd1d1..4f892b56 100644 --- a/flexviz/trace/base.py +++ b/flexviz/trace/base.py @@ -717,8 +717,15 @@ def _physical_to_temporal_series( values: Sequence[float] | pl.Series, dtype: pl.DataType, name: str = "" ) -> pl.Series: """Cast physical bin centers (floats in ``dtype``'s physical unit) back to - the temporal ``dtype`` (rounded to the nearest physical unit).""" + the temporal ``dtype`` (rounded to the nearest physical unit). + + A ``Date`` center is rarely a whole day, so it becomes a ``Datetime("ms")`` + instead: Plotly draws and highlights a bar at its center, and a + day-rounded center puts the bar up to half a day off its bin.""" s = values if isinstance(values, pl.Series) else pl.Series(name, list(values)) + if dtype == pl.Date: + ms = s * _phys_epoch_ms_factor(dtype) + return ms.round().cast(pl.Int64).cast(pl.Datetime("ms")) return s.round().cast(pl.Int64).cast(dtype) diff --git a/tests/test_engine.py b/tests/test_engine.py index efd9269d..8c323ba0 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1874,10 +1874,10 @@ def test_incompatible_coordinate_units_do_not_share_init_domain(self): assert 0 <= min(centers["value"]) <= max(centers["value"]) <= 99 assert ( - base.date() + base <= min(centers["day"]) <= max(centers["day"]) - <= (base.date() + dt.timedelta(days=99)) + <= (base + dt.timedelta(days=99)) ) for col in ("stamp_us", "stamp_ms"): assert ( diff --git a/tests/test_trace_hist.py b/tests/test_trace_hist.py index 2a21794f..fcf3e9ec 100644 --- a/tests/test_trace_hist.py +++ b/tests/test_trace_hist.py @@ -613,6 +613,18 @@ def test_date_column_counts_all(self): assert len(update["x"]) == 10 assert sum(update["y"]) == 50 + def test_date_bars_sit_on_their_bin_centers(self): + # Plotly draws and highlights a bar at its x. Seven days in 16 bins + # gives centers between whole days: a day-rounded x moves the bar. + import datetime as dt + + days = [dt.date(2020, 1, 1) + dt.timedelta(days=i) for i in range(7)] + df = pl.DataFrame({"t": pl.Series("t", days, dtype=pl.Date)}) + update = _aggregate_hist(df, bins=16, x="t") + lo, step, n = update["x_edges"] + want = [lo + (k + 0.5) * step for k in range(n)] + assert update["x"].dt.epoch("ms").to_list() == pytest.approx(want, abs=1) + def test_centers_are_temporal_for_date_axis(self): # Centers must be a temporal Series (not raw epoch ints) so the renderer # auto-detects a date axis, consistent with the line trace. From 8a6b33489160b2c2a92154f37fd23ce6cb254371 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Tue, 29 Sep 2026 21:57:04 +0200 Subject: [PATCH 3/8] fix(cube): bin range free axes like the display kernel and snap brushes to the nearest edge --- Architecture.md | 60 +-- CHANGELOG.md | 2 + benchmarks/test_codspeed_engine.py | 19 + docs/cube-support-overview.md | 2 +- flexviz/adapters/js/plotly/events.js | 192 +++++---- flexviz/adapters/js/runtime/cube.js | 159 ++++---- flexviz/cube.py | 285 ++++++------- flexviz_polars/src/expressions.rs | 43 +- flexviz_polars/tests/test_plugin_functions.py | 42 +- tests/test_browser_cube.py | 307 +++++++++----- tests/test_cube.py | 375 +++++++++++------- tests/test_cube_server.py | 113 +++--- 12 files changed, 910 insertions(+), 689 deletions(-) diff --git a/Architecture.md b/Architecture.md index 1d339665..25049ad2 100644 --- a/Architecture.md +++ b/Architecture.md @@ -773,9 +773,12 @@ FlexvizExprNamespace — registered as pl.Expr.flexviz via @pl.api.register_exp └── fixed_line_envelope2d(y_expr, free_expr, x_lo, x_hi, free_lo, free_hi, n_buckets, p) → pl.Expr One-pass exact argmin/argmax-by-y envelope per (x bucket, free bin) cell. Returns Struct{bucket, free_bin, y_min, x_at_ymin, y_max, x_at_ymax} with - one row per non-empty cell, sorted by (free_bin, bucket). Bin arithmetic is - the cube's natural floor on both axes, no epsilon and no clip; ties keep the - first row in scan order. Used by `cube.py` to build the `line_env` measure. + one row per non-empty cell, sorted by (free_bin, bucket). The x bucket is a + natural floor, no epsilon and no clip, so the domain max has a degenerate + top bucket; the free bin is the display kernel's rule (round epsilon, + clamp to p - 1). Rows outside either domain are dropped. Ties keep the + first row in scan order. Used by `cube.py` to build the `line_env` measure + from the raw free value. flexviz_polars._minmax_pairs_line(x_expr, y_expr, n_buckets, x_domain) → pl.Expr Bucket pass that keeps the pairing: one row per non-empty bucket, in bucket @@ -1077,15 +1080,16 @@ implemented. A cube is one target trace's grouping × the brushed (free) axis, holding decomposable partial measures. A **range** free axis (hist / box / line source) is binned to a **fixed resolution -P = 2048** over the source figure's viewport domain; a **box2d** free axis (a 2-D box-select on a -hist2d source) is two range axes binned at **P₂D = 128** each and packed into one composite -`free_bin`; a **categorical** free axis (bar/pie/treemap source) is the exact tuple of -label/path column values — no binning, no domain, dictionary-encoded in sorted order. Fixed P — -rather than Mosaic's pixel resolution — makes the cube width-independent, -content-addressable, and shareable across sessions; shipping the whole cube to the browser -(rather than slicing it server-side per frame) makes every drag step a local computation. Rows -with an out-of-domain or null free/target value are **filtered, not clipped** during the build; -a range value exactly at the domain max lands in the degenerate top bin `P`. +P = 2048** over the source figure's viewport domain. A **box2d** free axis (a 2-D box-select on a hist2d source) is two range axes binned at +**P₂D = 128** each and packed into one composite `free_bin`. A **categorical** free axis +(bar/pie/treemap source) is the exact tuple of label/path column values — no binning, no domain, +dictionary-encoded in sorted order. A resolution fixed by the spec — rather than Mosaic's pixel +resolution — makes the cube width-independent, content-addressable, and shareable across +sessions; shipping the whole cube to the browser (rather than slicing it server-side per frame) +makes every drag step a local computation. Rows with an out-of-domain or null free/target value +are **filtered, not clipped** during the build. A range free axis bins like the display kernel +(`_fixed_hist_bin_expr`: round-epsilon, top clamp), so a value on a bin edge lands in the bin the +source draws it in and the domain max lands in the top bin `P-1`. #### Measures (partial algebra) @@ -1167,8 +1171,16 @@ range **or categorical (bar)** selection geometry: e.g. a numeric label dtype the client cannot see), so one incapable target never pins the gesture to mouseup-only. A failed/timed-out request degrades the gesture to mouseup-only. 2. **Each further `plotly_selecting`** (rAF-throttled, superseded frames dropped) re-slices - locally. A range source snaps the in-progress range outward to the P-grid - (`lo_bin = floor((a-lo)/span·P)` clamped to `[0, P]`); a categorical source matches the + locally. A range source selects each bin whose center, as the display draws it, the brush covers, ends included + (`fvCubeSnap` / `snap_brush`) — Plotly's highlight rule for bars. A frame whose brush covers no center shows the targets' + pre-gesture state (in overlay mode without the live fg presentation), and its commit clears + the figure's selection. A box2d source does the same + per axis. The commit writes the **kernel's bin boundaries**, not + `lo + k·step`: the first edge is `lo`, an inner edge the first double the kernel puts in bin + `k`, found by bisecting the doubles around the float `lo + (k − 1e-9)·step` along the kernel's own rule (a bounded search: near a zero crossing billions of doubles share one value of `v − lo`), and + a brush reaching the top bin ends at the domain top with `closed="both"`, otherwise + `closed="left"`. So a committed range keeps exactly the values the display counts in its bars. The redrawn selection box sits on the same bin + edges, so Plotly's highlight stays on the committed bins. A categorical source matches the covered labels to category codes (deduped per frame on the sorted label set). The slice accumulates partials per composite target key over CSR row ranges and finalizes per the measure table; per-trace delta synthesis mirrors the Python `_to_update`/`_to_grouped_update` @@ -1284,17 +1296,19 @@ flips the renderer into the ghost+fg presentation before any selection exists **Temporal sources.** Temporal free axes (and temporal binned target dims) carry a physical `unit` in the FVCube header — `us`/`ms` for `Datetime`, `day` for `Date`, derived from the schema dtype by the engine; `Datetime("ns")` and `Time` gate to no cube (the string round-trip -is µs-precision). `unit:"day"` switches to an **integer-day snap grid** (`day_grid`: width -`w = max(1, ceil(span/2048))` whole days, `P' = ceil(span/w)` bins, header carries `w`/`p_eff`) -so `YYYY-MM-DD` edges round-trip bit-exactly; `us`/`ms` keep P=2048. Client side: temporal +is µs-precision). Every unit uses the same grid as a numeric axis; edges may fall between two +days. Client side: temporal viewports key as self-consistent epoch-ms tokens (never sent to the server — the server parses the original date strings via the schema dtype in `_cube_axis_range`); the snap grid is adopted from the decoded header; drag ranges convert through `fvTemporalToPhysical` (manual UTC parse, -never bare `Date.parse`); commits emit snapped `closed="left"` **string** ranges rendered by -`fvPhysicalToTemporal` — the edge is ceil-ed to an integral count of the unit, which preserves -integer membership of the half-open range and makes the string parse back exactly through -`_typed_range_bounds`. The `plotly_selected` echo guard converts both sides to physical before -its half-bin comparison. +never bare `Date.parse`); commits emit snapped **string** ranges rendered by +`fvPhysicalToTemporal`, which ceils: the ceil of an exact edge is the first whole unit the +kernel puts in its bin, and a closed upper edge (the top bin) rounds down to the last unit. So +an integer row stays in the bar the display counts it in, and the string parses back exactly +through `_typed_range_bounds`. The selection box keeps each +bin edge's time of day instead (a µs string): a whole-day edge can pass the center of a bar +narrower than two days. The `plotly_selected` echo guard converts both sides to physical before +its half-bin comparison (half the source step). **Temporal binned *target* dims.** A binned target dim over a temporal column is built on the column's physical representation (epoch µs/ms, day index) and the header ships its `unit`. Most @@ -1344,7 +1358,7 @@ them. | hist | 1-D range, P=2048 over the viewport (or server-resolved full) domain; continuous + temporal (`us`/`ms`/`day` physical units — see “Temporal sources” below; `Datetime("ns")`/`Time` gate to no cube) | | box | 1-D range over the `data_col` (same shape/gates as hist) | | line | 1-D range over the **x** column only (line selection is x-only — see below); P=2048; source geometry independent of `downsample` | - | hist2d | **box2d**: two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` (`bin_y·(P₂D+1) + bin_x`); per-axis domains resolved by the engine; a rectangle brush slices a 2-D sub-grid | + | hist2d | **box2d**: two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` (`bin_y·P₂D + bin_x`); per-axis domains resolved by the engine; a rectangle brush slices a 2-D sub-grid | | bar / pie | categorical over the ordered label columns (`axis_range` ignored — label geometry is viewport-independent) | | treemap | categorical over the full `path` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9657afd7..6b3c5c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ independently. `flexviz` pins a compatible `flexviz-polars` range. ### Changed +- `flexviz` requires the `flexviz-polars` release that ships with it, because + the line envelope kernel now bins the free axis like the display kernel. - A notebook `show()` displays the server's own `/view` page in its iframe, like the browser path, instead of a `data:` page that called the server from another origin. For the deprecated ECharts renderer, `height` now sets only diff --git a/benchmarks/test_codspeed_engine.py b/benchmarks/test_codspeed_engine.py index 5dac8c34..a4178669 100644 --- a/benchmarks/test_codspeed_engine.py +++ b/benchmarks/test_codspeed_engine.py @@ -88,3 +88,22 @@ def test_build_cubes(benchmark, frame: pl.DataFrame) -> None: active = ActiveSource(figure_uid="fig_a", column="g", trace_uid=bar.uid) cubes, trace_cubes = benchmark(engine.build_cubes, infos, {}, [], active) assert cubes and trace_cubes == {hist.uid: 0} + + +def test_build_line_cubes(benchmark, frame: pl.DataFrame) -> None: + """Histogram source (a range free axis), line target: the envelope cube, + wrapper and kernel together.""" + hist = Histogram(x="z", bins=50) + line = TRACES["line-minmax"]() + engine = FlexEngine( + backend_lf=LFQueryBuilder(frame), + scalable_traces={hist.uid: hist, line.uid: line}, + source_name="bench", + ) + infos = [ + TraceInfo(hist.uid, hist._axes, hist.trace_type, figure_uid="fig_a"), + TraceInfo(line.uid, line._axes, line.trace_type, figure_uid="fig_b"), + ] + active = ActiveSource(figure_uid="fig_a", column="z", trace_uid=hist.uid) + cubes, trace_cubes = benchmark(engine.build_cubes, infos, {}, [], active) + assert cubes and trace_cubes == {line.uid: 0} diff --git a/docs/cube-support-overview.md b/docs/cube-support-overview.md index e99752e7..95d7aeb4 100644 --- a/docs/cube-support-overview.md +++ b/docs/cube-support-overview.md @@ -73,7 +73,7 @@ source free-axis kinds can feed each target measure.** Notes on the two `❌` columns: - **`box2d × {line_env, corr}` (#47):** a feasibility-free but real **cell-count / wire-size - wall** — a box2d `line_env` cube is `n_x_buckets × (P₂D+1)²` ≈ `500 × 129²` ≈ 8.3M cells per + wall** — a box2d `line_env` cube is `n_x_buckets × P₂D²` ≈ `500 × 128²` ≈ 8.2M cells per line. Gated off; those targets fall back to the per-commit recompute. Tracked in #47. - **`median` / `n_unique` / box quantiles:** mathematically not decomposable over a partition — never cube targets, by any source. diff --git a/flexviz/adapters/js/plotly/events.js b/flexviz/adapters/js/plotly/events.js index e2d1f02d..8b560e12 100644 --- a/flexviz/adapters/js/plotly/events.js +++ b/flexviz/adapters/js/plotly/events.js @@ -389,9 +389,10 @@ function _rangeSelectionPredicates(eventData, figUid) { // A drag in select mode becomes a *gesture*: the first plotly_selecting event // resolves the cube descriptor set and checks the client store (one // cube_request POST on a miss); each further selecting event re-slices -// locally, rAF-throttled — a range source snaps the in-progress range to the -// P-grid, a categorical (bar) source matches the covered labels to category -// codes. Commit (plotly_selected) snaps range predicates (closed="left"); +// locally, rAF-throttled — a range source snaps the in-progress range to its +// free-axis grid, a categorical (bar) source matches the covered labels to +// category codes. Commit (plotly_selected) snaps range predicates to the +// kernel's bin boundaries; // categorical predicates stay the legacy is_in shape byte-for-byte. Either // way the server round-trip is skipped entirely when every cross-filter // target was cube-served. Pie / treemap click commits run the same @@ -416,6 +417,7 @@ function fvCubeOverlayFgActive(figUid) { for (const gesture of Object.values(_fvCubeGestures)) { if (!gesture || gesture.inert || !gesture.live) continue; if (!(gesture.lastBins || gesture.lastKey)) continue; // no frame applied + if (gesture.emptyFrame) continue; // shows the pre-gesture state if ((gesture.targets || []).some(t => t.capable && t.figUid === figUid)) { return true; } @@ -818,27 +820,17 @@ function _fvCubeGestureStart(figUid, eventData) { } // Adopt a decoded range free header (or a remembered free record) as the -// gesture's snap grid: the physical domain, the temporal unit, and the -// integer-day grid for unit:"day" (contract G). Idempotent — first adoption -// wins; numeric zoomed gestures already hold their viewport snapDomain. +// gesture's snap grid: the physical domain and the temporal unit (contract +// G). Idempotent — first adoption wins; numeric zoomed gestures already hold +// their viewport snapDomain. function _fvCubeAdoptFreeGrid(gesture, free) { if (!free || free.kind === 'categorical' || !Array.isArray(free.domain)) return; if (!gesture.snapDomain) gesture.snapDomain = free.domain.slice(); if (free.unit && !gesture.unit) gesture.unit = free.unit; - if (free.unit === 'day' && !gesture.dayGrid) { - gesture.dayGrid = { w: free.w, pEff: free.p_eff }; - } } -// Snap a (physical) range on the gesture's grid: the integer-day grid for a -// unit:"day" source, the shared P-grid arithmetic otherwise — the latter -// byte-identical to Phases 1–2 (the edge arithmetic is parity-pinned). +// Snap a (physical) range on the gesture's P-grid. function _fvCubeGestureSnap(gesture, a, b) { - if (gesture.dayGrid) { - return fvCubeSnapDay( - gesture.snapDomain[0], gesture.dayGrid.w, gesture.dayGrid.pEff, a, b - ); - } return fvCubeSnap(gesture.snapDomain, _FV_CUBE_P, a, b); } @@ -864,7 +856,6 @@ function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { // never a snap domain (the column's physical unit may differ). snapDomain: (!temporalViewport && sourceDomain) || null, unit: null, - dayGrid: null, targets: _fvCubeEnumerateTargets(figUid, figSpec.source, freeDesc, passiveKey), live: false, pendingRange: null, @@ -888,32 +879,23 @@ function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { } // Adopt a decoded box2d free header (contract H) as the gesture's per-axis -// snap grid: each axis's physical domain, temporal unit, and integer-day grid -// (from the header's "units"/"grids" 2-element lists). Idempotent — first -// adoption per axis wins; zoomed numeric axes already hold their viewport -// snapDomain. +// snap grid: each axis's physical domain and temporal unit (from the header's +// "units" list). Idempotent — first adoption per axis wins; zoomed numeric +// axes already hold their viewport snapDomain. function _fvCubeAdoptBox2dGrid(gesture, free) { if (!free || free.kind !== 'box2d' || !Array.isArray(free.domains)) return; const units = free.units || [null, null]; - const grids = free.grids || [null, null]; for (let a = 0; a < 2; a++) { const ax = gesture.axes[a]; const dom = free.domains[a]; if (!Array.isArray(dom)) continue; if (!ax.snapDomain) ax.snapDomain = dom.slice(); if (units[a] && !ax.unit) ax.unit = units[a]; - if (units[a] === 'day' && !ax.dayGrid && grids[a]) { - ax.dayGrid = { w: grids[a].w, pEff: grids[a].p_eff }; - } } } -// Snap one box2d axis on its grid (integer-day grid for a unit:"day" axis, -// the shared P=128 arithmetic otherwise). Returns the fvCubeSnap result. +// Snap one box2d axis on its P=128 grid. Returns the fvCubeSnap result. function _fvCubeBox2dSnapAxis(ax, a, b) { - if (ax.dayGrid) { - return fvCubeSnapDay(ax.snapDomain[0], ax.dayGrid.w, ax.dayGrid.pEff, a, b); - } return fvCubeSnap(ax.snapDomain, _FV_CUBE_BOX2D_P, a, b); } @@ -947,8 +929,8 @@ function _fvCubeBox2dGestureStart(figUid, figSpec, constraint) { // Per-axis snap grids. A zoomed NUMERIC viewport is the snap domain // directly; a temporal/unzoomed axis defers to the decoded header. axes: [ - { snapDomain: (!tempX && domX) || null, unit: null, dayGrid: null }, - { snapDomain: (!tempY && domY) || null, unit: null, dayGrid: null }, + { snapDomain: (!tempX && domX) || null, unit: null }, + { snapDomain: (!tempY && domY) || null, unit: null }, ], targets: _fvCubeEnumerateTargets(figUid, figSpec.source, freeDesc, passiveKey), live: false, @@ -1212,6 +1194,17 @@ function _fvCubeApplyTargetSlices(targets, rangesForEntry, gesture) { return dirty; } +// Put back the pre-drag layer data of every target the live loop touched. +// Returns the figures to redraw. +function _fvCubeRestoreSavedLayers(gesture) { + const dirty = new Set(); + for (const saved of Object.values(gesture.savedLayers || {})) { + saved.restore(); + dirty.add(saved.figUid); + } + return dirty; +} + // Snap the drag range and slice every cube-served target (range gestures); // de-dupes on unchanged snapped bins. Temporal sources convert the drag's // date strings to the cube's physical unit first (contract G). @@ -1227,6 +1220,15 @@ function _fvCubeApplyRange(gesture, range) { && gesture.lastBins[1] === snap.hiBin ) return; gesture.lastBins = [snap.loBin, snap.hiBin]; + // A brush that covers no bin center shows the pre-gesture state: its commit + // clears the selection, so an empty slice would flash every target empty + // first. The flag also keeps an overlay target off its live fg + // presentation (fvCubeOverlayFgActive). + gesture.emptyFrame = snap.hiBin < snap.loBin; + if (gesture.emptyFrame) { + _fvCubeRenderFigures(_fvCubeRestoreSavedLayers(gesture)); + return; + } const dirty = _fvCubeApplyTargetSlices( gesture.targets, () => [[snap.loBin, snap.hiBin]], gesture ); @@ -1235,7 +1237,7 @@ function _fvCubeApplyRange(gesture, range) { // Snap a box2d gesture's drag box on BOTH axes (contract H). Temporal axes // convert their date-string range to the axis's physical unit first. Returns -// the fvCubeSnap2d-shaped result, or null when a range is malformed. +// {x, y} fvCubeSnap results, or null when a range is malformed. function _fvCubeBox2dSnap(gesture, range) { if (!Array.isArray(range.x) || range.x.length !== 2) return null; if (!Array.isArray(range.y) || range.y.length !== 2) return null; @@ -1266,6 +1268,13 @@ function _fvCubeApplyBox2d(gesture, range) { x: [snap.x.loBin, snap.x.hiBin], y: [snap.y.loBin, snap.y.hiBin], }; + // Covering no bin center on either axis is an empty frame, as in + // _fvCubeApplyRange. + gesture.emptyFrame = snap.x.hiBin < snap.x.loBin || snap.y.hiBin < snap.y.loBin; + if (gesture.emptyFrame) { + _fvCubeRenderFigures(_fvCubeRestoreSavedLayers(gesture)); + return; + } const dirty = _fvCubeApplyTargetSlices( gesture.targets, entry => fvCubeRectRanges(entry.header, snap), gesture ); @@ -1318,11 +1327,7 @@ function _fvCubeGestureTake(figUid) { function _fvCubeGestureAbort(figUid) { const gesture = _fvCubeGestureTake(figUid); if (!gesture || (!gesture.savedLayers && !gesture.createdBg)) return; - const dirty = new Set(); - for (const saved of Object.values(gesture.savedLayers || {})) { - saved.restore(); - dirty.add(saved.figUid); - } + const dirty = _fvCubeRestoreSavedLayers(gesture); for (const created of (gesture.createdBg || [])) { for (const restore of created.restores) restore(); bgYExtentByFig[created.figUid] = created.prevYExtent; @@ -1364,7 +1369,7 @@ function handleSelecting(eventData, figUid) { // outline (group move) or its edge handles (resize) arms a rAF loop that // converts the outline path's bbox back to data coordinates and replays it // through handleSelecting — the same gesture path as a fresh draw, so cube -// liveness gating, P-grid snapping, and the conditional commit apply +// liveness gating, grid snapping, and the conditional commit apply // unchanged, and without cube coverage the gesture stays mouseup-only // exactly as today. A categorical (bar) source has no range gesture: its // replay resolves the covered bars from the outline span and feeds them as @@ -1525,12 +1530,14 @@ function handleSelectionEditPointerDown(evt, figUid) { rafId = requestAnimationFrame(frame); } -// On commit, replace the raw range predicate with the snapped closed="left" -// one (feature-level snap policy, spec §8.2 — applies to every gesture on -// cube-eligible source geometry while live_brush is "auto", live or not) and -// report whether the commit can stay local. Returns null when no gesture ran -// (programmatic selections stay byte-for-byte legacy) or no snap domain is -// known (cold failed request on an unzoomed axis degrades to unsnapped). +// On commit, replace the raw range predicate with the snapped one: the +// kernel's bin boundaries, closed="left", or closed="both" when it reaches the +// top bin (feature-level snap policy, spec §8.2 — applies to every gesture on +// cube-eligible source geometry while live_brush is "auto", live or not), and +// report whether the commit can stay local. Returns {clear: true} when the +// brush covers no bin center, and null when no gesture ran (programmatic +// selections stay byte-for-byte legacy) or no grid is known yet (a commit +// before the first cube header lands stays unsnapped). function _fvCubeCommitOverride(figUid, range, box) { const gesture = _fvCubeGestureTake(figUid); if (!gesture || gesture.inert) return null; @@ -1545,28 +1552,24 @@ function _fvCubeCommitOverride(figUid, range, box) { if (gesture.unit) r = r.map(v => fvTemporalToPhysical(v, gesture.unit)); if (!r.every(Number.isFinite)) return null; const snap = _fvCubeGestureSnap(gesture, r[0], r[1]); - const edgeLoOut = gesture.unit - ? fvPhysicalToTemporal(snap.edgeLo, gesture.unit) - : snap.edgeLo; - const edgeHiOut = gesture.unit - ? fvPhysicalToTemporal(snap.edgeHi, gesture.unit) - : snap.edgeHi; + if (snap.hiBin < snap.loBin) return { clear: true }; + const [edgeLoOut, edgeHiOut] = _fvCubeCommitEdges(snap, gesture.unit); const predicates = [{ clauses: [{ column: gesture.column, range: [edgeLoOut, edgeHiOut], - closed: 'left', + closed: snap.closed, }], }]; - // The stored/rendered selection shape uses the same snapped edges. + const [boxLo, boxHi] = _fvCubeBoxEdges(snap, gesture.unit); const snappedBox = { ...(box || {}) }; if (gesture.role === 'x') { - snappedBox.x0 = edgeLoOut; - snappedBox.x1 = edgeHiOut; + snappedBox.x0 = boxLo; + snappedBox.x1 = boxHi; snappedBox.xref = snappedBox.xref || 'x'; } else { - snappedBox.y0 = edgeLoOut; - snappedBox.y1 = edgeHiOut; + snappedBox.y0 = boxLo; + snappedBox.y1 = boxHi; snappedBox.yref = snappedBox.yref || 'y'; } if (gesture.live) { @@ -1577,6 +1580,27 @@ function _fvCubeCommitOverride(figUid, range, box) { return { predicates, box: snappedBox, skipPost: _fvCubeSkipPost(gesture) }; } +// A snap's committed edges in the clause's value space. A temporal edge is a +// string of whole units: fvPhysicalToTemporal ceils, which gives the first +// unit in the bin; a closed upper edge (the top bin) rounds down instead. +function _fvCubeCommitEdges(snap, unit) { + if (!unit) return [snap.edgeLo, snap.edgeHi]; + const hi = snap.closed === 'both' ? Math.floor(snap.edgeHi) : snap.edgeHi; + return [fvPhysicalToTemporal(snap.edgeLo, unit), fvPhysicalToTemporal(hi, unit)]; +} + +// A snap's edges for the stored and rendered selection box: the bin edges +// themselves, in the axis's value space. Plotly highlights a bar when its +// center is inside the box, so the highlight stays on the committed bins. A +// whole-day commit edge can pass the center of a bar that is less than two +// days wide, so a temporal box edge keeps its time of day (µs string). +function _fvCubeBoxEdges(snap, unit) { + if (!unit) return [snap.edgeLo, snap.edgeHi]; + return [snap.edgeLo, snap.edgeHi].map( + v => fvPhysicalToTemporal(fvPhysicalToEpochMs(v, unit) * 1000, 'us') + ); +} + // Conditional commit predicate (shared by every range/box2d gesture): local // only when EVERY trace in EVERY other figure was cube-served this gesture. // Any non-capable target (box plot, median bar, a server-demoted target) @@ -1595,30 +1619,32 @@ function _fvCubeSkipPost(gesture) { } // Commit a box2d gesture (contract H): ONE predicate with TWO snapped -// closed="left" clauses (x and y) — today's two-clause hist2d selection -// shape, snapped to the cube grid. The stored/rendered box carries the same -// snapped edges on both axes. Conditional-commit skipPost is shared +// clauses (x and y) — today's two-clause hist2d selection shape, snapped to +// the cube grid. The stored/rendered box sits on the bin edges of both axes +// (_fvCubeBoxEdges). Conditional-commit skipPost is shared // (_fvCubeSkipPost). Returns null if either axis has no snap domain (cold // degrade) or the range is malformed. function _fvCubeBox2dCommitOverride(gesture, range, box) { if (!gesture.axes[0].snapDomain || !gesture.axes[1].snapDomain) return null; const snap = _fvCubeBox2dSnap(gesture, range); if (!snap) return null; - const edge = (ax, e) => (ax.unit ? fvPhysicalToTemporal(e, ax.unit) : e); - const exLo = edge(gesture.axes[0], snap.x.edgeLo); - const exHi = edge(gesture.axes[0], snap.x.edgeHi); - const eyLo = edge(gesture.axes[1], snap.y.edgeLo); - const eyHi = edge(gesture.axes[1], snap.y.edgeHi); + if (snap.x.hiBin < snap.x.loBin || snap.y.hiBin < snap.y.loBin) { + return { clear: true }; + } + const [exLo, exHi] = _fvCubeCommitEdges(snap.x, gesture.axes[0].unit); + const [eyLo, eyHi] = _fvCubeCommitEdges(snap.y, gesture.axes[1].unit); const predicates = [{ clauses: [ - { column: gesture.cols[0], range: [exLo, exHi], closed: 'left' }, - { column: gesture.cols[1], range: [eyLo, eyHi], closed: 'left' }, + { column: gesture.cols[0], range: [exLo, exHi], closed: snap.x.closed }, + { column: gesture.cols[1], range: [eyLo, eyHi], closed: snap.y.closed }, ], }]; + const [bxLo, bxHi] = _fvCubeBoxEdges(snap.x, gesture.axes[0].unit); + const [byLo, byHi] = _fvCubeBoxEdges(snap.y, gesture.axes[1].unit); const snappedBox = { ...(box || {}), - x0: exLo, x1: exHi, xref: (box && box.xref) || 'x', - y0: eyLo, y1: eyHi, yref: (box && box.yref) || 'y', + x0: bxLo, x1: bxHi, xref: (box && box.xref) || 'x', + y0: byLo, y1: byHi, yref: (box && box.yref) || 'y', }; if (gesture.live) _fvCubeApplyBox2d(gesture, range); return { predicates, box: snappedBox, skipPost: _fvCubeSkipPost(gesture) }; @@ -1700,7 +1726,7 @@ function _fvCubeClickCommit(figUid, tsSpec, predicates) { // re-emit plotly_selected (asynchronously) with a pixel-roundtripped range. // With a snapped cube commit the stored box is no longer pixel-aligned, so // the echoed range drifts off the stored edges and would re-enter the legacy -// path as a "new" unsnapped selection (replacing the committed closed="left" +// path as a "new" unsnapped selection (replacing the committed snapped // predicate and double-POSTing). Discriminator: no live-brush gesture ran // (programmatic re-emission never fires plotly_selecting) AND the range // matches the stored selection box within half a snap bin — below snap @@ -1729,31 +1755,27 @@ function _fvCubeEchoOfStoredSelection(figUid, range) { const tolByRole = { x: 0, y: 0 }; const unitByRole = { x: null, y: null }; // A box2d source's per-axis snap grid is remembered as a box2d free header - // (units/grids/domains lists), not a per-column free record; resolve once. - const box2dGrid = constraint && constraint.kind === 'box2d' && figSpec && figSpec.source + // (units/domains lists), not a per-column free record; resolve once. + const isBox2d = !!constraint && constraint.kind === 'box2d'; + const box2dGrid = isBox2d && figSpec && figSpec.source ? _fvCubeRememberedBox2dGrid(figSpec.source, [constraint.x.column, constraint.y.column]) : null; for (let ci = 0; ci < axisConstraints.length; ci++) { const ac = axisConstraints[ci]; - const isBox2d = constraint.kind === 'box2d'; const p = isBox2d ? _FV_CUBE_BOX2D_P : _FV_CUBE_P; - // 1-D sources remember a per-(source, column) free record; box2d axes read - // their {domain, unit, grid} from the remembered box2d header by index. + // 1-D sources remember a per-(source, column, p) free record; box2d axes + // read their {domain, unit} from the remembered box2d header by index. let recDomain = null; let unit = null; - let dayW = null; if (isBox2d && box2dGrid) { recDomain = (box2dGrid.domains || [])[ci] || null; unit = (box2dGrid.units || [])[ci] || null; - const grid = (box2dGrid.grids || [])[ci]; - dayW = grid && grid.w; } else if (!isBox2d) { const rec = figSpec && figSpec.source ? fvCubeFreeDomain(figSpec.source, ac.column, p) : null; recDomain = rec && rec.domain; unit = (rec && rec.unit) || null; - dayW = rec && rec.w; } unitByRole[ac.role] = unit; let domain = _fvCubeViewportDomain(figUid, ac.anchor); @@ -1766,9 +1788,7 @@ function _fvCubeEchoOfStoredSelection(figUid, range) { domain = recDomain; } if (Array.isArray(domain)) { - tolByRole[ac.role] = unit === 'day' && dayW - ? dayW / 2 - : (domain[1] - domain[0]) / p / 2; + tolByRole[ac.role] = (domain[1] - domain[0]) / p / 2; } } // The echoed values are date strings on temporal axes — convert both @@ -1833,6 +1853,12 @@ function handleSelected(eventData, figUid) { plotlySelectionBox = _plotlySelectionBoxFromRange(eventData, figUid); if (predicates) { const cubeCommit = _fvCubeCommitOverride(figUid, eventData.range, plotlySelectionBox); + if (cubeCommit && cubeCommit.clear) { + // The brush covers no bin center, so it selects nothing. + clearFigureSelection(figUid); + _reapplyCanonicalSelectionBoxes(figUid); + return; + } if (cubeCommit) { predicates = cubeCommit.predicates; plotlySelectionBox = cubeCommit.box; diff --git a/flexviz/adapters/js/runtime/cube.js b/flexviz/adapters/js/runtime/cube.js index cbf6a465..793c1cd4 100644 --- a/flexviz/adapters/js/runtime/cube.js +++ b/flexviz/adapters/js/runtime/cube.js @@ -155,8 +155,7 @@ function fvTemporalToPhysical(value, unit) { // unchanged (lo: v >= e ⟺ v >= ceil(e); hi open: v < e ⟺ v < ceil(e)) and // the string then represents the bound exactly — the server's // _typed_range_bounds parses it back bit-exactly (§8.2 round-trip). unit -// "day" renders YYYY-MM-DD (edges are integer days by the day_grid -// construction); us/ms render µs-precision datetimes. +// "day" renders YYYY-MM-DD; us/ms render µs-precision datetimes. function fvPhysicalToTemporal(value, unit) { if (unit === 'day') { return new Date(Math.ceil(value) * 86400000).toISOString().slice(0, 10); @@ -187,28 +186,6 @@ function fvPhysicalToEpochMs(value, unit) { return value; // 'ms' } -// JS mirror of Python's day_grid (contract G): integer-day snap grid for -// unit:"day" free axes — w whole days per bin, P' = ceil(span/w) bins. -function fvCubeDayGrid(lo, hi, p) { - const span = hi - lo; - const w = Math.max(1, Math.ceil(span / p)); - return { w, pEff: Math.max(0, Math.ceil(span / w)) }; -} - -// Day-grid snap: same shape as fvCubeSnap but over the integer-day grid -// (width w, count pEff) — all arithmetic exact over integer days. -function fvCubeSnapDay(lo, w, pEff, a, b) { - let loBin = Math.max(0, Math.min(pEff, Math.floor((a - lo) / w))); - let hiBin = Math.max(0, Math.min(pEff, Math.floor((b - lo) / w))); - if (hiBin < loBin) { const t = loBin; loBin = hiBin; hiBin = t; } - return { - loBin, - hiBin, - edgeLo: lo + loBin * w, - edgeHi: lo + (hiBin + 1) * w, - }; -} - // Mirror of the trace classes' overlay_style for the engine's active-selection // rule (filtered_only traces do not recompute bg while fg is active). overlay_style is // not serialized in TraceSpec — it is a pure function of trace_type (plus @@ -340,8 +317,7 @@ fvCacheReset = function () { // Server-resolved full-data free blocks seen in decoded headers, keyed by // (source, column, p). Lets a later gesture on the same unzoomed axis snap // even before its own cube response lands. Each record carries the -// {domain, unit, w, p_eff} subset of the header free block (unit/w/p_eff -// null/undefined for plain continuous axes). +// {domain, unit} subset of the header free block. const _fvCubeFreeDomains = new Map(); function fvCubeRememberFreeDomain(sourceName, column, p, free) { @@ -349,8 +325,6 @@ function fvCubeRememberFreeDomain(sourceName, column, p, free) { _fvCubeFreeDomains.set(JSON.stringify([sourceName, column, p]), { domain: [free.domain[0], free.domain[1]], unit: free.unit || null, - w: free.w, - p_eff: free.p_eff, }); } @@ -422,24 +396,9 @@ function decodeFVCube(bytes) { } // CSR offsets over the free_bin-sorted rows: rows of bin b are - // binStart[b] .. binStart[b+1]-1. Range kinds span b in 0..P inclusive - // (degenerate top bin P included), hence P+2 slots — P being the - // EFFECTIVE bin count (p_eff for an integer-day grid, p otherwise); a - // categorical free axis has exactly one bin per category tuple (no - // degenerate bin). A box2d free axis (contract H) has composite bins - // by*S + bx with S = p+1, so every index 0..S²-1 is reachable ⇒ S² + 1 - // slots (the prefix-sum entry indexes every composite bin). - let slots; - if (header.free.kind === 'categorical') { - slots = header.free.categories.length + 1; - } else if (header.free.kind === 'box2d') { - const s = header.free.p + 1; - slots = s * s + 1; - } else { - slots = (header.free.p_eff ?? header.free.p) + 2; - } + // binStart[b] .. binStart[b+1]-1, for every bin 0.._fvCubeMaxBin. const freeBin = cols.free_bin; - const binStart = new Uint32Array(slots); + const binStart = new Uint32Array(_fvCubeMaxBin(header) + 2); for (let i = 0; i < freeBin.length; i++) binStart[freeBin[i] + 1]++; for (let b = 1; b < binStart.length; b++) binStart[b] += binStart[b - 1]; bytesTotal += binStart.byteLength; @@ -494,59 +453,95 @@ function decodeCubeBundle(arrayBuffer) { // Snap + slice + delta (Shared arithmetic; mirrors Histogram._to_update) // --------------------------------------------------------------------------- -// Natural-floor bin indices of the brush endpoints, clamped to [0, P] — P -// included so a brush reaching the domain max selects the degenerate top bin. -// edge(b) = lo + b*span/P; committed range = [edge(loBin), edge(hiBin+1)). -function fvCubeSnap(domain, p, a, b) { - const lo = domain[0]; - const span = (domain[1] - lo) || 1.0; - let loBin = Math.max(0, Math.min(p, Math.floor((a - lo) / span * p))); - let hiBin = Math.max(0, Math.min(p, Math.floor((b - lo) / span * p))); - if (hiBin < loBin) { const t = loBin; loBin = hiBin; hiBin = t; } - return { - loBin, - hiBin, - edgeLo: lo + loBin * span / p, - edgeHi: lo + (hiBin + 1) * span / p, - }; +// Mirrors the display kernel's FIXED_HIST_ROUND_EPS (flexviz.cube). +const _FV_CUBE_ROUND_EPS = 1e-9; + +// Doubles in order as 64-bit integer keys (sign folded), so bisecting keys +// bisects doubles. +const _fvF64 = new Float64Array(1); +const _fvI64 = new BigInt64Array(_fvF64.buffer); +function _fvKey(x) { + _fvF64[0] = x; + const b = _fvI64[0]; + return b < 0n ? -(b & 0x7fffffffffffffffn) : b; +} +function _fvFromKey(k) { + _fvI64[0] = k < 0n ? -k - (1n << 63n) : k; + return _fvF64[0]; } -// Snap a 2-D box on a box2d free axis (contract H): snap each axis -// independently with the same per-axis arithmetic as fvCubeSnap (or the -// integer-day grid for a unit:"day" axis), against that axis's decoded -// physical domain / day-grid. `axes` is [{domain, unit, dayGrid}, ...] for x -// then y. Returns {x:{loBin,hiBin,edgeLo,edgeHi}, y:{...}}. -function fvCubeSnap2d(axes, p, box) { - const snapAxis = (ax, a, b) => { - if (ax && ax.dayGrid) { - return fvCubeSnapDay(ax.domain[0], ax.dayGrid.w, ax.dayGrid.pEff, a, b); +// Snap a brush [a, b] to p bins over domain (mirrors flexviz.cube.snap_brush). +// A bin is selected when the brush covers its center as the display draws it, +// (i + 0.5) * step + lo, ends included: Plotly's highlight rule for bars. +// hiBin < loBin means no center is covered. Each committed edge is the first +// double the kernel puts in its bin, floor((v - lo) * scale + eps) >= k, found +// by bisecting the doubles around the float edge lo + (k - eps) * step: a +// bounded search, because near a zero crossing billions of doubles share one +// value of v - lo. So the range keeps exactly the values the display counts +// in those bars, and the ceil of an edge is the first whole unit of a +// temporal bin. The first bin starts at lo; the last ends at hi inclusive +// (the kernel's top clamp). A zero span (a constant column) draws every bar +// on lo, so the brush covers all or none. +function fvCubeSnap(domain, p, a, b) { + const [lo, hi] = domain; + const [from, to] = a <= b ? [a, b] : [b, a]; + if (!(hi > lo)) { + const hit = from <= lo && lo <= to; + return { loBin: 0, hiBin: hit ? p - 1 : -1, edgeLo: lo, edgeHi: hi, closed: 'both' }; + } + const step = (hi - lo) / p; + const scale = p / (hi - lo); + const center = i => (i + 0.5) * step + lo; + // e0: the first bar whose center is at or above from; e1: one past the + // last bar whose center is at or below to. The estimate is off by at most + // one bar in float noise; the loops settle it on the drawn centers. + let e0 = Math.max(0, Math.min(p, Math.ceil((from - lo) / step - 0.5))); + while (e0 > 0 && center(e0 - 1) >= from) e0--; + while (e0 < p && center(e0) < from) e0++; + let e1 = Math.max(0, Math.min(p, Math.floor((to - lo) / step + 0.5))); + while (e1 > 0 && center(e1 - 1) > to) e1--; + while (e1 < p && center(e1) <= to) e1++; + const bin = v => Math.floor((v - lo) * scale + _FV_CUBE_ROUND_EPS); + const bound = k => { + if (k === 0) return lo; + // Bracket by doubling steps away from the guess, then bisect: bin(a) < k, + // bin(b) >= k, at most about 2 x 64 steps. + let a = _fvKey(lo + (k - _FV_CUBE_ROUND_EPS) * step); + let b = a; + for (let d = 1n; bin(_fvFromKey(a)) >= k; d *= 2n) a -= d; + for (let d = 1n; bin(_fvFromKey(b)) < k; d *= 2n) b += d; + while (b - a > 1n) { + const m = (a + b) >> 1n; + if (bin(_fvFromKey(m)) >= k) b = m; + else a = m; } - return fvCubeSnap(ax.domain, p, a, b); + return _fvFromKey(b); }; + const top = e1 === p; return { - x: snapAxis(axes[0], box.x[0], box.x[1]), - y: snapAxis(axes[1], box.y[0], box.y[1]), + loBin: e0, + hiBin: e1 - 1, + edgeLo: bound(e0), + edgeHi: top ? hi : bound(e1), + closed: top ? 'both' : 'left', }; } // The highest valid free-bin index for an entry: the last category for a -// categorical axis, the composite top S²-1 for box2d (contract H), the -// effective top bin otherwise. Used to clamp slice ranges. +// categorical axis, the composite top p*p-1 for box2d (contract H), the top +// bin p-1 otherwise. Used to clamp slice ranges. function _fvCubeMaxBin(header) { if (header.free.kind === 'categorical') return header.free.categories.length - 1; - if (header.free.kind === 'box2d') { - const s = header.free.p + 1; - return s * s - 1; - } - return header.free.p_eff ?? header.free.p; + if (header.free.kind === 'box2d') return header.free.p * header.free.p - 1; + return header.free.p - 1; } // Build the per-row composite free-bin ranges of a snapped 2-D box (contract // H): for each by in [ly..hy], one inclusive range [by*S+lx, by*S+hx]. The // rows of one by form a contiguous CSR block, so the generalized slice walks -// binStart[by*S+lx] .. binStart[by*S+hx+1] exactly. S = header.free.p + 1. +// binStart[by*S+lx] .. binStart[by*S+hx+1] exactly. S = p. function fvCubeRectRanges(header, snap2d) { - const s = header.free.p + 1; + const s = header.free.p; const lx = snap2d.x.loBin, hx = snap2d.x.hiBin; const ly = snap2d.y.loBin, hy = snap2d.y.hiBin; const ranges = []; @@ -1365,11 +1360,9 @@ window.fvCubeHeaderMatchesKey = fvCubeHeaderMatchesKey; window.fvTemporalToPhysical = fvTemporalToPhysical; window.fvPhysicalToTemporal = fvPhysicalToTemporal; window.fvPhysicalToEpochMs = fvPhysicalToEpochMs; -window.fvCubeDayGrid = fvCubeDayGrid; window.fvCubeStoreReset = fvCubeStoreReset; window.fvDecodeFVCube = decodeFVCube; window.fvDecodeCubeBundle = decodeCubeBundle; -window.fvCubeSnap2d = fvCubeSnap2d; window.fvCubeRectRanges = fvCubeRectRanges; window.fvLineEnvCells = fvLineEnvCells; window.fvApplyLineGaps = fvApplyLineGaps; diff --git a/flexviz/cube.py b/flexviz/cube.py index aa8bbb75..1b20432d 100644 --- a/flexviz/cube.py +++ b/flexviz/cube.py @@ -104,20 +104,79 @@ def temporal_unit(dtype: pl.DataType | None) -> str | None: return None -def day_grid(lo: float, hi: float, p: int) -> tuple[int, int]: - """The integer-day snap grid for a ``unit="day"`` free axis. - - A fixed P=2048 grid would yield fractional-day edges that ``YYYY-MM-DD`` - cannot represent, breaking the round-trip contract. Instead the bin - width is ``w = max(1, ceil(span_days / p))`` whole days and the bin - count ``P' = ceil(span_days / w)`` (≤ p) — every snap edge - ``lo + k*w`` is an integer day, so date strings round-trip bit-exactly. - All other arithmetic is the shared arithmetic with ``P = P'`` (natural - floor, filter-don't-clip, degenerate top bin ``P'``).""" - span = hi - lo - w = max(1, math.ceil(span / p)) - p_eff = max(0, math.ceil(span / w)) - return w, p_eff +def _double_key(x: float) -> int: + """``x`` as a 64-bit integer key that orders like the doubles (sign + folded), so bisecting keys bisects doubles. Mirrors ``_fvKey``.""" + b = struct.unpack("= 0 else -(b & 0x7FFF_FFFF_FFFF_FFFF) + + +def _double_from_key(k: int) -> float: + return struct.unpack("= 0 else -k - (1 << 63)))[0] + + +def snap_brush( + lo: float, hi: float, p: int, a: float, b: float +) -> tuple[int, int, float, float, str]: + """Snap a brush ``[a, b]`` to a free axis of ``p`` bins over ``[lo, hi]``. + + Mirrors ``fvCubeSnap``. A bin is selected when the brush covers its center + as the display draws it, ``(i + 0.5) * step + lo``, ends included (Plotly's + highlight rule for bars). Returns ``(lo_bin, hi_bin, edge_lo, edge_hi, + closed)``; ``hi_bin < lo_bin`` means the brush covers no center. + + Each committed edge is the first float the kernel puts in its bin, found by + bisecting the floats around ``lo + (k - eps) * step`` along + ``_fixed_hist_bin_expr``'s rule. So the range keeps exactly the values the + display counts in those bars. The first bin starts at ``lo`` and the last + bin ends at ``hi`` inclusive (the top clamp), because the build filters + rows to ``[lo, hi]``. A zero span (a constant column) draws every bar on + ``lo``, so the brush covers all of them or none. + """ + a, b = sorted((a, b)) + if not hi > lo: + return 0, p - 1 if a <= lo <= b else -1, lo, hi, "both" + step = (hi - lo) / p + scale = p / (hi - lo) + + def _center(i: int) -> float: + return (i + 0.5) * step + lo + + e0 = max(0, min(p, math.ceil((a - lo) / step - 0.5))) + while e0 > 0 and _center(e0 - 1) >= a: + e0 -= 1 + while e0 < p and _center(e0) < a: + e0 += 1 + e1 = max(0, min(p, math.floor((b - lo) / step + 0.5))) + while e1 > 0 and _center(e1 - 1) > b: + e1 -= 1 + while e1 < p and _center(e1) <= b: + e1 += 1 + + def _bin(v: float) -> int: + return math.floor((v - lo) * scale + _FIXED_HIST_ROUND_EPS) + + def _bound(k: int) -> float: + if k == 0: + return lo + lo_key = hi_key = _double_key(lo + (k - _FIXED_HIST_ROUND_EPS) * step) + d = 1 + while _bin(_double_from_key(lo_key)) >= k: + lo_key, d = lo_key - d, d * 2 + d = 1 + while _bin(_double_from_key(hi_key)) < k: + hi_key, d = hi_key + d, d * 2 + while hi_key - lo_key > 1: + mid = (lo_key + hi_key) // 2 + if _bin(_double_from_key(mid)) >= k: + hi_key = mid + else: + lo_key = mid + return _double_from_key(hi_key) + + if e1 == p: + return e0, e1 - 1, _bound(e0), hi, "both" + return e0, e1 - 1, _bound(e0), _bound(e1), "left" @dataclass(frozen=True) @@ -145,7 +204,7 @@ class FreeAxisSpec: primary ``active_source.column`` join key), ``p = P₂D = 128`` per axis, and the per-axis domains live in ``domains = ((lox,hix),(loy,hiy))`` (the single-axis ``domain`` stays ``None``). Each axis is binned with the shared - arithmetic; the composite free bin is ``bin_y * (p+1) + bin_x``. + arithmetic; the composite free bin is ``bin_y * p + bin_x``. ``unit`` is per-axis for box2d — encoded as a 2-tuple ``(unit_x, unit_y)`` — and is set by the engine from the schema dtypes. """ @@ -156,8 +215,7 @@ class FreeAxisSpec: domain: tuple[float, float] | None = None columns: tuple[str, ...] | None = None # Physical unit for kind="temporal" (contract G); the engine sets it from - # the schema dtype (Datetime("ns") gates to no cube at all). unit="day" - # switches the snap grid to integer days (see ``day_grid``). For box2d + # the schema dtype (Datetime("ns") gates to no cube at all). For box2d # (contract H) this is a per-axis 2-tuple ``(unit_x, unit_y)`` (each # element None or a TemporalUnit), set by the engine. unit: Any = None @@ -303,8 +361,8 @@ class CubeResult: (``_MEASURE_PARTIALS``). For a range (continuous/temporal) free axis the free key is ``free_bin`` (Int32); the client slices by combining partials over the free bins inside the snapped brush range. Valid free bins are - ``0..P`` *inclusive*: a value exactly equal to the domain max lands in the - degenerate top bin ``P`` (natural un-clamped floor, Mosaic-style). + ``0..P-1``: rows bin like the display kernel, so a value at the domain max + lands in the top bin ``P-1`` (``_fixed_hist_bin_expr``). For a **categorical** free axis the free key is the tuple of typed ``__free__{col}`` columns (``free_key_cols``); there is no ``free_bin`` @@ -334,7 +392,7 @@ def slice_count(self, free_lo: float, free_hi: float) -> pl.DataFrame: """Reference (server-side) slice: count per target cell over a free range. Mirrors what ``cube.js`` does client-side — provided here for parity tests - and as the fallback path. ``[free_lo, free_hi]`` is snapped to the P-grid. + and as the fallback path. ``[free_lo, free_hi]`` is snapped with ``snap_brush``. """ lo_bin, hi_bin = self._snap(free_lo, free_hi) tgt = list(self.group_cols) @@ -374,7 +432,7 @@ def slice_agg_box2d( then combine + finalize like ``slice_agg``. Mirrors the client's rectangle slice (``fvCubeSliceRect``).""" (lx, hx), (ly, hy) = self._snap_box2d(x_lo, x_hi, y_lo, y_hi) - s = box2d_composite_stride(self.spec.free.p) + s = self.spec.free.p # the composite stride codes: list[int] = [] for by in range(ly, hy + 1): row = by * s @@ -389,30 +447,14 @@ def slice_agg_box2d( def _snap_box2d( self, x_lo: float, x_hi: float, y_lo: float, y_hi: float ) -> tuple[tuple[int, int], tuple[int, int]]: - """Per-axis natural-floor bin pairs of the box's corners, each clamped - to ``[0, p_eff]`` (degenerate top bin included). Mirrors ``fvCubeSnap`` - per axis against each axis's resolved physical domain / day-grid.""" + """Per-axis ``snap_brush`` bin pairs of the box's corners, against each + axis's grid.""" free = self.spec.free (lox, hix), (loy, hiy) = free.domains # type: ignore[misc] - unit_x, unit_y = _box2d_units(free) - - def _axis(lo: float, hi: float, p: int, unit: str | None, a: float, b: float): - if unit == "day": - w, p_eff = day_grid(lo, hi, p) - lo_b = max(0, min(p_eff, int((a - lo) / w))) - hi_b = max(0, min(p_eff, int((b - lo) / w))) - else: - span = (hi - lo) or 1.0 - lo_b = max(0, min(p, int((a - lo) / span * p))) - hi_b = max(0, min(p, int((b - lo) / span * p))) - if hi_b < lo_b: - lo_b, hi_b = hi_b, lo_b - return lo_b, hi_b - - return ( - _axis(lox, hix, free.p, unit_x, x_lo, x_hi), - _axis(loy, hiy, free.p, unit_y, y_lo, y_hi), - ) + px = py = free.p + x = snap_brush(lox, hix, px, x_lo, x_hi) + y = snap_brush(loy, hiy, py, y_lo, y_hi) + return (x[0], x[1]), (y[0], y[1]) def corr_matrix( self, @@ -569,16 +611,9 @@ def _combine_corr(self, sliced: pl.DataFrame) -> pl.DataFrame: ) def _snap(self, free_lo: float, free_hi: float) -> tuple[int, int]: - """Natural-floor bin indices of the brush endpoints, clamped to - ``[0, P]`` — ``P`` included so a brush reaching the domain max selects - the degenerate top bin (same arithmetic as the build side).""" + """``snap_brush`` bin indices of the brush endpoints.""" lo, hi = self.spec.free.domain - p = self.spec.free.p - span = (hi - lo) or 1.0 - lo_bin = max(0, min(p, int((free_lo - lo) / span * p))) - hi_bin = max(0, min(p, int((free_hi - lo) / span * p))) - if hi_bin < lo_bin: - lo_bin, hi_bin = hi_bin, lo_bin + lo_bin, hi_bin, *_ = snap_brush(lo, hi, self.spec.free.p, free_lo, free_hi) return lo_bin, hi_bin @@ -672,24 +707,6 @@ def _free_value_expr(free: FreeAxisSpec) -> pl.Expr: return pl.col(free.column) -def _free_bin_expr(value: pl.Expr, lo: float, hi: float, p: int) -> pl.Expr: - """Natural (un-clamped) floor bin: ``floor((v-lo)/(hi-lo)*p)``. - - Rows are pre-filtered to ``[lo, hi]``, so indices are ``0..p`` inclusive — - ``p`` being the degenerate top bin for ``v == hi``. Int32 holds bin ``p`` - and matches the descriptor spec's wire type. - """ - span = (hi - lo) or 1.0 - return ((value - lo) / span * p).floor().cast(pl.Int32).alias("free_bin") - - -def _day_free_bin_expr(value: pl.Expr, lo: float, w: int) -> pl.Expr: - """Integer-day grid bin (contract G): ``floor((v - lo) / w)`` with whole- - day width ``w``. Values and edges are integer days, so the division is - exact; the degenerate top bin is ``P'`` for ``v == hi``.""" - return ((value - lo) / w).floor().cast(pl.Int32).alias("free_bin") - - def _box2d_axis(column: str, unit: str | None) -> pl.Expr: """A box2d axis value as Float64 — physical representation for a temporal unit (contract G, per axis), the raw column otherwise.""" @@ -699,15 +716,6 @@ def _box2d_axis(column: str, unit: str | None) -> pl.Expr: return val.cast(pl.Float64) -def _box2d_axis_bin_expr(value: pl.Expr, lo: float, hi: float, p: int) -> pl.Expr: - """Natural floor bin on one box2d axis — the shared range arithmetic - (verbatim ``_free_bin_expr``), un-aliased so the two axes compose into the - composite ``free_bin``. ``unit="day"`` axes pass ``hi``/``p`` derived from - ``day_grid`` so the per-axis arithmetic stays the integer-day grid.""" - span = (hi - lo) or 1.0 - return ((value - lo) / span * p).floor().cast(pl.Int32) - - def _box2d_units(free: FreeAxisSpec) -> tuple[str | None, str | None]: """The per-axis temporal units of a box2d free axis as a 2-tuple. ``unit`` is stored as a tuple/list ``(unit_x, unit_y)`` (engine-set) or ``None``.""" @@ -934,33 +942,13 @@ def _env_part(part: pl.DataFrame) -> pl.DataFrame: "domain=None (full data domain) before building" ) f_lo, f_hi = free.domain - free_val = _free_value_expr(free) - filters: list[pl.Expr] = [] - if free.unit == "day": - # The integer-day snap grid (contract G) is not a uniform P-grid over - # the domain, so the kernel cannot bin raw day values directly. Remap - # each row to its day-grid bin MIDPOINT ``b + 0.5`` over domain - # ``[0, P'+1]`` with ``p = P'+1``: ``floor((b+0.5)/(P'+1)*(P'+1)) == b`` - # is robust to the double rounding of the kernel's true division (the - # value sits half a bin from any edge), and the resulting free_bin - # equals ``_day_free_bin_expr``'s bit-exactly. Out-of-domain rows are - # pre-filtered: a row just above the domain max could otherwise remap - # back into ``0..P'`` (the grid may overshoot the span). - w, p_eff = day_grid(f_lo, f_hi, free.p) - day_bin = _day_free_bin_expr(free_val, f_lo, w) - free_expr = (day_bin.cast(pl.Float64) + 0.5).alias("__f") - k_lo, k_hi, k_p = 0.0, float(p_eff + 1), p_eff + 1 - filters.append(free_val.is_between(f_lo, f_hi)) - else: - free_expr = free_val.cast(pl.Float64).alias("__f") - k_lo, k_hi, k_p = float(f_lo), float(f_hi), free.p - - base = ldf.filter(*filters) if filters else ldf - df = base.select( + # The kernel bins the free axis with the display kernel's rule and filters + # out-of-domain rows itself, so it takes the raw value. + df = ldf.select( *[pl.col(c) for c in cat_cols], _target_dim_value_expr(bucket).cast(pl.Float64).alias("__x"), pl.col(spec.measure.value_col).cast(pl.Float64).alias("__y"), - free_expr, + _free_value_expr(free).cast(pl.Float64).alias("__f"), ).collect(engine="streaming") def _envelope(part: pl.DataFrame) -> pl.DataFrame: @@ -971,10 +959,10 @@ def _envelope(part: pl.DataFrame) -> pl.DataFrame: pl.col("__f"), pl.lit(float(x_lo)), pl.lit(float(x_hi)), - pl.lit(k_lo), - pl.lit(k_hi), + pl.lit(float(f_lo)), + pl.lit(float(f_hi)), bucket.bins, - k_p, + free.p, ) ) .to_series() @@ -1109,11 +1097,9 @@ def _build_corr_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: ) free_val = _free_value_expr(free) lo, hi = free.domain - if free.unit == "day": - w, _ = day_grid(lo, hi, free.p) - free_bin = _day_free_bin_expr(free_val, lo, w) - else: - free_bin = _free_bin_expr(free_val, lo, hi, free.p) + # Bin like the display kernel, so a brush over bins k..m slices the rows + # the source trace draws in them. + free_bin = _fixed_hist_bin_expr(free_val, lo, hi, free.p, "free_bin") frame = ( ldf.filter(free_val.is_between(lo, hi)) .with_columns(free_bin) @@ -1124,38 +1110,14 @@ def _build_corr_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: return CubeResult(spec=spec, frame=frame, group_cols=(), corr_means=means) -def box2d_composite_stride(p: int) -> int: - """The composite-index stride ``S = p + 1`` (contract H): the free bin is - ``bin_y * S + bin_x``, with ``S`` large enough for the degenerate top bin - ``p`` on the x axis. ``P₂D = 128 ⇒ S = 129``.""" - return p + 1 - - -def _box2d_axis_grid( - lo: float, hi: float, p: int, unit: str | None -) -> tuple[int | float, int]: - """The (width-or-hi, effective-p) per box2d axis. A ``unit="day"`` axis - uses the integer-day snap grid (contract G) so its arithmetic is - ``floor((v-lo)/w)`` with whole-day width ``w`` and ``p_eff`` bins; every - other axis is the uniform P-grid (``floor((v-lo)/(hi-lo)*p)``).""" - if unit == "day": - w, p_eff = day_grid(lo, hi, p) - # Re-express the integer-day grid as a uniform-P arithmetic over - # [lo, lo + p_eff*w]: floor((v-lo)/(p_eff*w)*p_eff) == floor((v-lo)/w). - return float(lo + p_eff * w), p_eff - return float(hi), p - - def _build_box2d_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: """Build a 2-D box (hist2d source) cube (contract H). - A range-like build: each of the two axes is binned with the shared - arithmetic (natural floor, filter-don't-clip via ``is_between`` per axis, - degenerate top bin ``p`` per axis). The composite free key is - ``free_bin = bin_y * S + bin_x`` with ``S = p + 1`` (so every composite - index ``0..S²-1`` is reachable). Temporal axes run on their physical - representation per axis (contract G), and a ``unit="day"`` axis uses the - integer-day snap grid. Reuses ``_target_group_exprs`` / ``_measure_exprs`` + A range-like build: each of the two axes is binned like the display + kernel (``_fixed_hist_bin_expr``, filter-don't-clip via ``is_between`` per axis). + The composite free key is ``free_bin = bin_y * p + bin_x`` (indices + ``0..p*p-1``). Temporal axes run on their physical representation per + axis (contract G). Reuses ``_target_group_exprs`` / ``_measure_exprs`` (a pure source build has no target dims, but the path stays general). """ free = spec.free @@ -1172,17 +1134,13 @@ def _build_box2d_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: cx, cy = free.columns # type: ignore[misc] (lox, hix), (loy, hiy) = free.domains unit_x, unit_y = _box2d_units(free) - p = free.p - s = box2d_composite_stride(p) + px = py = free.p val_x = _box2d_axis(cx, unit_x) val_y = _box2d_axis(cy, unit_y) - hix_eff, px = _box2d_axis_grid(lox, hix, p, unit_x) - hiy_eff, py = _box2d_axis_grid(loy, hiy, p, unit_y) - - bin_x = _box2d_axis_bin_expr(val_x, lox, hix_eff, px) - bin_y = _box2d_axis_bin_expr(val_y, loy, hiy_eff, py) - free_bin = (bin_y * s + bin_x).cast(pl.Int32).alias("free_bin") + bin_x = _fixed_hist_bin_expr(val_x, lox, hix, px, "__bx") + bin_y = _fixed_hist_bin_expr(val_y, loy, hiy, py, "__by") + free_bin = (bin_y * px + bin_x).cast(pl.Int32).alias("free_bin") pre, group_cols, filters = _target_group_exprs(spec) frame = ( @@ -1283,11 +1241,9 @@ def build_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: ) free_val = _free_value_expr(free) lo, hi = free.domain - if free.unit == "day": - w, _ = day_grid(lo, hi, free.p) - free_bin = _day_free_bin_expr(free_val, lo, w) - else: - free_bin = _free_bin_expr(free_val, lo, hi, free.p) + # Bin like the display kernel, so a brush over bins k..m slices the rows + # the source trace draws in them. + free_bin = _fixed_hist_bin_expr(free_val, lo, hi, free.p, "free_bin") frame = ( ldf.filter(free_val.is_between(lo, hi), *filters) @@ -1469,11 +1425,9 @@ def encode_fvcube(result: CubeResult, cube_id: str) -> bytes: elif spec.free.kind == "box2d": # The composite free_bin is already a u32 (build cast it to Int32); the # encode below treats it like any range free_bin. The header free block - # carries per-axis domains and the composite stride is recovered from - # "p" (S = p + 1). Per-axis temporal handling mirrors the single-axis - # block but as 2-element lists: "units" = [unit_x|null, unit_y|null] - # and "grids" = [{w, p_eff}|null, ...] (a grid entry is present only - # for a "day" axis). The decoder rebuilds the per-axis grids from these. + # carries the per-axis grid ("p" per axis and "domains"); the + # composite stride is p. Temporal axes add "units" = + # [unit_x|null, unit_y|null]. frame = result.frame.sort(["free_bin", *group_cols]) (lox, hix), (loy, hiy) = spec.free.domains # type: ignore[misc] unit_x, unit_y = _box2d_units(spec.free) @@ -1485,14 +1439,6 @@ def encode_fvcube(result: CubeResult, cube_id: str) -> bytes: } if unit_x is not None or unit_y is not None: free_block["units"] = [unit_x, unit_y] - grids: list = [] - for unit, lo, hi in ((unit_x, lox, hix), (unit_y, loy, hiy)): - if unit == "day": - w, p_eff = day_grid(lo, hi, spec.free.p) - grids.append({"w": w, "p_eff": p_eff}) - else: - grids.append(None) - free_block["grids"] = grids else: frame = result.frame.sort(["free_bin", *group_cols]) free_block = { @@ -1502,10 +1448,6 @@ def encode_fvcube(result: CubeResult, cube_id: str) -> bytes: } if spec.free.unit is not None: free_block["unit"] = spec.free.unit - if spec.free.unit == "day": - w, p_eff = day_grid(spec.free.domain[0], spec.free.domain[1], spec.free.p) - free_block["w"] = w - free_block["p_eff"] = p_eff is_corr = spec.measure.agg == "corr" if is_corr: # corr partials are DYNAMIC (6 per pair) — generated from the pairs, @@ -1773,7 +1715,8 @@ def cube_content_key(spec: CubeSpec) -> str: ], "m": _measure_content_block(spec.measure), "p": spec.passive_key, - "v": 1, # cube schema version + # cube schema version: 2 = free axes bin like the display kernel + "v": 2, } canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) return hashlib.blake2b(canonical.encode("utf-8"), digest_size=16).hexdigest() diff --git a/flexviz_polars/src/expressions.rs b/flexviz_polars/src/expressions.rs index 7231df04..983a6ba4 100644 --- a/flexviz_polars/src/expressions.rs +++ b/flexviz_polars/src/expressions.rs @@ -84,7 +84,8 @@ struct FixedHistKwargs { const FIXED_HIST_ROUND_EPS: f64 = 1e-9; /// Per-axis bin scale for the 2D binners (`fixed_hist2d`, its rayon twin, and -/// `fixed_hist2d_reduce`): `nb / (hi - lo)`, which gives the same bins as the +/// `fixed_hist2d_reduce`) and the free axis of `fixed_line_envelope2d`: +/// `nb / (hi - lo)`, which gives the same bins as the /// 1D `fixed_hist` (that kernel takes its `count_degenerate` branch first). /// The `.min(max_idx)` clamp at every call site folds a value at `hi` into /// the top bin, so the scale needs no span pad. A pad in absolute data units @@ -1482,11 +1483,13 @@ fn fixed_hist2d(inputs: &[Series], kwargs: FixedHist2DKwargs) -> PolarsResult PolarsResult { let stride = n_buckets + 1; // buckets 0..=n_buckets (degenerate top bin) - let Some(n_cells) = stride.checked_mul(p + 1) else { + let Some(n_cells) = stride.checked_mul(p) else { polars_bail!( InvalidOperation: - "fixed_line_envelope2d: (n_buckets + 1) * (p + 1) overflows usize" + "fixed_line_envelope2d: (n_buckets + 1) * p overflows usize" ); }; let mut acc = EnvelopeAcc::new(n_cells); - // Shared arithmetic: floor((v - lo) / span * n) with true IEEE division — - // matching the JS client's `/` bit-exactly. (Polars' own scalar division, - // as in cube.py's _free_bin_expr, is multiply-by-reciprocal and can differ - // by 1 ulp at exact bin edges; the Polars test reference forces true - // division via a materialized span column.) A degenerate (lo == hi) - // domain admits only v == lo, which bins to 0. + // x: floor((v - lo) / span * n) with true IEEE division — matching the JS + // client's `/` bit-exactly. (Polars' own division by a scalar is + // multiply-by-reciprocal and can differ by 1 ulp at exact bin edges; the + // Polars test reference forces true division via a materialized span + // column.) A degenerate (lo == hi) domain admits only v == lo, which bins + // to 0. The free bin uses `hist2d_axis_scale`, as the display kernel does. let x_span = if x_hi > x_lo { x_hi - x_lo } else { 1.0 }; - let f_span = if f_hi > f_lo { f_hi - f_lo } else { 1.0 }; let nb_f64 = n_buckets as f64; - let p_f64 = p as f64; + let f_scale = hist2d_axis_scale(f_lo, f_hi, p); + let max_bf = p - 1; let mut visit = |xv: f64, yv: f64, fv: f64| { // Filter, don't clip: NaN fails these range checks too, so NaN in any @@ -1596,10 +1599,10 @@ fn envelope_scan( if !(xv >= x_lo && xv <= x_hi && fv >= f_lo && fv <= f_hi) || yv.is_nan() { return; } - // In-domain ⇒ the ratio is in [0, 1] exactly, so bx <= n_buckets and - // bf <= p (degenerate top bins included) — idx is always in bounds. + // In-domain ⇒ the x ratio is in [0, 1] exactly, so bx <= n_buckets, + // and bf is clamped to p - 1 — idx is always in bounds. let bx = ((xv - x_lo) / x_span * nb_f64).floor() as usize; - let bf = ((fv - f_lo) / f_span * p_f64).floor() as usize; + let bf = (((fv - f_lo) * f_scale + FIXED_HIST_ROUND_EPS) as usize).min(max_bf); acc.update(bf * stride + bx, xv, yv); }; @@ -1713,7 +1716,7 @@ fn fixed_line_envelope2d( let mut x_at_ymin = Vec::with_capacity(n_out); let mut y_max = Vec::with_capacity(n_out); let mut x_at_ymax = Vec::with_capacity(n_out); - for bf in 0..=kwargs.p { + for bf in 0..kwargs.p { for bx in 0..stride { let idx = bf * stride + bx; if acc.seen[idx] { diff --git a/flexviz_polars/tests/test_plugin_functions.py b/flexviz_polars/tests/test_plugin_functions.py index 80673b4f..9d27c7ae 100644 --- a/flexviz_polars/tests/test_plugin_functions.py +++ b/flexviz_polars/tests/test_plugin_functions.py @@ -164,12 +164,15 @@ def _envelope_reference( """Pure-Polars reference for fixed_line_envelope2d (contract J, plan 2026-06-11 — pins kernel parity bit-exactly). - Shared-arithmetic semantics on BOTH axes: - * natural floor bin = ``floor((v - lo) / span * n)`` with true IEEE - division (see the divisor note below); NO epsilon, NO clip; + Bin semantics: + * x bucket = ``floor((v - lo) / span * n)`` with true IEEE division (see + the divisor note below); NO epsilon, NO clip, so a value exactly at the + domain max lands in the degenerate top bucket and buckets run + ``0..=n_buckets``; + * free bin = the display kernel's rule, ``floor((v - lo) * scale + eps)`` + clamped to ``p - 1`` with ``scale = p / (hi - lo)`` (as flexviz's + ``_fixed_hist_bin_expr``), so free bins run ``0..p``; * rows outside ``[lo, hi]`` on either axis are FILTERED (not clipped); - * a value exactly at the domain max lands in the degenerate top bin, so - indices run ``0..=n_buckets`` and ``0..=p`` inclusive; * null or NaN in x, y, or free ⇒ row filtered; * ties (equal y within a cell): FIRST row in scan order wins for both min and max (Polars arg_min/arg_max return the first occurrence). @@ -179,12 +182,12 @@ def _envelope_reference( float division *by a scalar* into multiplication by the reciprocal (e.g. ``49.0 / lit(49.0)`` → ``0.999…`` on frames with ≥2 rows), which is not IEEE division and can shift a domain-max value out of its - degenerate top bin. The kernel — like the JS client — uses true - division, and column/column division in Polars is true division too, - so dividing by a materialized column pins exactly that. + degenerate top bucket. The kernel — like the JS client — uses true + division for x, and column/column division in Polars is true division + too, so dividing by a materialized column pins exactly that. """ x_span = (x_hi - x_lo) or 1.0 - f_span = (free_hi - free_lo) or 1.0 + f_scale = p / (free_hi - free_lo) if free_hi > free_lo else 0.0 n_rows = len(x) return ( pl.DataFrame( @@ -193,7 +196,6 @@ def _envelope_reference( "__y": y.cast(pl.Float64), "__f": free.cast(pl.Float64), "__xspan": pl.Series([x_span] * n_rows, dtype=pl.Float64), - "__fspan": pl.Series([f_span] * n_rows, dtype=pl.Float64), } ) .lazy() @@ -208,8 +210,9 @@ def _envelope_reference( .floor() .cast(pl.UInt32) .alias("bucket"), - ((pl.col("__f") - free_lo) / pl.col("__fspan") * float(p)) + ((pl.col("__f") - free_lo) * f_scale + 1e-9) .floor() + .clip(0, p - 1) .cast(pl.UInt32) .alias("free_bin"), ) @@ -1302,8 +1305,9 @@ def test_nan_rows_filtered(self): assert out.height == 2 assert out["y_min"].to_list() == [1.0, 4.0] - def test_degenerate_top_bin_both_axes(self): - # Values exactly at the domain max land in bin n_buckets / bin p. + def test_domain_max_lands_in_the_top_bucket_and_free_bin(self): + # x == x_hi lands in the degenerate bucket n_buckets; free == free_hi + # is clamped into the top free bin p - 1, as the display kernel does. x = pl.Series("x", [0.0, 10.0, 10.0], dtype=pl.Float64) y = pl.Series("y", [1.0, 2.0, 3.0], dtype=pl.Float64) f = pl.Series("f", [0.0, 0.0, 5.0], dtype=pl.Float64) @@ -1311,7 +1315,17 @@ def test_degenerate_top_bin_both_axes(self): cells = set(zip(out["bucket"].to_list(), out["free_bin"].to_list())) assert (0, 0) in cells assert (5, 0) in cells, "x == x_hi must land in degenerate bucket 5" - assert (5, 4) in cells, "free == free_hi must land in degenerate bin 4" + assert (5, 3) in cells, "free == free_hi must land in the top free bin 3" + + def test_free_value_on_an_edge_lands_in_the_display_bin(self): + # Over [0, 0.2] in 4 bins, 0.15 sits on the edge of bin 3. A plain + # floor of 0.15 / 0.2 * 4 = 2.9999999999999996 drops it into bin 2; + # the display kernel's rule puts it in bin 3. + x = pl.Series("x", [5.0], dtype=pl.Float64) + y = pl.Series("y", [1.0], dtype=pl.Float64) + f = pl.Series("f", [0.15], dtype=pl.Float64) + out = _assert_envelope_parity(x, y, f, 0.0, 10.0, 0.0, 0.2, 4, 4) + assert out["free_bin"].to_list() == [3] def test_out_of_domain_filtered_not_clipped(self): # Out-of-domain rows vanish entirely instead of contaminating edge bins. diff --git a/tests/test_browser_cube.py b/tests/test_browser_cube.py index 8aa954b1..147e6f95 100644 --- a/tests/test_browser_cube.py +++ b/tests/test_browser_cube.py @@ -71,6 +71,7 @@ import json import math +import random import polars as pl import pytest @@ -83,9 +84,11 @@ FreeAxisSpec, MeasureSpec, TargetDimSpec, + _fixed_hist_bin_expr, build_cube, encode_cube_bundle, encode_fvcube, + snap_brush, ) from flexviz.trace.bin_grid import snap_range from tests.test_browser import ( @@ -487,26 +490,55 @@ def _committed_edges(page: Page, column: str) -> tuple[float, float]: return clause["range"][0], clause["range"][1] +def _free_grid( + df: pl.DataFrame, col: str = "a", bins: int = _P +) -> tuple[float, float, int]: + """An unzoomed source's free axis ``(lo, hi, p)``: *bins* over the full + data domain.""" + lo, hi = float(df[col].min()), float(df[col].max()) + return lo, hi, bins + + +def _committed_bins( + edge_lo: float, edge_hi: float, grid: tuple[float, float, int] +) -> tuple[int, int]: + """The bins a commit covers. Its edges must be the kernel's boundaries of + those bins, bit-equal to ``snap_brush``.""" + lo, hi, p = grid + lo_bin, hi_bin, e_lo, e_hi, _closed = snap_brush(lo, hi, p, edge_lo, edge_hi) + assert 0 <= lo_bin <= hi_bin < p + assert (edge_lo, edge_hi) == (e_lo, e_hi) + return lo_bin, hi_bin + + +def _bins_commit_edges( + bins: list[int], grid: tuple[float, float, int] +) -> tuple[float, float]: + """The committed edges of a gesture's snapped bins (its ``lastBins``).""" + lo, hi, p = grid + step = (hi - lo) / p + brush = (lo + (bins[0] + 0.25) * step, lo + (bins[1] + 0.75) * step) + return snap_brush(lo, hi, p, *brush)[2:4] + + def _reference_slice_counts( - df: pl.DataFrame, edge_lo: float, edge_hi: float, free_col: str = "a" + df: pl.DataFrame, + edge_lo: float, + edge_hi: float, + free_col: str = "a", + bins: int = _P, ) -> list[float]: """Python reference for the target hist: ``flexviz.cube`` build + slice over the snapped bins recovered from committed ``closed="left"`` edges (the §8.2 parity property — identical to the legacy server recompute).""" - a_lo, a_hi = df[free_col].min(), df[free_col].max() - span = a_hi - a_lo - lo_bin = round((edge_lo - a_lo) / span * _P) - hi_bin = round((edge_hi - a_lo) / span * _P) - 1 - assert 0 <= lo_bin <= hi_bin <= _P - # Committed edges must lie exactly on the P-grid over the full domain. - assert edge_lo == pytest.approx(a_lo + lo_bin * span / _P, abs=1e-9) - assert edge_hi == pytest.approx(a_lo + (hi_bin + 1) * span / _P, abs=1e-9) + grid = _free_grid(df, free_col, bins) + lo_bin, hi_bin = _committed_bins(edge_lo, edge_hi, grid) b_lo, b_hi = df["b"].min(), df["b"].max() spec = CubeSpec( source_name="_reference", free=FreeAxisSpec( - column=free_col, kind="continuous", p=_P, domain=(a_lo, a_hi) + column=free_col, kind="continuous", p=grid[2], domain=grid[:2] ), target_dims=( TargetDimSpec( @@ -536,18 +568,13 @@ def _reference_grouped_slice_counts( """Per-group variant of ``_reference_slice_counts``: a grouped cube (binned ``b`` × categorical *group_col*) built + sliced over the snapped bins recovered from committed ``closed="left"`` edges.""" - a_lo, a_hi = df["a"].min(), df["a"].max() - span = a_hi - a_lo - lo_bin = round((edge_lo - a_lo) / span * _P) - hi_bin = round((edge_hi - a_lo) / span * _P) - 1 - assert 0 <= lo_bin <= hi_bin <= _P - assert edge_lo == pytest.approx(a_lo + lo_bin * span / _P, abs=1e-9) - assert edge_hi == pytest.approx(a_lo + (hi_bin + 1) * span / _P, abs=1e-9) + grid = _free_grid(df) + lo_bin, hi_bin = _committed_bins(edge_lo, edge_hi, grid) b_lo, b_hi = df["b"].min(), df["b"].max() spec = CubeSpec( source_name="_reference", - free=FreeAxisSpec(column="a", kind="continuous", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec(column="a", kind="continuous", p=grid[2], domain=grid[:2]), target_dims=( TargetDimSpec( column="b", @@ -672,30 +699,23 @@ def test_commit_counts_match_reference(self, page: Page, server_port: int): page.mouse.up() page.wait_for_timeout(800) - # Committed predicate: snapped to the P-grid, closed="left". + # Committed predicate: the kernel boundaries of the brushed bars. sels = page.evaluate("DASHBOARD_SPEC.state.selections") assert len(sels) == 1 clause = sels[0]["predicates"][0]["clauses"][0] assert clause["column"] == "a" assert clause.get("closed") == "left" edge_lo, edge_hi = clause["range"] - - # Shared arithmetic (plan doc): bin(v) = floor((v-lo)/s*P), - # edge(b) = lo + b*s/P; committed range = [edge(lo_bin), edge(hi_bin+1)). - a_lo, a_hi = df["a"].min(), df["a"].max() - span = a_hi - a_lo - lo_bin = round((edge_lo - a_lo) / span * _P) - hi_bin = round((edge_hi - a_lo) / span * _P) - 1 - assert 0 <= lo_bin <= hi_bin <= _P - # The committed edges must lie exactly on the P-grid. - assert edge_lo == pytest.approx(a_lo + lo_bin * span / _P, abs=1e-9) - assert edge_hi == pytest.approx(a_lo + (hi_bin + 1) * span / _P, abs=1e-9) + grid = _free_grid(df) + lo_bin, hi_bin = _committed_bins(edge_lo, edge_hi, grid) # Python reference: flexviz.cube build + slice over the same bins. b_lo, b_hi = df["b"].min(), df["b"].max() spec = CubeSpec( source_name="_cube_browser_auto_ref", - free=FreeAxisSpec(column="a", kind="continuous", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec( + column="a", kind="continuous", p=grid[2], domain=grid[:2] + ), target_dims=( TargetDimSpec( column="b", @@ -761,12 +781,10 @@ def test_live_brush_off_is_legacy(self, page: Page, server_port: int): lo, hi = clause["range"] # Unsnapped: an arbitrary pixel-derived range almost surely misses the # exact P-grid (probability ~0 of both edges landing on it). - df = _cube_df() - a_lo, a_hi = df["a"].min(), df["a"].max() - span = a_hi - a_lo + a_lo, a_hi, bars = _free_grid(_cube_df()) def _on_grid(v: float) -> bool: - scaled = (v - a_lo) / span * _P + scaled = (v - a_lo) / (a_hi - a_lo) * bars return math.isclose(scaled, round(scaled), abs_tol=1e-6) assert not (_on_grid(lo) and _on_grid(hi)), ( @@ -4123,8 +4141,8 @@ def test_zoomed_temporal_source_gesture(self, page: Page, server_port: int): assert hi <= dt.datetime(2020, 1, 25) def test_date_source_commits_integer_day_edges(self, page: Page, server_port: int): - """A Date-typed source brush commits YYYY-MM-DD edges (the - integer-day snap grid) with full parity.""" + """A Date-typed source brush commits YYYY-MM-DD edges (whole days, + ceil-ed from the kernel's bar boundaries) with full parity.""" df = _temporal_browser_df("date") url = _temporal_dashboard_url(server_port, "_cube_browser_tsday", kind="date") bodies = _capture_updates(page) @@ -4223,13 +4241,8 @@ def test_zoomed_source_brush_snaps_to_viewport_grid( assert types == ["cube_request"], types edge_lo, edge_hi = _committed_edges(page, "a") - span = zoom[1] - zoom[0] - # Edges lie exactly on the zoomed P-grid. - lo_bin = round((edge_lo - zoom[0]) / span * _P) - hi_bin = round((edge_hi - zoom[0]) / span * _P) - assert edge_lo == pytest.approx(zoom[0] + lo_bin * span / _P, abs=1e-9) - assert edge_hi == pytest.approx(zoom[0] + hi_bin * span / _P, abs=1e-9) - assert zoom[0] <= edge_lo < edge_hi <= zoom[1] + span / _P + # Edges are bin boundaries of the zoomed P-grid. + _committed_bins(edge_lo, edge_hi, (zoom[0], zoom[1], _P)) sels = page.evaluate("DASHBOARD_SPEC.state.selections") expr = _selection_expr(df, sels[0]) @@ -4331,7 +4344,8 @@ def test_zoomed_source_with_foreign_selection(self, page: Page, server_port: int clause = by_fig[fig_uids[0]]["predicates"][0]["clauses"][0] assert clause["column"] == "a" and clause.get("closed") == "left" edge_lo, edge_hi = clause["range"] - assert 10.0 <= edge_lo < edge_hi <= 80.0 + (80.0 - 10.0) / _P + # The commit stays inside the zoomed grid, never past its top. + assert 10.0 <= edge_lo < edge_hi <= 80.0 expected = _hist_counts_ref(df, expr_src & expr_third, "b", _TGT_BINS) assert 0 < sum(expected) < df.height _wait_for_hist_equals(page, "#fv-plot-1", expected) @@ -4411,7 +4425,7 @@ def test_box_source_brush_live_updates_hist(self, page: Page, server_port: int): # Committed client-side: one snapped closed="left" clause on "a". edge_lo, edge_hi = _committed_edges(page, "a") - expected = _reference_slice_counts(df, edge_lo, edge_hi, free_col="a") + expected = _reference_slice_counts(df, edge_lo, edge_hi, "a", bins=_P) assert 0 < sum(expected) < df.height _wait_for_hist_equals(page, "#fv-plot-1", expected) @@ -4451,7 +4465,7 @@ def test_line_source_brush_x_only_snapped_commit( ), "line commit must not carry a y-column clause" edge_lo, edge_hi = clause["range"] - expected = _reference_slice_counts(df, edge_lo, edge_hi, free_col="t") + expected = _reference_slice_counts(df, edge_lo, edge_hi, "t", bins=_P) assert 0 < sum(expected) < df.height _wait_for_hist_equals(page, "#fv-plot-1", expected) @@ -4519,11 +4533,8 @@ def _reference_line_envelope( ``line_env`` build + ``slice_agg`` over the snapped bins recovered from the committed ``closed="left"`` edges, then expanded into the same ``{x, y}`` two-points-per-bucket shape the client emits.""" - a_lo, a_hi = float(df["a"].min()), float(df["a"].max()) - span = a_hi - a_lo - lo_bin = round((edge_lo - a_lo) / span * _P) - hi_bin = round((edge_hi - a_lo) / span * _P) - 1 - assert 0 <= lo_bin <= hi_bin <= _P + grid = _free_grid(df) + _committed_bins(edge_lo, edge_hi, grid) b_lo, b_hi = float(df["b"].min()), float(df["b"].max()) target_dims = [ @@ -4539,14 +4550,12 @@ def _reference_line_envelope( target_dims.append(TargetDimSpec(column="cat", kind="categorical")) spec = CubeSpec( source_name="_reference", - free=FreeAxisSpec(column="a", kind="continuous", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec(column="a", kind="continuous", p=grid[2], domain=grid[:2]), target_dims=tuple(target_dims), measure=MeasureSpec(agg="line_env", value_col="a"), ) result = build_cube(build_df.lazy(), spec) - sliced = result.slice_agg( - a_lo + lo_bin * span / _P, a_lo + (hi_bin + 1) * span / _P - ) + sliced = result.slice_agg(edge_lo, edge_hi) if group_value is not None: sliced = sliced.filter(pl.col("cat") == group_value) sliced = sliced.sort("__bin__b") @@ -5043,10 +5052,7 @@ def test_hist_source_live_updates_line_target(self, page: Page, server_port: int return g && g.lastBins ? g.lastBins : null; }""") assert snap is not None, "expected an active live gesture mid-drag" - a_lo, a_hi = float(df["a"].min()), float(df["a"].max()) - span = a_hi - a_lo - edge_lo = a_lo + snap[0] * span / _P - edge_hi = a_lo + (snap[1] + 1) * span / _P + edge_lo, edge_hi = _bins_commit_edges(snap, _free_grid(df)) expected = _reference_line_envelope(df, edge_lo, edge_hi) assert len(expected["x"]) > 0 _assert_xy_close(live_xy, expected) @@ -5274,10 +5280,7 @@ def test_grouped_line_target_live_updates_children( return g && g.lastBins ? g.lastBins : null; }""") assert snap is not None - a_lo, a_hi = float(df["a"].min()), float(df["a"].max()) - span = a_hi - a_lo - edge_lo = a_lo + snap[0] * span / _P - edge_hi = a_lo + (snap[1] + 1) * span / _P + edge_lo, edge_hi = _bins_commit_edges(snap, _free_grid(df)) by_name = {t["name"]: t for t in children_mid} for gv in ("A", "B"): @@ -5368,10 +5371,10 @@ def _reference_corr_matrix( """Python reference for the LIVE corr matrix: a ``flexviz.cube`` corr build + ``corr_matrix`` finalize over the committed ``closed="left"`` brush edges (== the client ``corrDeltaFromEntry`` path).""" - a_lo, a_hi = float(df["a"].min()), float(df["a"].max()) + lo, hi, bars = _free_grid(df) spec = CubeSpec( source_name="_reference", - free=FreeAxisSpec(column="a", kind="continuous", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec(column="a", kind="continuous", p=bars, domain=(lo, hi)), target_dims=(), measure=MeasureSpec(agg="corr", columns=tuple(columns)), ) @@ -5408,9 +5411,7 @@ def _snap_edges_from_gesture(page: Page, df: pl.DataFrame) -> tuple[float, float return g && g.lastBins ? g.lastBins : null; }""") assert snap is not None, "expected an active live gesture mid-drag" - a_lo, a_hi = float(df["a"].min()), float(df["a"].max()) - span = a_hi - a_lo - return a_lo + snap[0] * span / _P, a_lo + (snap[1] + 1) * span / _P + return _bins_commit_edges(snap, _free_grid(df)) class TestCorrTargetCube: @@ -5614,7 +5615,8 @@ def test_spearman_corr_mixed_commit_self_heals(self, page: Page, server_port: in # handleSelecting and so works for box2d unchanged. The tests below prove the # standard path (live updates + one cube_request) and the edit-drag replay. -_BOX2D_P = 128 +_BOX2D_NX, _BOX2D_NY = 8, 6 +_BOX2D_P = 128 # the fixed per-axis grid of a box2d free axis def _box2d_df() -> pl.DataFrame: @@ -5643,7 +5645,7 @@ def _box2d_dashboard_url(port: int, source_name: str, live_brush: str = "auto") dash = Dashboard(df) dash.add_figure(title="Box2dSource").add_histogram2d( - x="a", y="b", x_bins=8, y_bins=6 + x="a", y="b", x_bins=_BOX2D_NX, y_bins=_BOX2D_NY ) dash.add_figure(title="Hist").add_histogram(x="c", bins=_TGT_BINS) dash.add_figure(title="Bar").add_bar(labels="g") @@ -5666,17 +5668,6 @@ def _box2d_drag_coords(page: Page) -> dict: } -def _edges_to_bins(lo: float, hi: float, dlo: float, dhi: float) -> tuple[int, int]: - """Recover the [lo_bin, hi_bin] inclusive bin range from snapped closed-left - edges over a P=128 grid (the committed edges are edge(lo_bin) and - edge(hi_bin+1) — do NOT re-snap them, which would add one bin).""" - span = dhi - dlo - lo_bin = round((lo - dlo) / span * _BOX2D_P) - hi_bin = round((hi - dlo) / span * _BOX2D_P) - 1 - assert 0 <= lo_bin <= hi_bin <= _BOX2D_P - return lo_bin, hi_bin - - def _reference_box2d_hist_counts( df: pl.DataFrame, ex: tuple[float, float], ey: tuple[float, float] ) -> list[float]: @@ -5706,9 +5697,9 @@ def _reference_box2d_hist_counts( measure=MeasureSpec(agg="count"), ) result = build_cube(df.lazy(), spec) - lx, hx = _edges_to_bins(ex[0], ex[1], a_lo, a_hi) - ly, hy = _edges_to_bins(ey[0], ey[1], b_lo, b_hi) - s = _BOX2D_P + 1 + lx, hx = _committed_bins(ex[0], ex[1], (a_lo, a_hi, _BOX2D_P)) + ly, hy = _committed_bins(ey[0], ey[1], (b_lo, b_hi, _BOX2D_P)) + s = _BOX2D_P codes = [] for by in range(ly, hy + 1): codes.extend(range(by * s + lx, by * s + hx + 1)) @@ -5828,15 +5819,8 @@ def test_box2d_commit_skips_post_with_two_clauses( ex = tuple(by_col["a"]["range"]) ey = tuple(by_col["b"]["range"]) - # The edges lie on the per-axis P=128 grid over the full domains. - a_lo, a_hi = df["a"].min(), df["a"].max() - b_lo, b_hi = df["b"].min(), df["b"].max() - for (lo, hi), (dlo, dhi) in ((ex, (a_lo, a_hi)), (ey, (b_lo, b_hi))): - span = dhi - dlo - klo = round((lo - dlo) / span * _BOX2D_P) - khi = round((hi - dlo) / span * _BOX2D_P) - assert lo == pytest.approx(dlo + klo * span / _BOX2D_P, abs=1e-9) - assert hi == pytest.approx(dlo + khi * span / _BOX2D_P, abs=1e-9) + # The edges are the kernel's bin boundaries of the P=128 grid over the + # full domains (_reference_box2d_hist_counts checks them). expected = _reference_box2d_hist_counts(df, ex, ey) assert 0 < sum(expected) < df.height @@ -6541,3 +6525,138 @@ def test_mean_agg_treemap_live_and_commit_parity( assert len(committed["values"]) == len(ref["values"]) for got, want in zip(committed["values"], ref["values"]): assert got == pytest.approx(want, rel=1e-9, abs=1e-9), (got, want) + + +# --------------------------------------------------------------------------- +# Review follow-ups: empty frames, header guard, source choice, snap parity +# --------------------------------------------------------------------------- + + +class TestCubeGridGuards: + def test_js_snap_matches_python_snap_brush(self, page: Page, server_port: int): + """fvCubeSnap and flexviz.cube.snap_brush agree bit for bit.""" + url = _two_hist_dashboard_url(server_port, "_cube_snap_parity", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + cases = [] + for lo, hi in ( + (0.0, 100.0), + (-50.0, -10.25), + (0.0, 1.0), + (-1.0, 1.0), # an edge next to zero + (1.6e15, 1.6e15 + 8.64e10), + # Past 2**53 a float skips whole µs. + (1.04e16, 1.04e16 + 1000.0), + (1.5778368e15, 2.534022144e17), + ): + for p in (1, 20, 2048): + step = (hi - lo) / p + for a, b in ( + (lo - 3 * step, hi + 3 * step), + (lo + 0.5 * step, lo + 2.5 * step), + (lo + 2.5 * step, lo + 0.5 * step), + (lo + 0.25 * step, lo + 0.45 * step), + (lo + (p - 0.5) * step, hi), + (lo + 0.3 * step, lo + 0.3 * step), + ): + cases.append([lo, hi, p, a, b]) + # A zero span: a brush over lo covers every bar, one beside it none. + cases += [ + [1e7, 1e7, 20, 1e7 - 0.01, 1e7 + 0.01], + [1e7, 1e7, 20, 1e7 + 0.1, 1e7 + 0.3], + ] + js = page.evaluate( + """(cases) => cases.map(([lo, hi, p, a, b]) => { + const s = fvCubeSnap([lo, hi], p, a, b); + return [s.loBin, s.hiBin, s.edgeLo, s.edgeHi, s.closed]; + })""", + cases, + ) + # A whole-valued double comes back as a Python int; float() restores it. + js = [ + [lo_bin, hi_bin, float(e_lo), float(e_hi), c] + for lo_bin, hi_bin, e_lo, e_hi, c in js + ] + py = [list(snap_brush(*c)) for c in cases] + assert js == py + + def test_temporal_commit_edges_are_the_kernel_bin_starts( + self, page: Page, server_port: int + ): + """A committed µs edge is the first whole µs the kernel puts in its + bin, so a restored selection keeps the rows the brush showed. The ceil + of lo + (k - eps) * step alone rounds to 0.25 µs near today's epochs + and lands about one edge in ten one µs low.""" + from datetime import datetime, timedelta + + url = _two_hist_dashboard_url(server_port, "_cube_unit_edges", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + rng = random.Random(7) + cases = [] + for _ in range(300): + # A zoomed domain starts on a lattice point, rarely a whole µs. + lo = 1.7e15 + rng.randrange(10**11) + rng.choice((0.0, 0.25, 0.5)) + hi = lo + rng.randrange(10**6, 10**13) + p = rng.choice((20, 37, 2048)) + cases.append([lo, hi, p, rng.randrange(1, p - 1)]) + edges = page.evaluate( + """(cases) => cases.map(([lo, hi, p, k]) => { + const step = (hi - lo) / p; + const a = lo + (k + 0.25) * step, b = lo + (k + 0.75) * step; + return _fvCubeCommitEdges(fvCubeSnap([lo, hi], p, a, b), 'us'); + })""", + cases, + ) + epoch = datetime(1970, 1, 1) + for (lo, hi, p, k), strs in zip(cases, edges): + e_lo, e_hi = ( + (datetime.fromisoformat(s) - epoch) // timedelta(microseconds=1) + for s in strs + ) + probe = pl.Series([e_lo - 1, e_lo, e_hi - 1, e_hi], dtype=pl.Float64) + bins = pl.select(_fixed_hist_bin_expr(pl.lit(probe), lo, hi, p, "b")) + assert bins.to_series().to_list() == [k - 1, k, k, k + 1], (lo, hi, p, k) + + def test_commit_edges_past_exact_float_integers_return( + self, page: Page, server_port: int + ): + """Past 2**53 µs (the year 2255) a float skips whole µs. A 9999-12-31 + sentinel puts even the first bar's upper edge there: the commit of a + brush over that bar must return, on the first µs of each bin.""" + from datetime import datetime, timedelta + + url = _two_hist_dashboard_url(server_port, "_cube_far_edges", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + lo, hi = 1.5778368e15, 2.534022144e17 # 2020-01-01, 9999-12-31 in µs + edges = page.evaluate( + """([lo, hi]) => { + const snap = fvCubeSnap([lo, hi], 20, lo, lo + 0.7 * (hi - lo) / 20); + return _fvCubeCommitEdges(snap, 'us'); + }""", + [lo, hi], + ) + _, _, _, e_hi, _ = snap_brush(lo, hi, 20, lo, lo + 0.7 * (hi - lo) / 20) + assert edges[0] == "2020-01-01 00:00:00.000000" + want = datetime(1970, 1, 1) + timedelta(microseconds=math.ceil(e_hi)) + assert edges[1] == want.strftime("%Y-%m-%d %H:%M:%S.%f") + + def test_constant_column_brush_selects_every_row( + self, page: Page, server_port: int + ): + """A constant column draws every bar on its one value. A narrow brush + over it highlights them all, so the commit must keep every row.""" + df = pl.DataFrame({"a": [1e7] * 10, "b": [float(v) for v in range(10)]}) + url = _two_hist_dashboard_url(server_port, "_cube_constant", "auto", df=df) + page.goto(url) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + _brush_commit(page, 0, 0.48, 0.52) + page.wait_for_timeout(300) + selections = page.evaluate("DASHBOARD_SPEC.state.selections") + assert len(selections) == 1 + (clause,) = selections[0]["predicates"][0]["clauses"] + assert clause["range"] == [1e7, 1e7] + assert clause["closed"] == "both" + assert sum(_target_y(page)) == 10 diff --git a/tests/test_cube.py b/tests/test_cube.py index 1450ac1e..574a4ade 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -19,6 +19,7 @@ FreeAxisSpec, MeasureSpec, TargetDimSpec, + _fixed_hist_bin_expr, build_cube, cube_content_key, decode_cube_bundle, @@ -254,36 +255,167 @@ def test_zoomed_domain_edge_bins_not_inflated(self, df): (pl.col("active") >= lo) & (pl.col("active") < lo + width) ).height assert bin0 == direct0 - # Bin P-1 covers [75 - width, 75): integer data has no values there, and - # rows above 75 must not be clipped into it. + # Bin P-1 covers [75 - width, 75] (the top clamp takes the domain + # max), and rows above 75 must not be clipped into it. top = cube.frame.filter(pl.col("free_bin") == p - 1)["count"].sum() direct_top = df.filter( - (pl.col("active") >= hi - width) & (pl.col("active") < hi) + (pl.col("active") >= hi - width) & (pl.col("active") <= hi) ).height assert top == direct_top - def test_domain_max_lands_in_degenerate_top_bin(self): + def test_domain_max_lands_in_top_bin(self): df = pl.DataFrame({"active": [0.0, 50.0, 100.0], "cat": ["a", "a", "a"]}) cube = build_cube(df.lazy(), _cat_spec()) - assert cube.frame.filter(pl.col("free_bin") == 64)["count"].sum() == 1 - # A full-range slice includes the degenerate bin. + assert cube.frame["free_bin"].max() == 63 + assert cube.frame.filter(pl.col("free_bin") == 63)["count"].sum() == 1 assert cube.slice_count(0.0, 100.0)["count"].sum() == 3 - def test_snap_clamps_to_degenerate_top_bin(self, df): + def test_integer_values_on_bin_edges_bin_like_the_kernel(self): + # 0..100 in 58 bins puts 50 on an edge, and float rounding moves that + # edge a hair above 50: a plain floor drops 50 one bin low, the + # kernel's round-epsilon keeps it. + hi, p = 100.0, 58 + df = pl.DataFrame( + {"active": [float(v) for v in range(101)], "cat": ["a"] * 101} + ) + spec = CubeSpec( + source_name="s", + free=FreeAxisSpec(column="active", p=p, domain=(0.0, hi)), + target_dims=(TargetDimSpec(column="cat", kind="categorical"),), + measure=MeasureSpec(agg="count"), + ) + cube = build_cube(df.lazy(), spec) + got = dict(zip(cube.frame["free_bin"], cube.frame["count"])) + kernel = ( + df.select( + pl.col("active") + .flexviz.fixed_hist(pl.lit(0.0), pl.lit(hi), n_bins=p) + .implode() + ) + .item() + .struct.unnest()["count"] + .to_list() + ) + assert [got.get(k, 0) for k in range(p)] == kernel + + def test_snap_is_nearest_edge(self, df): cube = build_cube(df.lazy(), _cat_spec()) - assert cube._snap(0.0, 100.0) == (0, 64) - assert cube._snap(100.0, 100.0) == (64, 64) - assert cube._snap(-10.0, 200.0) == (0, 64) - # Reversed endpoints swap, as before. - assert cube._snap(100.0, 0.0) == (0, 64) + # 64 bins over [0, 100]: the step is 1.5625. + assert cube._snap(0.0, 100.0) == (0, 63) + assert cube._snap(-10.0, 200.0) == (0, 63) + assert cube._snap(100.0, 0.0) == (0, 63) + # Covers the center of bin 0 only. + assert cube._snap(0.5, 1.2) == (0, 0) + # Covers no bin center: an empty snap. + lo_bin, hi_bin = cube._snap(0.1, 0.5) + assert hi_bin < lo_bin + + def test_zero_span_snap_covers_every_bar_or_none(self): + # A constant column draws every bar on lo: a brush over lo selects + # all of them, however narrow, and a brush beside it selects none. + from flexviz.cube import snap_brush + + v = 1e7 + assert snap_brush(v, v, 20, v - 0.01, v + 0.01) == (0, 19, v, v, "both") + lo_bin, hi_bin, *_ = snap_brush(v, v, 20, v + 0.1, v + 0.3) + assert hi_bin < lo_bin + + def test_brush_end_on_a_bar_center_includes_that_bar(self): + # Plotly's box selection includes a point on its edge, so a brush + # from center to center highlights both bars, at either end. + from flexviz.cube import snap_brush + + assert snap_brush(0.0, 4.0, 4, 0.5, 1.5)[:2] == (0, 1) + assert snap_brush(0.0, 4.0, 4, 1.5, 0.5)[:2] == (0, 1) + assert snap_brush(0.0, 4.0, 4, 1.5, 1.5)[:2] == (1, 1) + # A center as the display draws it, (i + 0.5) * step + lo, can carry + # float noise: bar 1 of 20 over [0, 1] sits at 0.07500000000000001. + step = 1.0 / 20 + c1, c3 = (1 + 0.5) * step + 0.0, (3 + 0.5) * step + 0.0 + assert snap_brush(0.0, 1.0, 20, c1, 0.1)[:2] == (1, 1) + assert snap_brush(0.0, 1.0, 20, 0.1, c3)[:2] == (2, 3) + + def test_edge_next_to_zero_is_found_in_few_steps(self): + # Near a zero crossing billions of doubles share one value of v - lo, + # so the first double of bin 10 must be searched, not walked to. + from flexviz.cube import snap_brush + + lo_bin, hi_bin, e_lo, _, _ = snap_brush(-1.0, 1.0, 20, 0.02, 0.08) + assert (lo_bin, hi_bin) == (10, 10) + + def kernel_bin(v: float) -> int: + return math.floor((v + 1.0) * 10.0 + 1e-9) + + assert kernel_bin(e_lo) == 10 + assert kernel_bin(math.nextafter(e_lo, -math.inf)) == 9 + + @pytest.mark.parametrize("dtype", [pl.Float64, pl.Int64]) + def test_committed_edges_keep_the_kernel_rows(self, dtype): + # Values packed around every edge, on the column's own value grid: a + # committed range keeps exactly the rows the kernel counts in its bars. + # The float edge lo + (k - eps) * step alone can put a value one ulp + # (or, for a large integer, one unit) on the wrong side. + from flexviz.cube import snap_brush + from flexviz.predicates import predicates_to_expr + from flexviz.spec import ClauseFilter, SelectionPredicate + + rng = np.random.default_rng(7) + for _ in range(40): + p = int(rng.choice([3, 7, 20, 2048])) + span = float(rng.choice([1.0, rng.uniform(1e-3, 1e4), 3e10])) + # The last choice is a domain across zero. + lo = float(rng.choice([0.0, rng.uniform(-1e3, 1e3), 1.7e15, -span / 2])) + if dtype == pl.Int64: + lo, span = float(math.floor(lo)), float(math.floor(span) + p) + hi = lo + span + step = (hi - lo) / p + vals = [lo, hi] + for k in range(1, min(p, 50)): + for e in (lo + k * step, lo + (k - 1e-9) * step): + if dtype == pl.Int64: + vals += [float(math.floor(e) + d) for d in (-1, 0, 1)] + else: + vals += [ + e, + math.nextafter(e, -math.inf), + math.nextafter(e, math.inf), + ] + df = pl.DataFrame({"a": pl.Series(vals).cast(dtype)}) + df = df.filter(pl.col("a").is_between(lo, hi)) + bins = df.select(_fixed_hist_bin_expr(pl.col("a"), lo, hi, p, "b"))["b"] + kernel = ( + df.select( + pl.col("a") + .flexviz.fixed_hist(pl.lit(lo), pl.lit(hi), n_bins=p) + .implode() + ) + .item() + .struct.unnest()["count"] + .to_list() + ) + assert bins.value_counts().sort("b")["count"].to_list() == [ + c for c in kernel if c + ] + for _ in range(10): + k = int(rng.integers(0, min(p, 50))) + m = int(rng.integers(k, min(p, 50))) + lo_bin, hi_bin, e_lo, e_hi, closed = snap_brush( + lo, hi, p, lo + (k + 0.25) * step, lo + (m + 0.75) * step + ) + if hi_bin < lo_bin: + continue + clause = ClauseFilter(column="a", range=(e_lo, e_hi), closed=closed) + pred = SelectionPredicate(clauses=[clause]) + got = df.select(predicates_to_expr([pred], df.schema))[:, 0] + assert got.equals(bins.is_between(lo_bin, hi_bin)), (lo, hi, p, k, m) class TestFixedHistParity: def test_binned_target_matches_fixed_hist_kernel(self): - # Same setup hist.py produces: hi = axis_hi + 1e-10 epsilon, integer - # values on every visual bin boundary, plus values exactly at every - # internal bin edge of the (lo, hi, n) grid. - lo, hi, n = 0.0, 100.0 + 1e-10, 10 + # Integer values, one of them (50) a hair below a bin edge after + # float rounding, plus values exactly at every internal bin edge of + # the (lo, hi, n) grid. + lo, hi, n = 0.0, 100.0, 58 step = (hi - lo) / n values = [float(v) for v in range(101)] values += [lo + k * step for k in range(1, n)] @@ -1119,17 +1251,17 @@ def test_rebuilt_sum_partials_equivalent(self, agg): b = build_cube(hostile.lazy(), _measure_spec(agg)) _assert_rebuild_equivalent(encode_fvcube(a, "k"), encode_fvcube(b, "k")) - def test_degenerate_top_bin_survives_round_trip(self): + def test_top_bin_survives_round_trip(self): df = pl.DataFrame({"active": [0.0, 50.0, 100.0], "cat": ["a", "a", "a"]}) cube = build_cube(df.lazy(), _cat_spec()) blob = encode_fvcube(cube, "k") header = decode_fvcube_header(blob) free_bin = _read_u32_col(blob, header, "free_bin") - assert 64 in free_bin # bin == P present in the encoded rows - # A reslice reaching the domain max includes the degenerate bin… + assert 63 in free_bin # the domain max is in the top bin P-1 + # A reslice reaching the domain max includes the top bin… full = self._reslice(blob, cube, 0.0, 100.0) assert sum(full.values()) == 3 - # …and a reslice stopping short of it does not. + # …and a reslice ending before its center does not. partial = self._reslice(blob, cube, 0.0, 99.0) assert sum(partial.values()) == 2 @@ -1618,10 +1750,7 @@ def test_header_carries_unit(self, unit): header = decode_fvcube_header(blob) assert header["free"]["kind"] == "temporal" assert header["free"]["unit"] == unit - if unit == "day": - assert "w" in header["free"] and "p_eff" in header["free"] - else: - assert "w" not in header["free"] + assert header["free"]["p"] == 2048 def test_ns_not_supported_by_temporal_unit(self): from flexviz.cube import temporal_unit @@ -1630,31 +1759,24 @@ def test_ns_not_supported_by_temporal_unit(self): assert temporal_unit(pl.Time) is None assert temporal_unit(pl.Float64) is None - def test_day_grid_short_span_unit_width(self): - from flexviz.cube import day_grid - - # span < 2048 days ⇒ w = 1, P' = span (whole days). - w, p_eff = day_grid(0.0, 364.0, 2048) - assert w == 1 - assert p_eff == 364 - - def test_day_grid_long_span_integer_width(self): - from flexviz.cube import day_grid - - # span 10000 days ⇒ w = ceil(10000/2048) = 5, P' = ceil(10000/5) = 2000. - w, p_eff = day_grid(0.0, 10_000.0, 2048) - assert w == 5 - assert p_eff == 2000 - # Every snap edge lo + k*w is an integer day by construction. - @pytest.mark.parametrize( - "unit,span_days", [("us", 365), ("ms", 365), ("day", 365), ("day", 9000)] + "unit,span_days,p", + [ + ("us", 365, 2048), + ("ms", 365, 2048), + ("day", 365, 2048), + ("day", 9000, 2048), + # Histogram-sized grids put fractional-day edges between the days. + ("day", 365, 20), + ("day", 100, 20), + ], ) - def test_slice_membership_equals_string_predicate(self, unit, span_days): - """§8.2 parity per unit: a cube slice over snapped bins selects - exactly the rows the committed (rendered-string, closed="left") - predicate selects through _typed_range_bounds — bit-exact counts.""" - from flexviz.cube import day_grid + def test_slice_membership_equals_string_predicate(self, unit, span_days, p): + """§8.2 parity per unit: for a brush over bins k..m, the cube slice + selects exactly the rows the committed string predicate selects + through _typed_range_bounds — for every lower edge k and for a brush + up to the top bin (closed="both").""" + from flexviz.cube import snap_brush from flexviz.predicates import predicates_to_expr from flexviz.spec import ClauseFilter, SelectionPredicate @@ -1663,7 +1785,7 @@ def test_slice_membership_equals_string_predicate(self, unit, span_days): spec = CubeSpec( source_name="src", free=FreeAxisSpec( - column="t", kind="temporal", p=2048, domain=(lo, hi), unit=unit + column="t", kind="temporal", p=p, domain=(lo, hi), unit=unit ), target_dims=( TargetDimSpec(column="b", kind="binned", bins=12, domain=(0.0, 100.0)), @@ -1671,49 +1793,40 @@ def test_slice_membership_equals_string_predicate(self, unit, span_days): measure=MeasureSpec(agg="count"), ) result = build_cube(df.lazy(), spec) - - if unit == "day": - w, p_eff = day_grid(lo, hi, 2048) - else: - w, p_eff = (hi - lo) / 2048, 2048 - - # Snap a brush to [lo_bin, hi_bin] (roughly the middle half). - lo_bin = p_eff // 4 - hi_bin = (3 * p_eff) // 4 - edge_lo = lo + lo_bin * w - edge_hi = lo + (hi_bin + 1) * w - sliced = ( - result.frame.filter(pl.col("free_bin").is_between(lo_bin, hi_bin))[ - "count" - ].sum() - or 0 - ) - - str_lo = _physical_to_temporal_str(edge_lo, unit) - str_hi = _physical_to_temporal_str(edge_hi, unit) - if unit == "day": - # Integer-day grid: the strings round-trip the edges bit-exactly. - assert date.fromisoformat(str_lo) == date(1970, 1, 1) + timedelta( - days=edge_lo + step = (hi - lo) / p + brushes = [(k, min(k + p // 4, p - 1)) for k in range(0, p, max(1, p // 20))] + brushes.append((p // 2, p - 1)) + for k, m in brushes: + # A brush covering the centers of bins k..m. + lo_bin, hi_bin, e_lo, e_hi, closed = snap_brush( + lo, hi, p, lo + (k + 0.25) * step, lo + (m + 0.75) * step ) - assert date.fromisoformat(str_hi) == date(1970, 1, 1) + timedelta( - days=edge_hi + assert (lo_bin, hi_bin) == (k, m) + sliced = ( + result.frame.filter(pl.col("free_bin").is_between(lo_bin, hi_bin))[ + "count" + ].sum() + or 0 ) - pred = SelectionPredicate( - clauses=[ClauseFilter(column="t", range=(str_lo, str_hi), closed="left")] - ) - direct = df.filter(predicates_to_expr([pred], df.schema)).height - assert sliced == direct - assert 0 < sliced < df.height - - def test_day_degenerate_top_bin(self): - """A row at the exact domain max lands in the degenerate top bin - P' and a slice reaching it picks the row up.""" - from flexviz.cube import day_grid + # Mirrors _fvCubeCommitEdges: a closed top edge rounds down. + str_lo = _physical_to_temporal_str(e_lo, unit) + str_hi = _physical_to_temporal_str( + math.floor(e_hi) if closed == "both" else e_hi, unit + ) + pred = SelectionPredicate( + clauses=[ + ClauseFilter(column="t", range=(str_lo, str_hi), closed=closed) + ] + ) + direct = df.filter(predicates_to_expr([pred], df.schema)).height + assert sliced == direct, (k, m) + def test_day_top_bin_clamps(self): + """A row at the exact domain max lands in the top bin P-1 and a slice + reaching it picks the row up.""" df = _temporal_df("day", span_days=365) lo, hi = _physical_domain(df, "t") - _w, p_eff = day_grid(lo, hi, 2048) + top_bin = 2047 spec = CubeSpec( source_name="src", free=FreeAxisSpec( @@ -1725,9 +1838,9 @@ def test_day_degenerate_top_bin(self): measure=MeasureSpec(agg="count"), ) result = build_cube(df.lazy(), spec) - assert result.frame["free_bin"].max() == p_eff + assert result.frame["free_bin"].max() == top_bin n_max_rows = df.filter(pl.col("t").to_physical().cast(pl.Float64) == hi).height - top = result.frame.filter(pl.col("free_bin") == p_eff)["count"].sum() + top = result.frame.filter(pl.col("free_bin") == top_bin)["count"].sum() assert top == n_max_rows > 0 def test_temporal_binned_target_dim_carries_unit_and_bins_physical(self): @@ -1774,9 +1887,10 @@ def _line_env_reference( """Pure-Polars build reference for the line envelope (replicated from ``flexviz_polars/tests/test_plugin_functions.py::_envelope_reference``). - Shared-arithmetic semantics on BOTH axes: natural floor bin, NO epsilon, - NO clip, out-of-domain rows FILTERED, degenerate top bins (``0..=n`` - inclusive), null/NaN rows filtered, first-row-in-scan-order tie wins. + The x bucket bins with a natural floor, NO epsilon, NO clip (degenerate + top bucket ``n``); the free axis bins like the display kernel + (``_fixed_hist_bin_expr``, top bin ``p-1``). Out-of-domain rows are + FILTERED, null/NaN rows filtered, first-row-in-scan-order tie wins. The span divisors are materialized as REAL COLUMNS: Polars rewrites float division *by a scalar* into multiplication by the reciprocal, which is @@ -1809,10 +1923,9 @@ def _line_env_reference( .floor() .cast(pl.UInt32) .alias("bucket"), - ((pl.col("__f") - free_lo) / pl.col("__fspan") * float(p)) - .floor() - .cast(pl.UInt32) - .alias("free_bin"), + _fixed_hist_bin_expr(pl.col("__f"), free_lo, free_hi, p, "free_bin").cast( + pl.UInt32 + ), ) .group_by("bucket", "free_bin") .agg( @@ -1870,8 +1983,8 @@ def line_df() -> pl.DataFrame: # x sweeps [0, 100) with sub-integer jitter; y is a distinct-valued # permutation (no y ties — quantized order == exact order, ulp-bound # comparisons stay row-stable); free covers [0, 100). Tail rows pin edge - # cases: domain-max x AND free (both degenerate top bins), out-of-domain - # x, and null/NaN y (all filtered or binned per the shared arithmetic). + # cases: domain-max x (the degenerate top bucket) AND free (the top free + # bin), out-of-domain x, and null/NaN y (all filtered or binned). n = 4096 x = [float(i % 100) + (i % 7) * 0.01 for i in range(n)] y = [((i * 2641) % 4096) * 0.0001 + 0.05 for i in range(n)] @@ -1919,6 +2032,7 @@ def test_rejects_box2d_free_axis(self, line_df): column="free", columns=("free", "x"), kind="box2d", + p=8, domains=((0.0, 100.0), (0.0, 100.0)), ), target_dims=( @@ -1981,9 +2095,9 @@ def test_build_equals_polars_reference(self, line_df): ).select("__bin__x", "free_bin", "y_min", "x_at_ymin", "y_max", "x_at_ymax") got = cube.frame.sort(["free_bin", "__bin__x"]).select(want.columns) assert got.equals(want) - # Degenerate top bins on both axes are present (x == 100, free == 100). + # x == 100 is the degenerate top bucket; free == 100 is top bin P-1. assert cube.frame["__bin__x"].max() == 32 - assert cube.frame["free_bin"].max() == 64 + assert cube.frame["free_bin"].max() == 63 def test_grouped_build_equals_per_group_reference(self, line_df): gdf = line_df.with_columns( @@ -2112,11 +2226,9 @@ def test_tie_keeps_earlier_free_bin_row(self): assert row["x_at_ymax"] == expected_x def test_day_free_axis_full_range_slice(self): - """unit="day" free axis: integer-day grid bins (degenerate bin P'), + """unit="day" free axis: the top bin takes the domain max, and the full-range combine matches the exact per-bucket envelope (y integers ⇒ f32-exact; x within bucket_width/65535).""" - from flexviz.cube import day_grid - n = 244 base = date(2020, 1, 1) df = pl.DataFrame( @@ -2138,10 +2250,8 @@ def test_day_free_axis_full_range_slice(self): measure=MeasureSpec(agg="line_env", value_col="y"), ) cube = build_cube(df.lazy(), spec) - w, p_eff = day_grid(lo, hi, 2048) - assert (w, p_eff) == (1, 60) - # Domain-max rows land in the degenerate day bin P'. - assert cube.frame["free_bin"].max() == p_eff + # Domain-max rows land in the top bin P-1. + assert cube.frame["free_bin"].max() == 2047 out = cube.slice_agg(lo, hi).sort("__bin__x") exact = ( df.with_columns(pl.Series("__span", [10.0] * df.height)) @@ -2490,7 +2600,7 @@ def corr_df() -> pl.DataFrame: x = rng.normal(5.0, 2.0, n) y = rng.normal(-3.0, 4.0, n) + 0.3 * x z = rng.normal(0.0, 1.0, n) - 0.5 * x - # free covers [0, 100); a few tail rows pin the domain-max degenerate bin. + # free covers [0, 100). free = [float((i * 37) % 100) for i in range(n)] return pl.DataFrame({"x": x, "y": y, "z": z, "free": free}) @@ -2527,6 +2637,7 @@ def test_build_rejects_box2d_free_axis(self, corr_df): column="free", columns=("free", "x"), kind="box2d", + p=8, domains=((0.0, 100.0), (0.0, 50.0)), ), target_dims=(), @@ -2879,10 +2990,9 @@ def test_non_corr_key_byte_identical_to_before(self): # 2-D box free axis (contract H — hist2d source, composite CSR free bin) # --------------------------------------------------------------------------- -from flexviz.cube import box2d_composite_stride -_BOX2D_P = 128 -_BOX2D_S = _BOX2D_P + 1 +_BOX2D_PX = _BOX2D_PY = 128 +_BOX2D_S = _BOX2D_PX def _box2d_spec( @@ -2890,7 +3000,7 @@ def _box2d_spec( value_col: str | None = None, x_domain: tuple[float, float] = (0.0, 100.0), y_domain: tuple[float, float] = (0.0, 50.0), - p: int = _BOX2D_P, + p: int = _BOX2D_PX, ) -> CubeSpec: return CubeSpec( source_name="s", @@ -2908,13 +3018,14 @@ def _box2d_spec( @pytest.fixture() def box2d_df() -> pl.DataFrame: - # x ∈ [0,100), y ∈ [0,50), a value column tracking x; some rows pin each - # axis's degenerate top bin (x==100 / y==50). + # x ∈ [0,100), y ∈ [0,50), a value column tracking x; some rows sit on + # each axis's domain max (x==100 / y==50), which the top clamp puts in the + # top cell. n = 5_000 x = [float((i * 37) % 100) for i in range(n)] y = [float((i * 53) % 50) for i in range(n)] val = [float((i * 11) % 200) for i in range(n)] - # A few rows exactly on each domain max → the degenerate top bins. + # A few rows exactly on each domain max → the top cells. x += [100.0, 100.0, 50.0] y += [25.0, 50.0, 50.0] val += [1.0, 2.0, 3.0] @@ -2966,10 +3077,6 @@ def test_categorical_rejects_domains(self): column="x", kind="categorical", columns=("x",), domains=((0, 1), (0, 1)) ) - def test_box2d_composite_stride(self): - assert box2d_composite_stride(_BOX2D_P) == 129 - assert box2d_composite_stride(2048) == 2049 - class TestBox2dBuildAndSlice: @pytest.mark.parametrize("agg", ["count", "mean"]) @@ -2992,10 +3099,10 @@ def test_subrange_slice_equals_filtered_direct_unzoomed(self, box2d_df, agg): cube = build_cube(box2d_df.lazy(), spec) (lx, hx), (ly, hy) = cube._snap_box2d(20.0, 70.0, 10.0, 40.0) # Recover the snapped closed-left rectangle edges. - x_lo = 0.0 + lx / _BOX2D_P * 100.0 - x_hi = 0.0 + (hx + 1) / _BOX2D_P * 100.0 - y_lo = 0.0 + ly / _BOX2D_P * 50.0 - y_hi = 0.0 + (hy + 1) / _BOX2D_P * 50.0 + x_lo = 0.0 + lx / _BOX2D_PX * 100.0 + x_hi = 0.0 + (hx + 1) / _BOX2D_PX * 100.0 + y_lo = 0.0 + ly / _BOX2D_PY * 50.0 + y_hi = 0.0 + (hy + 1) / _BOX2D_PY * 50.0 sliced = cube.slice_agg_box2d(20.0, 70.0, 10.0, 40.0) got = sliced["value"].item() if sliced.height else None want = _direct_box2d_agg(box2d_df, agg, x_lo, x_hi, y_lo, y_hi) @@ -3015,10 +3122,10 @@ def test_subrange_slice_equals_filtered_direct_zoomed(self, box2d_df, agg): ) cube = build_cube(box2d_df.lazy(), spec) (lx, hx), (ly, hy) = cube._snap_box2d(30.0, 65.0, 15.0, 35.0) - x_lo = 20.0 + lx / _BOX2D_P * 60.0 - x_hi = 20.0 + (hx + 1) / _BOX2D_P * 60.0 - y_lo = 10.0 + ly / _BOX2D_P * 35.0 - y_hi = 10.0 + (hy + 1) / _BOX2D_P * 35.0 + x_lo = 20.0 + lx / _BOX2D_PX * 60.0 + x_hi = 20.0 + (hx + 1) / _BOX2D_PX * 60.0 + y_lo = 10.0 + ly / _BOX2D_PY * 35.0 + y_hi = 10.0 + (hy + 1) / _BOX2D_PY * 35.0 sliced = cube.slice_agg_box2d(30.0, 65.0, 15.0, 35.0) got = sliced["value"].item() if sliced.height else None want = _direct_box2d_agg(box2d_df, agg, x_lo, x_hi, y_lo, y_hi) @@ -3027,14 +3134,14 @@ def test_subrange_slice_equals_filtered_direct_zoomed(self, box2d_df, agg): else: assert got == pytest.approx(want, rel=1e-9) - def test_degenerate_top_bins_each_axis(self, box2d_df): + def test_top_bins_clamp_each_axis(self, box2d_df): spec = _box2d_spec("count") cube = build_cube(box2d_df.lazy(), spec) - # A brush reaching both domain maxima selects the degenerate top bins - # (bin == P on each axis) — the composite index P*S + P must be present. - (_lx, hx), (_ly, hy) = cube._snap_box2d(100.0, 100.0, 50.0, 50.0) - assert hx == _BOX2D_P and hy == _BOX2D_P - top_code = _BOX2D_P * _BOX2D_S + _BOX2D_P + # A brush over both top cells selects bin n-1 on each axis, and the + # (x==100, y==50) row is clamped into that top cell. + (_lx, hx), (_ly, hy) = cube._snap_box2d(99.9, 100.0, 49.9, 50.0) + assert hx == _BOX2D_PX - 1 and hy == _BOX2D_PY - 1 + top_code = (_BOX2D_PY - 1) * _BOX2D_S + _BOX2D_PX - 1 codes = cube.frame["free_bin"].to_list() assert top_code in codes # the (x==100, y==50) row landed in the top cell # Slicing the full rectangle conserves the total count. @@ -3053,7 +3160,7 @@ def test_filter_dont_clip_drops_out_of_domain(self): def test_composite_free_bin_is_u32_range(self, box2d_df): cube = build_cube(box2d_df.lazy(), _box2d_spec("count")) codes = cube.frame["free_bin"].to_list() - assert all(0 <= c <= _BOX2D_S * _BOX2D_S - 1 for c in codes) + assert all(0 <= c <= _BOX2D_PX * _BOX2D_PY - 1 for c in codes) class TestBox2dCodec: @@ -3064,7 +3171,7 @@ def _reslice_rect(self, blob: bytes, cube: CubeResult, x: tuple, y: tuple) -> in free_bin = _read_u32_col(blob, header, "free_bin") count = _read_u32_col(blob, header, "count") (lx, hx), (ly, hy) = cube._snap_box2d(x[0], x[1], y[0], y[1]) - s = header["free"]["p"] + 1 + s = header["free"]["p"] codes = set() for by in range(ly, hy + 1): row = by * s diff --git a/tests/test_cube_server.py b/tests/test_cube_server.py index 10b00942..d4c8401b 100644 --- a/tests/test_cube_server.py +++ b/tests/test_cube_server.py @@ -19,7 +19,7 @@ import flexviz_polars # noqa: F401 — registers pl.Expr.flexviz namespace from flexviz.cache import get_cache, get_cube_cache -from flexviz.cube import decode_cube_bundle, decode_fvcube_header +from flexviz.cube import decode_cube_bundle, decode_fvcube_header, snap_brush from flexviz.dashboard import Dashboard from flexviz.server import app, register_source from flexviz.spec import AxisRange @@ -213,22 +213,20 @@ def _direct_hist( return raw.explode(empty_as_null=True).struct.unnest()["count"].to_list() -def _snap(domain: tuple[float, float], a: float, b: float): - """Shared P=2048 snap arithmetic (plan doc, copied verbatim).""" - lo, hi = domain - s = hi - lo +def _snap(free: dict, a: float, b: float): + """``snap_brush`` over a free block ``{"domain", "p"}`` for a brush that + ends below the top bin: ``(lo_bin, hi_bin, edge_lo, edge_hi)`` of a + ``closed="left"`` commit.""" + lo, hi = free["domain"] + lo_bin, hi_bin, edge_lo, edge_hi, closed = snap_brush(lo, hi, free["p"], a, b) + assert closed == "left" + return lo_bin, hi_bin, edge_lo, edge_hi - def _bin(v: float) -> int: - return math.floor((v - lo) / s * _P) - def _edge(bin_idx: int) -> float: - return lo + bin_idx * s / _P - - lo_bin = max(0, min(_P, _bin(a))) - hi_bin = max(0, min(_P, _bin(b))) - if hi_bin < lo_bin: - lo_bin, hi_bin = hi_bin, lo_bin - return lo_bin, hi_bin, _edge(lo_bin), _edge(hi_bin + 1) +def _hist_free(lo: float, hi: float) -> dict: + """An unzoomed histogram source's free block: the fixed P grid over the + full data domain.""" + return {"domain": [lo, hi], "p": _P} # --------------------------------------------------------------------------- @@ -264,18 +262,14 @@ def test_blob_reslice_matches_direct_recompute(self, client, df): # target domain = full data domain of "b" + epsilon. a_lo, a_hi = df["a"].min(), df["a"].max() b_lo, b_hi = df["b"].min(), df["b"].max() - assert header["free"] == { - "kind": "continuous", - "p": _P, - "domain": [a_lo, a_hi], - } + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} (dim,) = header["target_dims"] assert dim["name"] == "b" assert dim["bins"] == 12 assert dim["domain"] == [b_lo, b_hi] # Slice the blob over a snapped brush. - lo_bin, hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 12.3, 61.7) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 12.3, 61.7) free_bin = _read_u32_col(blob, header, "free_bin") tgt_bin = _read_u32_col(blob, header, "__bin__b") count = _read_u32_col(blob, header, "count") @@ -955,15 +949,11 @@ def test_sum_bar_target_from_hist_source_matches_direct_recompute( blob = base64.b64decode(body["cubes"][body["trace_cubes"][tgt_uid]]) header = decode_fvcube_header(blob) a_lo, a_hi = cat_df["a"].min(), cat_df["a"].max() - assert header["free"] == { - "kind": "continuous", - "p": _P, - "domain": [a_lo, a_hi], - } + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} assert header["measure"] == {"agg": "sum", "value_col": "b"} (dim,) = header["target_dims"] - lo_bin, hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 12.3, 61.7) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 12.3, 61.7) free_bin = _read_u32_col(blob, header, "free_bin") sub_code = _read_u32_col(blob, header, "sub") sums = _read_f64_col(blob, header, "sum") @@ -1300,7 +1290,7 @@ def test_baked_parity_count_and_mean(self, cat_client, cat_df): passive_expr = pl.col("cat").is_in(["alpha", "beta"]) a_lo, a_hi = cat_df["a"].min(), cat_df["a"].max() - lo_bin, hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 21.7, 68.3) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(_hist_free(a_lo, a_hi), 21.7, 68.3) active_expr = pl.col("a").is_between(edge_lo, edge_hi, closed="left") # --- count parity on the hist target --- @@ -1582,9 +1572,7 @@ def test_zoomed_source_slice_parity(self, client, df): assert header["free"]["domain"] == [10.0, 80.0] (dim,) = header["target_dims"] - # Snap a brush on the ZOOMED grid (finer than the full-domain grid). - lo_bin, hi_bin, edge_lo, edge_hi = _snap((10.0, 80.0), 23.4, 57.8) - assert (80.0 - 10.0) / _P < (df["a"].max() - df["a"].min()) / _P + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 23.4, 57.8) free_bin = _read_u32_col(blob, header, "free_bin") tgt_bin = _read_u32_col(blob, header, "__bin__b") count = _read_u32_col(blob, header, "count") @@ -1633,7 +1621,7 @@ def test_zoomed_source_with_passive_combined(self, client, df): assert header["free"]["domain"] == [10.0, 80.0] (dim,) = header["target_dims"] - lo_bin, hi_bin, edge_lo, edge_hi = _snap((10.0, 80.0), 23.4, 57.8) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 23.4, 57.8) free_bin = _read_u32_col(blob, header, "free_bin") tgt_bin = _read_u32_col(blob, header, "__bin__b") count = _read_u32_col(blob, header, "count") @@ -1672,7 +1660,10 @@ def counting(ldf, spec): assert calls["n"] == 1 assert _cube_body(r1)["cubes"] == _cube_body(r2)["cubes"] header = decode_fvcube_header(base64.b64decode(_cube_body(r1)["cubes"][0])) - assert header["free"]["domain"] == [df["a"].min(), df["a"].max()] + assert ( + header["free"]["domain"] + == _hist_free(df["a"].min(), df["a"].max())["domain"] + ) # --------------------------------------------------------------------------- @@ -1708,8 +1699,7 @@ class TestBoxLineRangeSources: def _slice_and_reference(self, df, blob, header, brush_lo, brush_hi): """Snapped cube slice vs the legacy snapped closed='left' recompute.""" (dim,) = header["target_dims"] - domain = tuple(header["free"]["domain"]) - lo_bin, hi_bin, edge_lo, edge_hi = _snap(domain, brush_lo, brush_hi) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], brush_lo, brush_hi) free_bin = _read_u32_col(blob, header, "free_bin") tgt_bin = _read_u32_col(blob, header, "__bin__b") count = _read_u32_col(blob, header, "count") @@ -1869,10 +1859,6 @@ def test_ns_box_source_returns_empty(self): # --------------------------------------------------------------------------- -_BOX2D_P = 128 -_BOX2D_S = _BOX2D_P + 1 - - def _hist2d_source_dashboard(df: pl.DataFrame, *, title_suffix: str = ""): """A hist2d(x=a, y=b) source figure; a hist(c) count target.""" dash = Dashboard(df) @@ -1883,17 +1869,8 @@ def _hist2d_source_dashboard(df: pl.DataFrame, *, title_suffix: str = ""): return dash.to_spec(source_name=_SRC) -def _snap_axis(domain: tuple[float, float], a: float, b: float, p: int = _BOX2D_P): - lo, hi = domain - s = (hi - lo) or 1.0 - - def _bin(v: float) -> int: - return max(0, min(p, math.floor((v - lo) / s * p))) - - lo_b, hi_b = _bin(a), _bin(b) - if hi_b < lo_b: - lo_b, hi_b = hi_b, lo_b - return lo_b, hi_b, lo + lo_b * s / p, lo + (hi_b + 1) * s / p +def _snap_axis(domain: tuple[float, float], a: float, b: float, p: int): + return snap_brush(domain[0], domain[1], p, a, b) class TestBox2dSourceCubeRequest: @@ -1903,9 +1880,10 @@ def _box2d_reslice_count(self, blob, header, x_box, y_box, df): rectangle edges.""" dx = tuple(header["free"]["domains"][0]) dy = tuple(header["free"]["domains"][1]) - lx, hx, ex0, ex1 = _snap_axis(dx, x_box[0], x_box[1]) - ly, hy, ey0, ey1 = _snap_axis(dy, y_box[0], y_box[1]) - s = header["free"]["p"] + 1 + px = py = header["free"]["p"] + lx, hx, ex0, ex1, cx = _snap_axis(dx, x_box[0], x_box[1], px) + ly, hy, ey0, ey1, cy = _snap_axis(dy, y_box[0], y_box[1], py) + s = px free_bin = _read_u32_col(blob, header, "free_bin") tgt_bin = _read_u32_col(blob, header, "__bin__c") count = _read_u32_col(blob, header, "count") @@ -1921,8 +1899,8 @@ def _box2d_reslice_count(self, blob, header, x_box, y_box, df): sliced[tb] += n direct = _direct_hist( df, - pl.col("a").is_between(ex0, ex1, closed="left") - & pl.col("b").is_between(ey0, ey1, closed="left"), + pl.col("a").is_between(ex0, ex1, closed=cx) + & pl.col("b").is_between(ey0, ey1, closed=cy), "c", dim["domain"][0], dim["domain"][1], @@ -2038,8 +2016,9 @@ def test_passive_selection_bakes_into_build(self, client, df): ] dx = tuple(header["free"]["domains"][0]) dy = tuple(header["free"]["domains"][1]) - _lx, _hx, ex0, ex1 = _snap_axis(dx, 30.0, 60.0) - _ly, _hy, ey0, ey1 = _snap_axis(dy, 10.0, 50.0) + px = py = header["free"]["p"] + _lx, _hx, ex0, ex1, _cx = _snap_axis(dx, 30.0, 60.0, px) + _ly, _hy, ey0, ey1, _cy = _snap_axis(dy, 10.0, 50.0, py) passive_expr = pl.col("a").is_between(20.0, 70.0, closed="left") sliced, _ = self._box2d_reslice_count( blob, header, (30.0, 60.0), (10.0, 50.0), df @@ -2136,7 +2115,7 @@ def test_serves_corr_cube_with_empty_target_dims(self, corr_client, corr_df): assert header["measure"]["pairs"] == [[0, 1], [0, 2], [1, 2]] assert set(header["measure"]["means"]) == {"p", "q", "rr"} a_lo, a_hi = corr_df["a"].min(), corr_df["a"].max() - assert header["free"] == {"kind": "continuous", "p": _P, "domain": [a_lo, a_hi]} + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} def test_blob_reslice_matches_legacy_recompute(self, corr_client, corr_df): """The client-equivalent reslice (cube ``corr_matrix``) over a snapped @@ -2163,7 +2142,7 @@ def test_blob_reslice_matches_legacy_recompute(self, corr_client, corr_df): base64.b64decode(body["cubes"][body["trace_cubes"][corr_uid]]) ) a_lo, a_hi = header["free"]["domain"] - _lo_bin, _hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 21.4, 73.6) + _lo_bin, _hi_bin, edge_lo, edge_hi = _snap(header["free"], 21.4, 73.6) # Client-equivalent: rebuild over the same (unfiltered) domain and # finalize the snapped slice through corr_matrix (== cube.js). @@ -2171,7 +2150,9 @@ def test_blob_reslice_matches_legacy_recompute(self, corr_client, corr_df): corr_df.lazy(), CubeSpec( source_name=_CORR_SRC, - free=FreeAxisSpec(column="a", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec( + column="a", p=header["free"]["p"], domain=(a_lo, a_hi) + ), target_dims=(), measure=MeasureSpec(agg="corr", columns=tuple(cols)), ), @@ -2271,8 +2252,9 @@ def test_passive_selection_bakes_into_corr_cube(self, corr_client, corr_df): ) # Free domain stays the UNFILTERED full ``a`` domain. a_lo, a_hi = corr_df["a"].min(), corr_df["a"].max() - assert header["free"]["domain"] == [a_lo, a_hi] - _lo_bin, _hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 18.0, 82.0) + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} + _lo_bin, _hi_bin, edge_lo, edge_hi = _snap(header["free"], 18.0, 82.0) + a_hi = header["free"]["domain"][1] passive_expr = pl.col("rr").is_between(0.0, 15.0) active_expr = pl.col("a").is_between(edge_lo, edge_hi, closed="left") @@ -2407,7 +2389,7 @@ def test_serves_hist2d_cube_count_header(self, hist2d_client, hist2d_df): assert header["target_dims"][0]["domain"] == [x_lo, x_hi] assert header["target_dims"][1]["domain"] == [y_lo, y_hi] a_lo, a_hi = hist2d_df["a"].min(), hist2d_df["a"].max() - assert header["free"] == {"kind": "continuous", "p": _P, "domain": [a_lo, a_hi]} + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} def test_serves_hist2d_cube_mean_header(self, hist2d_client, hist2d_df): spec = _hist2d_dashboard(hist2d_df, histfunc="mean", z="z") @@ -2457,8 +2439,7 @@ def test_blob_reslice_matches_legacy_recompute( body = _cube_body(resp) blob = base64.b64decode(body["cubes"][body["trace_cubes"][tgt_uid]]) header = decode_fvcube_header(blob) - a_lo, a_hi = header["free"]["domain"] - lo_bin, hi_bin, edge_lo, edge_hi = _snap((a_lo, a_hi), 21.4, 73.6) + lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 21.4, 73.6) free_bin = _read_u32_col(blob, header, "free_bin") bin_x = _read_u32_col(blob, header, "__bin__x") @@ -2628,7 +2609,7 @@ def test_serves_treemap_cube_header_well_formed(self, treemap_client, treemap_df assert header["measure"]["agg"] == "sum" assert header["measure"]["value_col"] == "val" a_lo, a_hi = treemap_df["a"].min(), treemap_df["a"].max() - assert header["free"] == {"kind": "continuous", "p": _P, "domain": [a_lo, a_hi]} + assert header["free"] == {"kind": "continuous", **_hist_free(a_lo, a_hi)} def test_second_identical_request_hits_cache( self, treemap_client, treemap_df, monkeypatch From ed6abe87aff4e89b9c19592477115719e395a819 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Tue, 29 Sep 2026 21:57:26 +0200 Subject: [PATCH 4/8] fix(cube): brush a histogram source on its own bars --- Architecture.md | 19 +- CHANGELOG.md | 15 ++ docs/cube-support-overview.md | 2 +- flexviz/adapters/js/plotly/events.js | 60 +++-- flexviz/adapters/js/runtime/cube.js | 62 ++--- flexviz/engine.py | 29 ++- flexviz/trace/bin_grid.py | 15 ++ flexviz/trace/hist.py | 19 +- tests/test_browser_cube.py | 361 ++++++++++++++++++++++++++- tests/test_cube_server.py | 204 ++++++++++++++- tests/test_perf_choices.py | 48 ++++ tests/test_trace_hist.py | 10 +- 12 files changed, 747 insertions(+), 97 deletions(-) diff --git a/Architecture.md b/Architecture.md index 25049ad2..ae3a8161 100644 --- a/Architecture.md +++ b/Architecture.md @@ -1079,8 +1079,11 @@ dataset size — and adds drag-time updates that flexviz previously did not have implemented. A cube is one target trace's grouping × the brushed (free) axis, holding decomposable partial -measures. A **range** free axis (hist / box / line source) is binned to a **fixed resolution -P = 2048** over the source figure's viewport domain. A **box2d** free axis (a 2-D box-select on a hist2d source) is two range axes binned at +measures. A **range** free axis on a histogram source is its bars (`P = bins` over the display +grid's domain, `bins + 1` when a zoom snaps to the bar lattice), so a snapped brush edge is a bar +edge, and the commit, the redrawn box, the cube slice and Plotly's highlight cover the same bars. +Box and line sources have no bins and keep a **fixed resolution P = 2048** over the viewport +domain. A **box2d** free axis (a 2-D box-select on a hist2d source) is two range axes binned at **P₂D = 128** each and packed into one composite `free_bin`. A **categorical** free axis (bar/pie/treemap source) is the exact tuple of label/path column values — no binning, no domain, dictionary-encoded in sorted order. A resolution fixed by the spec — rather than Mosaic's pixel @@ -1175,7 +1178,13 @@ range **or categorical (bar)** selection geometry: (`fvCubeSnap` / `snap_brush`) — Plotly's highlight rule for bars. A frame whose brush covers no center shows the targets' pre-gesture state (in overlay mode without the live fg presentation), and its commit clears the figure's selection. A box2d source does the same - per axis. The commit writes the **kernel's bin boundaries**, not + per axis. A 1-D grid (`p`, domain, unit) is adopted from the decoded header only: the server + resolves the unzoomed domain and a histogram snaps a zoomed axis to its bar lattice, so the + viewport alone does not give it (a commit before the first header lands stays unsnapped). Before a blob is stored, + `fvCubeHeaderMatchesKey` checks each range axis against the key: unzoomed, the header keeps the + key's `p`; zoomed, it has `p` or `p + 1` bins and (numeric axes) a domain that covers the + viewport. When a histogram and a non-binned trace share the brushed column in one figure, the + histogram is the source (`_fvCubeSourceConstraint`). The commit writes the **kernel's bin boundaries**, not `lo + k·step`: the first edge is `lo`, an inner edge the first double the kernel puts in bin `k`, found by bisecting the doubles around the float `lo + (k − 1e-9)·step` along the kernel's own rule (a bounded search: near a zero crossing billions of doubles share one value of `v − lo`), and a brush reaching the top bin ends at the domain top with `closed="both"`, otherwise @@ -1339,7 +1348,7 @@ them. | trace type | cube source? | cube target? | notes | |---|---:|---:|---| - | `histogram` | yes | yes | Source is a 1-D range axis at `P=2048`. Target is a binned count; grouped histograms add categorical group dims. | + | `histogram` | yes | yes | Source is a 1-D range axis on its own bar grid (`P = bins`, or `bins + 1` zoomed). Target is a binned count; grouped histograms add categorical group dims. | | `box` | yes | no | Source is a 1-D range over the box data axis. Box is not a target because quantiles are not decomposable. | | `line` | yes | yes, limited | Source is an x-only range at `P=2048`. Target requires `downsample="minmax"` and numeric `y`; the target is a live-only `line_env`, and commit always POSTs. | | `histogram2d` | yes | yes, limited | Source is a `box2d` free axis at `128 x 128`. Target is full-data-only 2-D binned count/reduce; zoomed target axes fall back to the normal POST path. | @@ -1355,7 +1364,7 @@ them. | trace | free axis | |---|---| - | hist | 1-D range, P=2048 over the viewport (or server-resolved full) domain; continuous + temporal (`us`/`ms`/`day` physical units — see “Temporal sources” below; `Datetime("ns")`/`Time` gate to no cube) | + | hist | 1-D range on the bar grid: `p = bins` and `domain=None` unzoomed (the engine resolves the full domain, unioned with the source's sibling histograms like its bars), the viewport snapped with `snap_range` zoomed (`bins` or `bins + 1`); continuous + temporal (`us`/`ms`/`day` physical units — see “Temporal sources” below; `Datetime("ns")`/`Time` gate to no cube) | | box | 1-D range over the `data_col` (same shape/gates as hist) | | line | 1-D range over the **x** column only (line selection is x-only — see below); P=2048; source geometry independent of `downsample` | | hist2d | **box2d**: two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` (`bin_y·P₂D + bin_x`); per-axis domains resolved by the engine; a rectangle brush slices a 2-D sub-grid | diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3c5c89..f673fd61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,21 @@ independently. `flexviz` pins a compatible `flexviz-polars` range. - `flexviz.server.show_server`. Use `Figure.show()`, `Dashboard.show()` or `flexviz serve`. +### Fixed + +- A brush on a histogram snaps to the edges of its bars. The bars that Plotly + highlights now hold exactly the rows that the other figures keep, also for a + value on a bar edge and on a `Date` axis. A brush that covers no bar center + clears the selection. Before, the brush snapped to a fixed grid of 2048 + steps, so the other figures also kept part of the rows of the two edge bars. + The live-brush data of a histogram source now grows with its number of bars + instead of the 2048 steps: with the default 20 bars it is about 50 times + smaller. +- A histogram or a histogram2d on a `Date` column draws each bar at its bin + center, and its hover shows the time of day of that center. Before, the + center was rounded to a whole day, so a bar could sit up to half a day off + its bin, and bars narrower than a day stacked on one day. + ## [0.1.0b5] - 2026-09-27 ### Added diff --git a/docs/cube-support-overview.md b/docs/cube-support-overview.md index 95d7aeb4..7ec5fe07 100644 --- a/docs/cube-support-overview.md +++ b/docs/cube-support-overview.md @@ -18,7 +18,7 @@ A source trace implements `get_cube_source_spec()` and produces a **free axis** | Source trace | Free-axis kind | Geometry | |---|---|---| -| `histogram` | **range** (continuous / temporal) | 1-D, P=2048 over the viewport (or full-data) domain | +| `histogram` | **range** (continuous / temporal) | 1-D, the histogram's own bars over the viewport (or full-data) domain | | `box` | **range** | 1-D over the `data_col` (same gates as hist) | | `line` | **range** | 1-D over the **x** column only (line selection is x-only); P=2048 | | `histogram2d` | **box2d** | two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` | diff --git a/flexviz/adapters/js/plotly/events.js b/flexviz/adapters/js/plotly/events.js index 8b560e12..118d4175 100644 --- a/flexviz/adapters/js/plotly/events.js +++ b/flexviz/adapters/js/plotly/events.js @@ -494,20 +494,26 @@ function _fvCubeTargetBgOk(target) { // trace (a hist2d) and no other pairs ⇒ // {kind:'box2d', x:{column,anchor}, y:{column,anchor}, uid}. // Any other count (two pairs on two different traces, three+ pairs, mixed) ⇒ -// null — today's "not a cube source" behavior. +// null — today's "not a cube source" behavior. When two traces share a +// (column, role) pair the later one wins, except that a binned trace +// (histogram, histogram2d) keeps the pair: its bins are what the user sees, +// so the brush snaps to them, not to a later line's fine grid. function _fvCubeSourceConstraint(figUid) { const figSpec = figSpecByUid[figUid]; if (!figSpec) return null; - const found = new Map(); // "col:role" -> {column, role, anchor, uid} + const found = new Map(); // "col:role" -> {column, role, anchor, uid, binned} for (const ts of (figSpec.traces || [])) { const sel = ts.selection; if (!sel || sel.kind !== 'range') continue; const axes = ts.axes || []; + const binned = ts.trace_type === 'histogram' || ts.trace_type === 'histogram2d'; for (const [anchor, col] of Object.entries(sel.axis_columns || {})) { if (typeof col !== 'string') continue; const role = anchor === axes[0] ? 'x' : (anchor === axes[1] ? 'y' : null); if (!role) continue; - found.set(col + ':' + role, { column: col, role, anchor, uid: ts.uid }); + const pairKey = col + ':' + role; + if (!binned && found.has(pairKey) && found.get(pairKey).binned) continue; + found.set(pairKey, { column: col, role, anchor, uid: ts.uid, binned }); } } if (found.size === 1) return found.values().next().value; @@ -820,26 +826,39 @@ function _fvCubeGestureStart(figUid, eventData) { } // Adopt a decoded range free header (or a remembered free record) as the -// gesture's snap grid: the physical domain and the temporal unit (contract -// G). Idempotent — first adoption wins; numeric zoomed gestures already hold -// their viewport snapDomain. +// gesture's snap grid: the bin count, physical domain, and temporal unit. +// The header is the only grid source: the server pads the domain and snaps a +// histogram's zoomed viewport to its bar lattice, which the viewport alone +// does not give. Idempotent — first adoption wins. function _fvCubeAdoptFreeGrid(gesture, free) { if (!free || free.kind === 'categorical' || !Array.isArray(free.domain)) return; - if (!gesture.snapDomain) gesture.snapDomain = free.domain.slice(); - if (free.unit && !gesture.unit) gesture.unit = free.unit; + if (gesture.snapDomain) return; + gesture.snapDomain = free.domain.slice(); + gesture.p = free.p; + gesture.unit = free.unit || null; } -// Snap a (physical) range on the gesture's P-grid. +// Snap a (physical) range on the gesture's grid. function _fvCubeGestureSnap(gesture, a, b) { - return fvCubeSnap(gesture.snapDomain, _FV_CUBE_P, a, b); + return fvCubeSnap(gesture.snapDomain, gesture.p, a, b); +} + +// The free-axis bin count a 1-D range source asks for: a histogram brushes its +// own bars, box and line sources a fixed fine grid. A zoom can add one bin, +// because the server snaps the viewport outward to a lattice of width +// (raw span / n). +function _fvCubeSourceP(traceUid) { + const ts = traceSpecByUid[traceUid]; + if (ts && ts.trace_type === 'histogram') return ts.params.bins; + return _FV_CUBE_P; } function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { const sourceDomain = _fvCubeViewportDomain(figUid, constraint.anchor); if (sourceDomain === undefined) return { inert: true }; - const temporalViewport = _fvCubeViewportIsTemporal(figUid, constraint.anchor); + const sourceP = _fvCubeSourceP(constraint.uid); const freeDesc = { - c: constraint.column, k: 'continuous', p: _FV_CUBE_P, d: sourceDomain, + c: constraint.column, k: 'continuous', p: sourceP, d: sourceDomain, }; const passiveKey = fvCubePassiveKey(DASHBOARD_SPEC.state.selections, figUid); const gesture = { @@ -850,11 +869,9 @@ function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { role: constraint.role, sourceName: figSpec.source, sourceZoomed: sourceDomain != null, - // Snapping needs a resolved PHYSICAL domain: the zoomed numeric - // viewport, or (temporal/unzoomed) the server-resolved domain learned - // from a decoded cube header — a temporal viewport's ms token is a key, - // never a snap domain (the column's physical unit may differ). - snapDomain: (!temporalViewport && sourceDomain) || null, + // The snap grid, adopted from the decoded header. + snapDomain: null, + p: null, unit: null, targets: _fvCubeEnumerateTargets(figUid, figSpec.source, freeDesc, passiveKey), live: false, @@ -865,7 +882,7 @@ function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { }; if (!gesture.sourceZoomed) { _fvCubeAdoptFreeGrid( - gesture, fvCubeFreeDomain(figSpec.source, constraint.column, _FV_CUBE_P) + gesture, fvCubeFreeDomain(figSpec.source, constraint.column, sourceP) ); } const capable = gesture.targets.filter(t => t.capable); @@ -1064,7 +1081,7 @@ async function _fvCubeFetchForGesture(gesture) { } if (!gesture.sourceZoomed) { fvCubeRememberFreeDomain( - gesture.sourceName, gesture.column, _FV_CUBE_P, entry.header.free + gesture.sourceName, gesture.column, entry.header.free ); } _fvCubeAdoptFreeGrid(gesture, entry.header.free); @@ -1760,9 +1777,12 @@ function _fvCubeEchoOfStoredSelection(figUid, range) { const box2dGrid = isBox2d && figSpec && figSpec.source ? _fvCubeRememberedBox2dGrid(figSpec.source, [constraint.x.column, constraint.y.column]) : null; + const sourceP = constraint ? _fvCubeSourceP(constraint.uid) : null; for (let ci = 0; ci < axisConstraints.length; ci++) { const ac = axisConstraints[ci]; - const p = isBox2d ? _FV_CUBE_BOX2D_P : _FV_CUBE_P; + // The nominal bin count: a zoom's extra lattice bin changes the step by + // at most 1/n, far below the half-bin tolerance. + const p = isBox2d ? _FV_CUBE_BOX2D_P : sourceP; // 1-D sources remember a per-(source, column, p) free record; box2d axes // read their {domain, unit} from the remembered box2d header by index. let recDomain = null; diff --git a/flexviz/adapters/js/runtime/cube.js b/flexviz/adapters/js/runtime/cube.js index 793c1cd4..f22f32e8 100644 --- a/flexviz/adapters/js/runtime/cube.js +++ b/flexviz/adapters/js/runtime/cube.js @@ -3,7 +3,9 @@ // // Cube live-brush path (see Architecture.md, "Cube Pre-Aggregation & Live Brushing"): // a cube is a pre-aggregation of one target trace's dims × the active brush -// column binned to P=2048 (or its category tuples for a categorical source). +// column binned to P bins (a histogram source's own bars, 128 per axis for a +// histogram2d, 2048 for box and line; or its category tuples for a +// categorical source). // The server builds + encodes it once (FVCube v1); this module decodes the // blob, stores it under a client-local canonical descriptor key, and // re-slices it per drag frame — turning every mousemove of a live brush into @@ -201,11 +203,11 @@ function fvCubeTraceFilteredOnly(ts) { } // Structural guard before every store put (plan step 0b): the decoded blob's -// free header must match the key's free descriptor — kind, column tuple, -// p, and domain when the client knows it (`d` null is a token for a -// server-resolved domain, not a constraint). A mismatched blob means the -// server resolved a different source trace (or the protocol drifted): the -// entry must NOT be stored — callers demote the target for the gesture. +// free header must match the key's free descriptor — kind, column tuple, and +// per range axis the grid rule of _fvCubeAxisMatchesKey. A mismatched blob +// means the server resolved a different source trace (or the protocol +// drifted): the entry must NOT be stored — callers demote the target for the +// gesture. function fvCubeHeaderMatchesKey(key, header) { let keyFree; try { keyFree = (JSON.parse(key) || {}).free; } catch (e) { return false; } @@ -224,35 +226,38 @@ function fvCubeHeaderMatchesKey(key, header) { && keyCols.every((c, i) => c === cols[i]); } if (keyFree.k === 'box2d') { - // The (x, y) column tuple and p must match; per-axis domains are checked - // only on axes the client knows (a null token is "server-resolved"). const keyCols = Array.isArray(keyFree.c) ? keyFree.c : [keyFree.c]; const cols = free.cols || []; if (keyCols.length !== 2 || cols.length !== 2) return false; if (!keyCols.every((c, i) => c === cols[i])) return false; - if (keyFree.p !== free.p) return false; - const doms = free.domains || []; - const units = free.units || [null, null]; const keyD = Array.isArray(keyFree.d) ? keyFree.d : [null, null]; + const units = free.units || [null, null]; + const doms = free.domains || []; for (let a = 0; a < 2; a++) { - // Temporal axis domains are epoch-ms tokens vs the header's physical - // unit — only comparable for unit:"ms"; skip the check otherwise. - if (keyD[a] != null && !units[a]) { - const d = doms[a] || []; - if (keyD[a][0] !== d[0] || keyD[a][1] !== d[1]) return false; + if (!_fvCubeAxisMatchesKey(keyFree.p, keyD[a], free.p, doms[a], !!units[a])) { + return false; } } return true; } - if (keyFree.p !== free.p) return false; - // Temporal key domains are epoch-ms tokens while the header domain is in - // the column's physical unit — only comparable for unit:"ms", so the - // domain check is skipped for temporal headers. - if (keyFree.d != null && free.kind !== 'temporal') { - const dom = free.domain || []; - if (keyFree.d[0] !== dom[0] || keyFree.d[1] !== dom[1]) return false; - } - return true; + return _fvCubeAxisMatchesKey( + keyFree.p, keyFree.d, free.p, free.domain, free.kind === 'temporal' + ); +} + +// One range axis of a header against its key slot. The key holds the source +// trace's nominal bin count and, when zoomed, the viewport. Unzoomed, the +// header keeps that count. Zoomed, the server pads the viewport and may snap +// it outward to a bin lattice, which adds at most one bin and covers the +// viewport. A temporal viewport is an epoch-ms token, not the header's +// physical unit, so only the count is compared there. +function _fvCubeAxisMatchesKey(keyP, keyD, p, domain, temporal) { + if (keyD == null) return p === keyP; + if (p !== keyP && p !== keyP + 1) return false; + if (temporal) return true; + if (!Array.isArray(domain)) return false; + const slack = (domain[1] - domain[0]) / p * 1e-6; // the lattice's own epsilon + return domain[0] <= keyD[0] + slack && keyD[1] <= domain[1] + slack; } // --------------------------------------------------------------------------- @@ -317,13 +322,14 @@ fvCacheReset = function () { // Server-resolved full-data free blocks seen in decoded headers, keyed by // (source, column, p). Lets a later gesture on the same unzoomed axis snap // even before its own cube response lands. Each record carries the -// {domain, unit} subset of the header free block. +// {domain, p, unit} subset of the header free block. const _fvCubeFreeDomains = new Map(); -function fvCubeRememberFreeDomain(sourceName, column, p, free) { +function fvCubeRememberFreeDomain(sourceName, column, free) { if (!free || !Array.isArray(free.domain) || free.domain.length !== 2) return; - _fvCubeFreeDomains.set(JSON.stringify([sourceName, column, p]), { + _fvCubeFreeDomains.set(JSON.stringify([sourceName, column, free.p]), { domain: [free.domain[0], free.domain[1]], + p: free.p, unit: free.unit || null, }); } diff --git a/flexviz/engine.py b/flexviz/engine.py index 74e3c5a5..0931208c 100644 --- a/flexviz/engine.py +++ b/flexviz/engine.py @@ -449,6 +449,17 @@ def _axis_range(axis: str | None) -> tuple[float, float] | None: # Only groups containing an actual target need resolving: a figure of # histograms none of which is cube-servable would otherwise pull its # columns through ``physical_minmax`` for nothing. + # The source figure groups its histograms the same way. When unzoomed, + # a group shares one axis viewport, so no sibling is zoomed either. + source_cols = self._histogram_domain_cols_by_uid( + ( + (ti, self._scalable_traces[ti.uid], False) + for ti in trace_infos + if ti.figure_uid == active_source.figure_uid + and ti.uid in self._scalable_traces + ), + schema=schema, + ).get(active_source.trace_uid, ()) free, domains = self._resolve_cube_domains( free_spec, [t for _, _, t in targets], @@ -458,6 +469,7 @@ def _axis_range(axis: str | None) -> tuple[float, float] | None: for ti, _, _ in targets for col in shared_domain_cols.get(ti.uid, ()) }, + source_cols=source_cols, ) if free is None: return [], {} @@ -662,14 +674,18 @@ def _resolve_cube_domains( target_specs: list[CubeTargetSpec], schema: pl.Schema | None, extra_cols: set[str] | None = None, + source_cols: tuple[str, ...] = (), ) -> tuple[FreeAxisSpec | None, dict[str, tuple[float, float]]]: """Resolve ``domain=None`` (= full data domain) to concrete floats. + ``source_cols`` is the sibling group of a histogram source. Its bars + span the union of the group's domains (``_histogram_bounds_exprs``), so + an unzoomed free axis spans that union too. + One batched min/max ``select`` over the **unfiltered** LazyFrame covers the free axis and every unresolved binned target column (the same unfiltered-domain rule the aggregation path uses; the builder itself is - never mutated). The free axis gets the min/max verbatim — the binned-dim - epsilon is applied later, uniformly, in ``_resolved_target_dims``. + never mutated). A **categorical** free axis (bar/pie/treemap source) is not binned and takes no domain: free-domain resolution is skipped entirely (``domain`` @@ -716,17 +732,20 @@ def _resolve_cube_domains( # free axis, box2d axes, and binned target dims, so a single per-column # (min, max) serves all three roles. needed: list[str] = [] + free_cols = [free_spec.column, *source_cols] if resolve_free: - needed.append(free_spec.column) + needed += free_cols needed += [col for _, col in box2d_axes] needed += unresolved_cols minmax = self._backend_lf.physical_minmax(needed, schema) if needed else {} free = free_spec if resolve_free: - lo, hi = minmax[free_spec.column] - if lo is None or hi is None: # empty / all-null free column + if None in minmax[free_spec.column]: # empty / all-null free column return None, {} + # Like the display, an all-null sibling does not widen the union. + bounds = [minmax[c] for c in free_cols if None not in minmax[c]] + lo, hi = min(b[0] for b in bounds), max(b[1] for b in bounds) free = replace(free, domain=(float(lo), float(hi))) if is_box2d: diff --git a/flexviz/trace/bin_grid.py b/flexviz/trace/bin_grid.py index 19a7f55f..8451b26b 100644 --- a/flexviz/trace/bin_grid.py +++ b/flexviz/trace/bin_grid.py @@ -43,6 +43,21 @@ def snap_range(lo: float, hi: float, n: int) -> tuple[float, float, int]: return k0 * width, k1 * width, k1 - k0 +def snapped_domain( + range_: tuple[float, float] | None, n: int +) -> tuple[tuple[float, float] | None, int]: + """A cube axis on a binned trace's display grid: its domain and bin count. + + Unzoomed (``None``), the domain stays ``None`` for the engine to resolve to + the full data domain. Zoomed, it is the physical viewport snapped with + ``snap_range``, so the cube bins land on the displayed bins. + """ + if range_ is None: + return None, n + lo, hi, n = snap_range(float(range_[0]), float(range_[1]), n) + return (lo, hi), n + + def snapped_axis( col: str, range_: tuple, n: int, schema: pl.Schema | None ) -> tuple[float, float, int, pl.Expr]: diff --git a/flexviz/trace/hist.py b/flexviz/trace/hist.py index 3231bd0f..4686cf4c 100644 --- a/flexviz/trace/hist.py +++ b/flexviz/trace/hist.py @@ -48,7 +48,7 @@ _to_col_tuple, ) from .batch_fold import hist1d_fold_plan -from .bin_grid import snap_range, snapped_axis +from .bin_grid import snapped_axis, snapped_domain # For 1-D histograms "histnorm" describes what the count-axis displays, so # "count" (raw bin counts) is a meaningful, natural value — not a no-op. @@ -252,12 +252,16 @@ def get_cube_source_spec( temporal dtype (Date/Datetime/Time), else ``"continuous"`` — including when no schema is available. Grouped histograms are still valid sources: the brush is on the shared data axis, independent of the - grouping. ``domain`` is the viewport range verbatim (``None`` = - unzoomed; the engine resolves it to the full data domain). + grouping. The free axis is the histogram's own bar grid, so a brush + over bars k..m commits their edges: ``p = bins`` and ``domain=None`` + unzoomed (the engine resolves the full data domain); zoomed, the + viewport snapped to the display lattice (``snapped_domain``, as for + the target). """ dtype = _dtype_for_col(schema, self.data_col) kind = "temporal" if dtype is not None and dtype.is_temporal() else "continuous" - return FreeAxisSpec(column=self.data_col, kind=kind, p=2048, domain=axis_range) + domain, bins = snapped_domain(axis_range, self.bins) + return FreeAxisSpec(column=self.data_col, kind=kind, p=bins, domain=domain) def get_cube_target_spec( self, @@ -284,12 +288,7 @@ def get_cube_target_spec( group_cols = self.group_by_cols or () if group_cols and not _categorical_dims_ok(schema, group_cols): return None - bins, domain = self.bins, None - if axis_range is not None: - lo, hi, bins = snap_range( - float(axis_range[0]), float(axis_range[1]), self.bins - ) - domain = (lo, hi) + domain, bins = snapped_domain(axis_range, self.bins) return CubeTargetSpec( target_dims=( TargetDimSpec( diff --git a/tests/test_browser_cube.py b/tests/test_browser_cube.py index 147e6f95..865958b2 100644 --- a/tests/test_browser_cube.py +++ b/tests/test_browser_cube.py @@ -101,6 +101,7 @@ pytestmark = pytest.mark.browser +# Box and line sources brush a fixed fine grid; a histogram source its bars. _P = 2048 _SRC_BINS = 16 _TGT_BINS = 12 @@ -491,10 +492,10 @@ def _committed_edges(page: Page, column: str) -> tuple[float, float]: def _free_grid( - df: pl.DataFrame, col: str = "a", bins: int = _P + df: pl.DataFrame, col: str = "a", bins: int = _SRC_BINS ) -> tuple[float, float, int]: - """An unzoomed source's free axis ``(lo, hi, p)``: *bins* over the full - data domain.""" + """An unzoomed source's free axis ``(lo, hi, p)``: *bins* over the + full data domain (a histogram's bars, or ``_P`` for box and line).""" lo, hi = float(df[col].min()), float(df[col].max()) return lo, hi, bins @@ -526,7 +527,7 @@ def _reference_slice_counts( edge_lo: float, edge_hi: float, free_col: str = "a", - bins: int = _P, + bins: int = _SRC_BINS, ) -> list[float]: """Python reference for the target hist: ``flexviz.cube`` build + slice over the snapped bins recovered from committed ``closed="left"`` edges @@ -2846,7 +2847,7 @@ def test_header_mismatch_demotes_instead_of_storing( self, page: Page, server_port: int ): """A cube_request answered with a blob whose free header does not - match the gesture's descriptor (here p=1024 instead of 2048) must be + match the gesture's descriptor (here p=1024 instead of 16 bars) must be refused: nothing stored, no live updates, and the now-mixed commit POSTs one legacy selection that self-heals the target.""" df = _cube_df() @@ -4224,9 +4225,9 @@ class TestZoomKeyInterplayBrowser: def test_zoomed_source_brush_snaps_to_viewport_grid( self, page: Page, server_port: int ): - """(a): a brush on a pre-zoomed source snaps to the VIEWPORT grid - (finer bins): one cube_request, live updates, local commit, edges on - the zoomed P-grid, full parity.""" + """(a): a brush on a pre-zoomed source snaps to the viewport's bar + lattice: one cube_request, live updates, local commit, edges on the + zoomed bar grid, full parity.""" df = _cube_df() zoom = (10.0, 80.0) url = _zoomed_dashboard_url(server_port, "_cube_browser_zsrc", src_zoom=zoom) @@ -4241,8 +4242,9 @@ def test_zoomed_source_brush_snaps_to_viewport_grid( assert types == ["cube_request"], types edge_lo, edge_hi = _committed_edges(page, "a") - # Edges are bin boundaries of the zoomed P-grid. - _committed_bins(edge_lo, edge_hi, (zoom[0], zoom[1], _P)) + # Edges are bar boundaries of the viewport snapped to the bar lattice. + lo, hi, n = snap_range(zoom[0], zoom[1], _SRC_BINS) + _committed_bins(edge_lo, edge_hi, (lo, hi, n)) sels = page.evaluate("DASHBOARD_SPEC.state.selections") expr = _selection_expr(df, sels[0]) @@ -4344,8 +4346,9 @@ def test_zoomed_source_with_foreign_selection(self, page: Page, server_port: int clause = by_fig[fig_uids[0]]["predicates"][0]["clauses"][0] assert clause["column"] == "a" and clause.get("closed") == "left" edge_lo, edge_hi = clause["range"] - # The commit stays inside the zoomed grid, never past its top. - assert 10.0 <= edge_lo < edge_hi <= 80.0 + # The commit stays inside the snapped bar grid, never past its top. + lo, hi, _n = snap_range(10.0, 80.0, _SRC_BINS) + assert lo <= edge_lo < edge_hi <= hi expected = _hist_counts_ref(df, expr_src & expr_third, "b", _TGT_BINS) assert 0 < sum(expected) < df.height _wait_for_hist_equals(page, "#fv-plot-1", expected) @@ -6527,12 +6530,223 @@ def test_mean_agg_treemap_live_and_commit_parity( assert got == pytest.approx(want, rel=1e-9, abs=1e-9), (got, want) +# --------------------------------------------------------------------------- +# A histogram source brushes its own bars (#133) +# --------------------------------------------------------------------------- + + +def _edge_df() -> pl.DataFrame: + """Integer ``a`` in 0..100: with 16 bars, 25, 50 and 75 sit on bar + edges.""" + n = 101 * 7 + return pl.DataFrame( + { + "a": [float(i % 101) for i in range(n)], + "b": [float(i % 13) for i in range(n)], + } + ) + + +def _bar_fracs(page: Page, lo: float, step: float, k: int, m: int): + """Plot-width fractions of a brush over the centers of bars k..m only. + *lo* and *step* are in the axis's linear units (epoch ms on a date axis).""" + r0, r1 = page.eval_on_selector("#fv-plot-0", "gd => gd._fullLayout.xaxis._rl") + a, b = lo + (k + 0.25) * step, lo + (m + 0.75) * step + return (a - r0) / (r1 - r0), (b - r0) / (r1 - r0) + + +class TestHistSourceBarGrid: + def test_brush_over_bars_commits_their_rows(self, page: Page, server_port: int): + """A brush over bars k..m commits exactly their kernel boundaries; the + bars Plotly highlights are the bars whose rows the commit filters, + including the integer values that sit on a bar edge.""" + df = _edge_df() + url = _two_hist_dashboard_url(server_port, "_cube_bar_grid", "auto", df=df) + page.goto(url) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + source_counts = _hist_y(page, "#fv-plot-0") + + grid = _free_grid(df) + lo, hi, bars = grid + step = (hi - lo) / bars + k, m = 4, 11 # bars 4 and 8 start on the edge values 25 and 50 + f_lo, f_hi = _bar_fracs(page, lo, step, k, m) + _brush_commit(page, 0, f_lo, f_hi, live_fig_idx=1) + + edge_lo, edge_hi = _committed_edges(page, "a") + assert _committed_bins(edge_lo, edge_hi, grid) == (k, m) + highlighted = page.eval_on_selector( + "#fv-plot-0", + "gd => gd.data.map(t => t.selectedpoints ? Array.from(t.selectedpoints) : null)" + ".filter(Boolean)", + ) + assert list(range(k, m + 1)) in highlighted, highlighted + + sels = page.evaluate("DASHBOARD_SPEC.state.selections") + expr = _selection_expr(df, sels[0]) + assert df.filter(expr).height == sum(source_counts[k : m + 1]) + expected = _hist_counts_ref(df, expr, "b", _TGT_BINS) + _wait_for_hist_equals(page, "#fv-plot-1", expected) + + def test_brush_covering_no_bar_center_clears(self, page: Page, server_port: int): + """A brush inside one bar that misses its center selects nothing: the + commit clears the figure's selection and the target shows all rows.""" + df = _edge_df() + url = _two_hist_dashboard_url(server_port, "_cube_bar_clear", "auto", df=df) + page.goto(url) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + y_unfiltered = _target_y(page) + + lo, hi, bars = _free_grid(df) + step = (hi - lo) / bars + f_lo, f_hi = _bar_fracs(page, lo, step, 2, 9) + _brush_commit(page, 0, f_lo, f_hi, live_fig_idx=1) + assert len(page.evaluate("DASHBOARD_SPEC.state.selections")) == 1 + + r0, r1 = page.eval_on_selector("#fv-plot-0", "gd => gd._fullLayout.xaxis.range") + # Outside the committed box: a drag inside it would move that box. + a, b = lo + 13.1 * step, lo + 13.4 * step + _brush_commit(page, 0, (a - r0) / (r1 - r0), (b - r0) / (r1 - r0)) + assert page.evaluate("DASHBOARD_SPEC.state.selections") == [] + _wait_for_hist_equals(page, "#fv-plot-1", y_unfiltered) + + # --------------------------------------------------------------------------- # Review follow-ups: empty frames, header guard, source choice, snap parity # --------------------------------------------------------------------------- +def _x_px(page: Page, fig_idx: int, value: float) -> float: + """Page x of a data value on figure *fig_idx*'s x axis.""" + box = page.locator(f"#fv-plot-{fig_idx} .nsewdrag").bounding_box() + r0, r1 = page.eval_on_selector( + f"#fv-plot-{fig_idx}", "gd => gd._fullLayout.xaxis.range" + ) + return box["x"] + box["width"] * (value - r0) / (r1 - r0) + + +def _wait_for_layers(page: Page, sel: str, layers: list[dict], equal: bool) -> None: + """Wait until the rendered layers (``_rendered_layers``) of *sel* equal + *layers*, or differ from them when *equal* is False.""" + page.wait_for_function( + """([sel, want, equal]) => { + const gd = document.querySelector(sel); + const got = (gd.data || []).map(t => ({ + uid: t.uid, opacity: t.opacity, y: Array.from(t.y || []), + })); + return (JSON.stringify(got) === JSON.stringify(want)) === equal; + }""", + arg=[sel, layers, equal], + timeout=10_000, + ) + + class TestCubeGridGuards: + @pytest.mark.parametrize("overlay", [False, True]) + def test_empty_snap_mid_drag_shows_pre_gesture_state( + self, page: Page, server_port: int, overlay: bool + ): + """A frame whose brush covers no bar center shows the targets as they + were before the gesture, not an empty slice. In overlay mode that is + the unfiltered layer at full opacity, not the dimmed ghost.""" + df = _cube_df() + if overlay: + url = _overlay_dashboard_url(server_port, "_cube_empty_frame_ovl") + else: + url = _two_hist_dashboard_url(server_port, "_cube_empty_frame", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + before = _rendered_layers(page, "#fv-plot-1") + lo, hi, bars = _free_grid(df) + step = (hi - lo) / bars + box = page.locator("#fv-plot-0 .nsewdrag").bounding_box() + y = box["y"] + box["height"] * 0.5 + + page.mouse.move(_x_px(page, 0, lo + 3.1 * step), y) + page.mouse.down() + page.mouse.move(_x_px(page, 0, lo + 8.5 * step), y, steps=8) + _wait_for_layers(page, "#fv-plot-1", before, equal=False) + # Back inside bar 3, short of its center: no bar is covered. + page.mouse.move(_x_px(page, 0, lo + 3.4 * step), y, steps=8) + _wait_for_layers(page, "#fv-plot-1", before, equal=True) + page.mouse.up() + page.wait_for_timeout(500) + assert page.evaluate("DASHBOARD_SPEC.state.selections") == [] + + def test_header_guard_rejects_another_traces_grid( + self, page: Page, server_port: int + ): + """A zoomed key accepts only a header with the key's bin count or one + more, over a domain that covers the viewport; unzoomed, the count must + match. A blob built for another trace's grid is rejected.""" + url = _two_hist_dashboard_url(server_port, "_cube_hdr_guard", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + got = page.evaluate("""() => { + const key = (free) => JSON.stringify({s: 's', free}); + const one = (p, d, hp, dom, kind = 'continuous') => fvCubeHeaderMatchesKey( + key({c: 'a', k: 'continuous', p, d}), {free: {kind, p: hp, domain: dom}} + ); + return { + zoomedLattice: one(16, [10, 80], 17, [8.75, 83.125]), + zoomedOtherTrace: one(16, [10, 80], 2048, [10, 80.0000000001]), + zoomedTwoMore: one(16, [10, 80], 18, [8.75, 83.125]), + zoomedNotCovering: one(16, [10, 80], 17, [12, 83]), + zoomedTemporal: one(16, [1, 2], 17, [5, 6], 'temporal'), + unzoomed: one(16, null, 16, [0, 99.9]), + unzoomedMore: one(16, null, 17, [0, 99.9]), + }; + }""") + assert got == { + "zoomedLattice": True, + "zoomedOtherTrace": False, + "zoomedTwoMore": False, + "zoomedNotCovering": False, + "zoomedTemporal": True, + "unzoomed": True, + "unzoomedMore": False, + } + + @pytest.mark.parametrize("line_first", [False, True]) + def test_histogram_is_the_source_over_a_line_on_its_column( + self, page: Page, server_port: int, line_first: bool + ): + """A histogram and a line brushed on the same column: the histogram's + bars set the grid, whichever trace comes later.""" + from flexviz.dashboard import Dashboard + from flexviz.server import register_source + from flexviz.spec import LayoutSpec, encode_spec + + df = _cube_df().sort("a") + register_source("_cube_hist_over_line", df, cache=True) + dash = Dashboard(df) + fig = dash.add_figure(title="Source") + if line_first: + fig.add_line(x="a", y="b").add_histogram(x="a", bins=_SRC_BINS) + else: + fig.add_histogram(x="a", bins=_SRC_BINS).add_line(x="a", y="b") + dash.add_figure(title="Target").add_histogram(x="b", bins=_TGT_BINS) + spec = dash.to_spec( + source_name="_cube_hist_over_line", layout=LayoutSpec(draggable=False) + ) + page.goto( + f"http://127.0.0.1:{server_port}/view?spec={encode_spec(spec)}" + "&renderer=plotly" + ) + _wait_for_init(page, "plotly") + hist_uid = next( + t.uid for t in spec.figures[0].traces if t.trace_type == "histogram" + ) + got = page.evaluate( + "(fig) => { const c = _fvCubeSourceConstraint(fig); " + "return [c.uid, _fvCubeSourceP(c.uid)]; }", + spec.figures[0].uid, + ) + assert got == [hist_uid, _SRC_BINS] + def test_js_snap_matches_python_snap_brush(self, page: Page, server_port: int): """fvCubeSnap and flexviz.cube.snap_brush agree bit for bit.""" url = _two_hist_dashboard_url(server_port, "_cube_snap_parity", "auto") @@ -6660,3 +6874,126 @@ def test_constant_column_brush_selects_every_row( assert clause["range"] == [1e7, 1e7] assert clause["closed"] == "both" assert sum(_target_y(page)) == 10 + + +def _short_date_df() -> pl.DataFrame: + """Seven days in 16 bars: bars 0.375 day wide, edges between the days.""" + import datetime as dt + + n = 7 * 40 + return pl.DataFrame( + { + "t": [dt.date(2020, 1, 1) + dt.timedelta(days=i % 7) for i in range(n)], + "b": [float(i % 13) for i in range(n)], + } + ) + + +def _short_date_page(page: Page, server_port: int): + """Open a Date source hist(t) over ``_short_date_df`` + target hist(b).""" + from flexviz.dashboard import Dashboard + from flexviz.server import register_source + from flexviz.spec import LayoutSpec, encode_spec + + df = _short_date_df() + register_source("_cube_short_date", df, cache=True) + dash = Dashboard(df) + dash.add_figure(title="Days").add_histogram(x="t", bins=_SRC_BINS) + dash.add_figure(title="Target").add_histogram(x="b", bins=_TGT_BINS) + spec = dash.to_spec( + source_name="_cube_short_date", layout=LayoutSpec(draggable=False) + ) + spec.client_state.live_brush = "auto" + page.goto( + f"http://127.0.0.1:{server_port}/view?spec={encode_spec(spec)}&renderer=plotly" + ) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + return df, spec + + +class TestDateHistSource: + # Bar 2 holds day 1 in its upper half and bar 11 holds no day, so a + # whole-day box would drop bar 2's center and take bar 11's. + @pytest.mark.parametrize("k,m", [(2, 10), (2, _SRC_BINS - 1)]) + def test_box_on_bar_edges_highlights_the_committed_bars( + self, page: Page, server_port: int, k: int, m: int + ): + """On bars narrower than a day, a brush over bars k..m commits whole + days that hold exactly their rows, and its box sits on the bar edges, + so Plotly highlights bars k..m. A brush up to the top bar commits + closed="both" with the last day. The echo guard tolerates half a bar.""" + df, spec = _short_date_page(page, server_port) + counts = _hist_y(page, "#fv-plot-0") + day_ms = 86_400_000 + lo, hi, bars = _free_grid(df.with_columns(pl.col("t").to_physical()), "t") + step = (hi - lo) / bars + _brush_commit( + page, 0, *_bar_fracs(page, lo * day_ms, step * day_ms, k, m), live_fig_idx=1 + ) + + sels = page.evaluate("DASHBOARD_SPEC.state.selections") + clause = sels[0]["predicates"][0]["clauses"][0] + top = m == bars - 1 + assert clause["closed"] == ("both" if top else "left") + if top: + assert clause["range"][1] == "2020-01-07" + assert df.filter(_selection_expr(df, sels[0])).height == sum(counts[k : m + 1]) + (highlighted,) = page.eval_on_selector( + "#fv-plot-0", + "gd => gd.data.map(t => t.selectedpoints ? Array.from(t.selectedpoints) : null)" + ".filter(Boolean)", + ) + # A bar without rows has no height, so only the others show a highlight. + assert [i for i in highlighted if counts[i]] == [ + i for i in range(k, m + 1) if counts[i] + ] + + echo = page.evaluate( + """([fig, step]) => { + const sel = fvFigureSelection(fig, DASHBOARD_SPEC.state.selections); + const box = sel._plotly_selection_box; + const d = v => fvTemporalToPhysical(v, 'day'); + const at = shift => _fvCubeEchoOfStoredSelection( + fig, {x: [d(box.x0) + shift * step, d(box.x1)], y: [0, 1]} + ); + return [at(0.4), at(0.6)]; + }""", + [spec.figures[0].uid, step], + ) + assert echo == [True, False] + + # In bar units, bar 8 (day 3) is centered at 8.5, and the midnight that + # starts day 3 sits at 8.0, where day-rounded centers drew bars 7 and 8. + # The first brush covers bar 8's center but not that midnight, the second + # the midnight but no center. Each margin is at least 0.2 bar. + @pytest.mark.parametrize("a,b,bar_rows", [(8.2, 9.2, True), (7.7, 8.3, False)]) + def test_brush_highlights_the_rows_it_commits( + self, page: Page, server_port: int, a: float, b: float, bar_rows: bool + ): + """Plotly highlights a bar whose drawn center is inside the brush. The + bar must be drawn at its bin center, not the whole day, or the brush + highlights one bar and commits another.""" + df, _spec = _short_date_page(page, server_port) + counts = _hist_y(page, "#fv-plot-0") + lo, hi, bars = _free_grid(df.with_columns(pl.col("t").to_physical()), "t") + day_ms = 86_400_000 + step = (hi - lo) / bars * day_ms + r0, r1 = page.eval_on_selector("#fv-plot-0", "gd => gd._fullLayout.xaxis._rl") + f_lo, f_hi = ((lo * day_ms + f * step - r0) / (r1 - r0) for f in (a, b)) + x1, x2, y = _fig_drag_coords(page, 0, f_lo, f_hi) + page.mouse.move(x1, y) + page.mouse.down() + page.mouse.move(x2, y, steps=8) + page.wait_for_timeout(500) + highlighted = page.eval_on_selector( + "#fv-plot-0", "gd => Array.from(gd.data[0].selectedpoints || [])" + ) + page.mouse.up() + page.wait_for_timeout(500) + + rows = counts[8] if bar_rows else 0 + assert sum(counts[i] for i in highlighted) == rows + sels = page.evaluate("DASHBOARD_SPEC.state.selections") + committed = df.filter(_selection_expr(df, sels[0])).height if sels else 0 + assert committed == rows diff --git a/tests/test_cube_server.py b/tests/test_cube_server.py index d4c8401b..a5b45cc5 100644 --- a/tests/test_cube_server.py +++ b/tests/test_cube_server.py @@ -30,6 +30,8 @@ _SRC = "_cube_src" _CAT_SRC = "_cube_cat_src" _P = 2048 +# Bars of every histogram source below: the free axis is that bar grid. +_SRC_BINS = 16 def _cube_body(resp) -> dict: @@ -224,9 +226,9 @@ def _snap(free: dict, a: float, b: float): def _hist_free(lo: float, hi: float) -> dict: - """An unzoomed histogram source's free block: the fixed P grid over the + """An unzoomed histogram source's free block: its bars over the full data domain.""" - return {"domain": [lo, hi], "p": _P} + return {"domain": [lo, hi], "p": _SRC_BINS} # --------------------------------------------------------------------------- @@ -445,9 +447,24 @@ def test_active_source_column_mismatch_returns_empty(self, client, df): assert body["cubes"] == [] assert body["trace_cubes"] == {} + def test_hist_source_cube_has_at_most_one_cell_per_bar_and_target_bin( + self, client, df + ): + """The free axis is the source's bars, so the cube holds at most one + cell per (bar, target bin) instead of per (2048-bin, target bin).""" + spec = _two_hist_dashboard(df) + resp = client.post( + "/dashboard/update", json=_cube_payload(spec, spec.figures[0].uid) + ) + blob = base64.b64decode(_cube_body(resp)["cubes"][0]) + header = decode_fvcube_header(blob) + (dim,) = header["target_dims"] + n_cells = len(_read_u32_col(blob, header, "free_bin")) + assert 0 < n_cells <= header["free"]["p"] * dim["bins"] == _SRC_BINS * 12 + def test_zoomed_viewports_resolve_cube_domains(self, client, df): - """Source viewport → free domain verbatim; target viewport → the - SNAPPED display grid.""" + """Source and target viewport → each histogram's SNAPPED display + grid.""" spec = _two_hist_dashboard(df) src_fig_uid = spec.figures[0].uid tgt_fig_uid = spec.figures[1].uid @@ -458,7 +475,9 @@ def test_zoomed_viewports_resolve_cube_domains(self, client, df): resp = client.post("/dashboard/update", json=_cube_payload(spec, src_fig_uid)) blob = base64.b64decode(_cube_body(resp)["cubes"][0]) header = decode_fvcube_header(blob) - assert header["free"]["domain"] == [10.0, 80.0] + lo, hi, n = snap_range(10.0, 80.0, _SRC_BINS) + assert header["free"]["domain"] == [lo, hi] + assert header["free"]["p"] == n (dim,) = header["target_dims"] lo, hi, n = snap_range(5.0, 60.0, tgt_bins) assert n == tgt_bins + 1 @@ -527,6 +546,29 @@ def test_zoomed_hist_target_keeps_the_display_rows(self, client): cube_rows = sum(_read_u32_col(blob, decode_fvcube_header(blob), "count")) assert display_rows == cube_rows == 4 + def test_sibling_hist_source_bins_on_its_display_grid(self, client, df): + """A histogram source draws its bars over the union with its sibling + histograms, so its free axis must span that union too: a brush snaps + to the bars the user sees.""" + wide = df.with_columns(w=pl.col("a") * 2 - 50) + register_source(_SRC, wide, cache=True) + dash = Dashboard(wide) + src = dash.add_figure(title="Source").add_histogram(x="a", bins=16) + src.add_histogram(x="w", bins=16) + dash.add_figure(title="Target").add_histogram(x="b", bins=12) + spec = dash.to_spec(source_name=_SRC) + src_fig = spec.figures[0] + + deltas = client.post("/dashboard/update", json=_init_payload(spec)).json() + lo, step, n = deltas["figure_deltas"][src_fig.uid][0]["updates"]["x_edges"] + assert lo < wide["a"].min() + body = _cube_body( + client.post("/dashboard/update", json=_cube_payload(spec, src_fig.uid)) + ) + free = decode_fvcube_header(base64.b64decode(body["cubes"][0]))["free"] + assert free["p"] == n + assert free["domain"] == pytest.approx([lo, lo + n * step]) + def test_sibling_hist_targets_share_the_display_bin_domain(self, client, df): """Two histograms on one figure bin over their *union* min/max in the legacy delta, so their cubes must resolve the same union — otherwise the @@ -1545,10 +1587,14 @@ def test_zoomed_temporal_source_resolves_viewport(self): resp = client.post("/dashboard/update", json=_cube_payload(spec, src_uid, "t")) header = decode_fvcube_header(base64.b64decode(_cube_body(resp)["cubes"][0])) epoch = dt.datetime(1970, 1, 1) - assert header["free"]["domain"] == [ + # The parsed physical range, snapped to the source's bar lattice. + lo, hi, n = snap_range( (lo_dt - epoch) / dt.timedelta(microseconds=1), (hi_dt - epoch) / dt.timedelta(microseconds=1), - ] + _SRC_BINS, + ) + assert header["free"]["domain"] == [lo, hi] + assert header["free"]["p"] == n # --------------------------------------------------------------------------- @@ -1569,7 +1615,10 @@ def test_zoomed_source_slice_parity(self, client, df): resp = client.post("/dashboard/update", json=_cube_payload(spec, src_uid)) blob = base64.b64decode(_cube_body(resp)["cubes"][0]) header = decode_fvcube_header(blob) - assert header["free"]["domain"] == [10.0, 80.0] + # The viewport snapped to the source's bar lattice. + lo, hi, n = snap_range(10.0, 80.0, _SRC_BINS) + assert header["free"]["domain"] == [lo, hi] + assert header["free"]["p"] == n == _SRC_BINS + 1 (dim,) = header["target_dims"] lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 23.4, 57.8) @@ -1618,7 +1667,8 @@ def test_zoomed_source_with_passive_combined(self, client, df): assert tgt_uid in body["trace_cubes"] blob = base64.b64decode(body["cubes"][body["trace_cubes"][tgt_uid]]) header = decode_fvcube_header(blob) - assert header["free"]["domain"] == [10.0, 80.0] + lo, hi, _n = snap_range(10.0, 80.0, _SRC_BINS) + assert header["free"]["domain"] == [lo, hi] (dim,) = header["target_dims"] lo_bin, hi_bin, edge_lo, edge_hi = _snap(header["free"], 23.4, 57.8) @@ -1729,7 +1779,10 @@ def test_box_source_serves_hist_target_slice_parity(self, client, df): header = decode_fvcube_header(blob) assert header["free"]["kind"] == "continuous" assert header["free"]["p"] == _P - assert header["free"]["domain"] == [df["a"].min(), df["a"].max()] + assert header["free"]["domain"] == [ + df["a"].min(), + df["a"].max(), + ] sliced, direct = self._slice_and_reference(df, blob, header, 12.3, 61.7) assert 0 < sum(sliced) < df.height @@ -1774,7 +1827,10 @@ def test_line_source_serves_hist_target_slice_parity(self, client, df): header = decode_fvcube_header(blob) assert header["free"]["kind"] == "continuous" assert header["free"]["p"] == _P - assert header["free"]["domain"] == [df["a"].min(), df["a"].max()] + assert header["free"]["domain"] == [ + df["a"].min(), + df["a"].max(), + ] sliced, direct = self._slice_and_reference(df, blob, header, 12.3, 61.7) assert 0 < sum(sliced) < df.height @@ -2265,7 +2321,7 @@ def test_passive_selection_bakes_into_corr_cube(self, corr_client, corr_df): corr_df.lazy().filter(passive_expr), CubeSpec( source_name=_CORR_SRC, - free=FreeAxisSpec(column="a", p=_P, domain=(a_lo, a_hi)), + free=FreeAxisSpec(column="a", p=_SRC_BINS, domain=(a_lo, a_hi)), target_dims=(), measure=MeasureSpec(agg="corr", columns=("p", "q", "rr")), ), @@ -2927,3 +2983,127 @@ def test_grouped_line_target_served(self, cat_client, cat_df): ) assert resp.status_code == 200, resp.text assert line_uid in _cube_body(resp)["trace_cubes"] + + +# --------------------------------------------------------------------------- +# A binned source brushes its own bins (#133) +# --------------------------------------------------------------------------- + +_GRID_SRC = "_cube_grid_src" + + +def _commit_clause(column: str, snap: tuple, unit: str | None = None): + """Mirror of the client commit (``_fvCubeCommitEdges``): one clause from a + ``snap_brush`` result, a Date edge rendered as a whole-day string.""" + import datetime as dt + + from flexviz.spec import ClauseFilter + + _lo_bin, _hi_bin, lo, hi, closed = snap + if unit == "day": + if closed == "both": + hi = math.floor(hi) + + def _day(v: float) -> str: + return (dt.date(1970, 1, 1) + dt.timedelta(days=math.ceil(v))).isoformat() + + lo, hi = _day(lo), _day(hi) + return ClauseFilter(column=column, range=(lo, hi), closed=closed) + + +def _committed_rows(df: pl.DataFrame, clauses: list) -> int: + from flexviz.predicates import predicates_to_expr + from flexviz.spec import SelectionPredicate + + pred = SelectionPredicate(clauses=clauses) + return df.filter(predicates_to_expr([pred], df.schema)).height + + +def _covering(lo: float, step: float, k: int, m: int) -> tuple[float, float]: + """A brush over the centers of bins k..m and no other center.""" + return lo + (k + 0.25) * step, lo + (m + 0.75) * step + + +class TestSourceBinGridCommit: + """A brush over bins k..m of a binned source commits exactly the rows the + source draws in those bins, including values that sit on a bin edge.""" + + def _grid(self, df: pl.DataFrame, spec, column: str, viewport=None): + """(source updates of the init delta, cube header free block).""" + register_source(_GRID_SRC, df, cache=True) + get_cache().clear() + get_cube_cache().clear() + client = TestClient(app) + src_fig = spec.figures[0] + for axis, rng in (viewport or {}).items(): + spec.state.viewport[f"{src_fig.uid}/{axis}"] = AxisRange( + min=rng[0], max=rng[1] + ) + (delta,) = client.post("/dashboard/update", json=_init_payload(spec)).json()[ + "figure_deltas" + ][src_fig.uid] + payload = _cube_payload( + spec, src_fig.uid, column=column, trace_uid=src_fig.traces[0].uid + ) + blob = base64.b64decode( + _cube_body(client.post("/dashboard/update", json=payload))["cubes"][0] + ) + get_cache().clear() + get_cube_cache().clear() + self.blob = blob + return delta["updates"], decode_fvcube_header(blob)["free"] + + def _sliced_rows(self, lo_bin: int, hi_bin: int) -> int: + """Rows the cube slices over free bins lo_bin..hi_bin (1-D source).""" + header = decode_fvcube_header(self.blob) + free_bin = _read_u32_col(self.blob, header, "free_bin") + count = _read_u32_col(self.blob, header, "count") + return sum(n for fb, n in zip(free_bin, count) if lo_bin <= fb <= hi_bin) + + # An Int64 column rounds each committed bound inward (_typed_range_bounds). + @pytest.mark.parametrize("dtype", [pl.Float64, pl.Int64]) + @pytest.mark.parametrize("viewport", [None, {"x": (10.3, 80.7)}]) + def test_hist_bars_k_to_m_commit_their_rows(self, viewport, dtype): + # Integer values 0..332 in 20 bars: float rounding puts the raw edge + # lo + k*step of bar 15 a hair above 249, so a commit on the raw edge + # would drop 249 from its bar. + n = 333 * 3 + df = pl.DataFrame( + { + "a": pl.Series([i % 333 for i in range(n)], dtype=dtype), + "b": [float(i % 13) for i in range(n)], + } + ) + dash = Dashboard(df) + dash.add_figure(title="Bars").add_histogram(x="a", bins=20) + dash.add_figure(title="Target").add_histogram(x="b", bins=6) + spec = dash.to_spec(source_name=_GRID_SRC) + updates, free = self._grid(df, spec, "a", viewport) + lo_e, step_e, n_bars = updates["x_edges"] + lo, hi = free["domain"] + assert free["p"] == n_bars + assert (lo, (hi - lo) / n_bars) == pytest.approx((lo_e, step_e), rel=1e-12) + counts, centers = updates["y"], updates["x"] + for k, m in [(k, k) for k in range(n_bars)] + [(2, 9), (4, n_bars - 1)]: + a, b = _covering(lo, step_e, k, m) + snap = snap_brush(lo, hi, n_bars, a, b) + assert snap[:2] == (k, m) + # Plotly highlights the bars whose center is inside the brush. + highlighted = [i for i, c in enumerate(centers) if a <= c <= b] + assert highlighted == list(range(k, m + 1)) + want = sum(counts[k : m + 1]) + assert _committed_rows(df, [_commit_clause("a", snap)]) == want, (k, m) + assert self._sliced_rows(k, m) == want, (k, m) + + def test_hist_brush_covering_no_bar_center_is_empty(self): + df = pl.DataFrame({"a": [float(i % 101) for i in range(505)]}) + dash = Dashboard(df) + dash.add_figure(title="Bars").add_histogram(x="a", bins=20) + dash.add_figure(title="Target").add_histogram(x="a", bins=6) + _updates, free = self._grid(df, dash.to_spec(source_name=_GRID_SRC), "a") + lo, hi = free["domain"] + step = (hi - lo) / free["p"] + lo_bin, hi_bin, *_ = snap_brush( + lo, hi, free["p"], lo + 3.1 * step, lo + 3.4 * step + ) + assert hi_bin < lo_bin diff --git a/tests/test_perf_choices.py b/tests/test_perf_choices.py index 17129f91..30d7fb1d 100644 --- a/tests/test_perf_choices.py +++ b/tests/test_perf_choices.py @@ -397,6 +397,54 @@ def encode() -> bytes: ) +CUBE_GRID_ROWS = 2_000_000 +CUBE_GRID_BINS = 20 + + +def test_histogram_source_cube_is_binned_on_its_own_bars() -> None: + """Decision: a histogram source brushes on its own bars, so its cube free + axis has `P = bins`, not the fixed P = 2048 that box and line sources + keep (#133). + + Evidence: `build_cube` + `encode_fvcube` on 2M rows, 20-bin source over a + 20-bin histogram target: 18,875 cells (227 KB) at P = 2048 against 305 + cells (4.2 KB) at P = 20. A grouped target (5 groups) went 1.18 MB to + 21.7 KB, a 20 x 20 hist2d target 2.16 MB to 51.9 KB. The finer grid bought + nothing: a brush snaps to bar edges either way. + + Check: the cube of a histogram source has at most bins x bins cells, at + least 50x fewer than the same cube at P = 2048. Cell counts, not timings. + """ + from dataclasses import replace + + from flexviz.trace.hist import Histogram + + rng = np.random.default_rng(0) + df = pl.DataFrame( + {"a": rng.random(CUBE_GRID_ROWS), "b": rng.random(CUBE_GRID_ROWS)} + ) + free = Histogram(x="a", bins=CUBE_GRID_BINS).get_cube_source_spec(None) + assert free is not None + free = replace(free, domain=(0.0, 1.0)) + target = Histogram(x="b", bins=CUBE_GRID_BINS).get_cube_target_spec(None) + assert target is not None + dims = tuple(replace(d, domain=(0.0, 1.0)) for d in target.target_dims) + + def cells(free_axis: FreeAxisSpec) -> int: + spec = CubeSpec( + source_name="s", + free=free_axis, + target_dims=dims, + measure=target.measure, + ) + return build_cube(df.lazy(), spec).n_cells + + own_grid = cells(free) + fixed_grid = cells(replace(free, p=2048)) + assert own_grid <= CUBE_GRID_BINS * CUBE_GRID_BINS + assert own_grid * 50 <= fixed_grid, (own_grid, fixed_grid) + + def test_overlay_backgrounds_share_one_aggregate_call() -> None: """Decision: in overlay mode the engine runs the unfiltered background of every partition (each selection-owning figure is its own partition) in one diff --git a/tests/test_trace_hist.py b/tests/test_trace_hist.py index fcf3e9ec..dcf9967d 100644 --- a/tests/test_trace_hist.py +++ b/tests/test_trace_hist.py @@ -865,14 +865,16 @@ def test_source_spec_unzoomed_domain_none(self): assert isinstance(spec, FreeAxisSpec) assert spec.column == "val" assert spec.kind == "continuous" - assert spec.p == 2048 + assert spec.p == 10 assert spec.domain is None - def test_source_spec_zoomed_domain_is_axis_range(self): + def test_source_spec_zoomed_is_the_snapped_bar_grid(self): trace = Histogram(x="val", bins=10) - spec = trace.get_cube_source_spec((10.0, 50.0)) + spec = trace.get_cube_source_spec((10.0, 51.0)) assert spec is not None - assert tuple(spec.domain) == (10.0, 50.0) + # The display lattice of width 4.1: [8.2, 53.3] holds 11 bars. + lo, hi, n = snap_range(10.0, 51.0, 10) + assert (tuple(spec.domain), spec.p) == ((lo, hi), n) == ((8.2, 53.3), 11) def test_source_spec_temporal_kind_from_schema(self): schema = pl.Schema({"ts": pl.Datetime("us")}) From 09c78e9dd4080def324a9b604e60aca9a4273707 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 09:05:55 +0200 Subject: [PATCH 5/8] fix(cube): brush a histogram2d source on its own cells --- Architecture.md | 23 ++--- CHANGELOG.md | 18 ++-- docs/cube-support-overview.md | 6 +- flexviz/adapters/js/plotly/events.js | 68 +++++-------- flexviz/adapters/js/runtime/cube.js | 42 ++++---- flexviz/cube.py | 41 ++++---- flexviz/engine.py | 90 ++++++----------- flexviz/trace/bar.py | 1 + flexviz/trace/base.py | 12 ++- flexviz/trace/box.py | 1 + flexviz/trace/hist.py | 1 + flexviz/trace/hist2d.py | 32 +++--- flexviz/trace/line.py | 1 + flexviz/trace/pie.py | 1 + flexviz/trace/treemap.py | 1 + tests/test_browser_cube.py | 140 ++++++++++++++++++++++++-- tests/test_cube.py | 26 +++-- tests/test_cube_server.py | 141 ++++++++++++++++++++++++++- tests/test_trace_hist2d.py | 28 ++++++ 19 files changed, 482 insertions(+), 191 deletions(-) diff --git a/Architecture.md b/Architecture.md index ae3a8161..b04c1106 100644 --- a/Architecture.md +++ b/Architecture.md @@ -1079,12 +1079,13 @@ dataset size — and adds drag-time updates that flexviz previously did not have implemented. A cube is one target trace's grouping × the brushed (free) axis, holding decomposable partial -measures. A **range** free axis on a histogram source is its bars (`P = bins` over the display -grid's domain, `bins + 1` when a zoom snaps to the bar lattice), so a snapped brush edge is a bar -edge, and the commit, the redrawn box, the cube slice and Plotly's highlight cover the same bars. -Box and line sources have no bins and keep a **fixed resolution P = 2048** over the viewport -domain. A **box2d** free axis (a 2-D box-select on a hist2d source) is two range axes binned at -**P₂D = 128** each and packed into one composite `free_bin`. A **categorical** free axis +measures. A **binned** source brushes its own grid: a **range** free axis on a histogram +source is its bars (`P = bins` over the display grid's domain, `bins + 1` when a zoom snaps to the +bar lattice), and a **box2d** free axis (a 2-D box-select on a hist2d source) is its cells — two +range axes of `(nx, ny)` cells over the display grid's domains, packed into one composite +`free_bin`. So a snapped brush edge is a bar or cell edge, and the commit, the redrawn box, the +cube slice and Plotly's highlight cover the same bins. Box and line sources have no bins and keep +a **fixed resolution P = 2048** over the viewport domain. A **categorical** free axis (bar/pie/treemap source) is the exact tuple of label/path column values — no binning, no domain, dictionary-encoded in sorted order. A resolution fixed by the spec — rather than Mosaic's pixel resolution — makes the cube width-independent, content-addressable, and shareable across @@ -1178,9 +1179,9 @@ range **or categorical (bar)** selection geometry: (`fvCubeSnap` / `snap_brush`) — Plotly's highlight rule for bars. A frame whose brush covers no center shows the targets' pre-gesture state (in overlay mode without the live fg presentation), and its commit clears the figure's selection. A box2d source does the same - per axis. A 1-D grid (`p`, domain, unit) is adopted from the decoded header only: the server - resolves the unzoomed domain and a histogram snaps a zoomed axis to its bar lattice, so the - viewport alone does not give it (a commit before the first header lands stays unsnapped). Before a blob is stored, + per axis. The grid (`p`, domain, unit) is adopted from the decoded header only: the server resolves + the unzoomed domain and a binned trace snaps a zoomed axis to its lattice, so the viewport alone + does not give it (a commit before the first header lands stays unsnapped). Before a blob is stored, `fvCubeHeaderMatchesKey` checks each range axis against the key: unzoomed, the header keeps the key's `p`; zoomed, it has `p` or `p + 1` bins and (numeric axes) a domain that covers the viewport. When a histogram and a non-binned trace share the brushed column in one figure, the @@ -1351,7 +1352,7 @@ them. | `histogram` | yes | yes | Source is a 1-D range axis on its own bar grid (`P = bins`, or `bins + 1` zoomed). Target is a binned count; grouped histograms add categorical group dims. | | `box` | yes | no | Source is a 1-D range over the box data axis. Box is not a target because quantiles are not decomposable. | | `line` | yes | yes, limited | Source is an x-only range at `P=2048`. Target requires `downsample="minmax"` and numeric `y`; the target is a live-only `line_env`, and commit always POSTs. | - | `histogram2d` | yes | yes, limited | Source is a `box2d` free axis at `128 x 128`. Target is full-data-only 2-D binned count/reduce; zoomed target axes fall back to the normal POST path. | + | `histogram2d` | yes | yes, limited | Source is a `box2d` free axis on its own `x_bins x y_bins` cell grid. Target is full-data-only 2-D binned count/reduce; zoomed target axes fall back to the normal POST path. | | `bar` | yes | yes | Source is categorical labels. Target dims are label columns plus optional string `group_by`; supports `count`/`sum`/`mean`/`min`/`max`. | | `pie` | yes | yes | Source is categorical labels. Target is equivalent to an ungrouped bar with the same labels and measure. | | `treemap` | yes | yes | Source is the categorical full `path`. Target stores leaf path cells, and the client rolls them up into the hierarchy. | @@ -1367,7 +1368,7 @@ them. | hist | 1-D range on the bar grid: `p = bins` and `domain=None` unzoomed (the engine resolves the full domain, unioned with the source's sibling histograms like its bars), the viewport snapped with `snap_range` zoomed (`bins` or `bins + 1`); continuous + temporal (`us`/`ms`/`day` physical units — see “Temporal sources” below; `Datetime("ns")`/`Time` gate to no cube) | | box | 1-D range over the `data_col` (same shape/gates as hist) | | line | 1-D range over the **x** column only (line selection is x-only — see below); P=2048; source geometry independent of `downsample` | - | hist2d | **box2d**: two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` (`bin_y·P₂D + bin_x`); per-axis domains resolved by the engine; a rectangle brush slices a 2-D sub-grid | + | hist2d | **box2d**: two range axes (x, y) on the source's cell grid, `p = (nx, ny)`, packed into one composite `free_bin` (`bin_y·nx + bin_x`); the trace emits `(x_bins, y_bins)` and snaps a zoomed axis (the engine passes one viewport per select anchor) with `snap_range`, the same lattice as the display grid (which can add a cell); the engine resolves an unzoomed axis to the full data domain; a rectangle brush slices a 2-D sub-grid | | bar / pie | categorical over the ordered label columns (`axis_range` ignored — label geometry is viewport-independent) | | treemap | categorical over the full `path` | diff --git a/CHANGELOG.md b/CHANGELOG.md index f673fd61..31d3f07a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,14 +43,16 @@ independently. `flexviz` pins a compatible `flexviz-polars` range. ### Fixed -- A brush on a histogram snaps to the edges of its bars. The bars that Plotly - highlights now hold exactly the rows that the other figures keep, also for a - value on a bar edge and on a `Date` axis. A brush that covers no bar center - clears the selection. Before, the brush snapped to a fixed grid of 2048 - steps, so the other figures also kept part of the rows of the two edge bars. - The live-brush data of a histogram source now grows with its number of bars - instead of the 2048 steps: with the default 20 bars it is about 50 times - smaller. +- A brush on a histogram or a histogram2d snaps to the edges of its bars or + cells. The bars that Plotly highlights now hold exactly the rows that the + other figures keep, also for a value on a bar edge and on a `Date` axis. A + brush that covers no bar or cell center clears the selection. Before, the + brush snapped to a fixed grid of 2048 steps (128 per axis for a + histogram2d), so the other figures also kept part of the rows of the edge + bars or cells. The live-brush data of such a source now grows with its bars + or cells instead of the fixed grid: with the default 20 bars or 20 × 20 + cells it is about 40 to 50 times smaller. A histogram2d finer than + 128 × 128 cells makes it larger than before. - A histogram or a histogram2d on a `Date` column draws each bar at its bin center, and its hover shows the time of day of that center. Before, the center was rounded to a whole day, so a bar could sit up to half a day off diff --git a/docs/cube-support-overview.md b/docs/cube-support-overview.md index 7ec5fe07..728a5d09 100644 --- a/docs/cube-support-overview.md +++ b/docs/cube-support-overview.md @@ -21,7 +21,7 @@ A source trace implements `get_cube_source_spec()` and produces a **free axis** | `histogram` | **range** (continuous / temporal) | 1-D, the histogram's own bars over the viewport (or full-data) domain | | `box` | **range** | 1-D over the `data_col` (same gates as hist) | | `line` | **range** | 1-D over the **x** column only (line selection is x-only); P=2048 | -| `histogram2d` | **box2d** | two range axes (x, y) at P₂D=128 each, packed into one composite `free_bin` | +| `histogram2d` | **box2d** | two range axes (x, y) on the source's own `x_bins × y_bins` cell grid, packed into one composite `free_bin` | | `bar` | **categorical** | over the ordered label column(s) | | `pie` | **categorical** | over the label column(s) | | `treemap` | **categorical** | over the full `path` (prefix selects a subtree) | @@ -73,8 +73,8 @@ source free-axis kinds can feed each target measure.** Notes on the two `❌` columns: - **`box2d × {line_env, corr}` (#47):** a feasibility-free but real **cell-count / wire-size - wall** — a box2d `line_env` cube is `n_x_buckets × P₂D²` ≈ `500 × 128²` ≈ 8.2M cells per - line. Gated off; those targets fall back to the per-commit recompute. Tracked in #47. + wall** — a box2d `line_env` cube is `n_x_buckets × nx·ny` cells per line, for example + `500 × 128²` ≈ 8.2M for a 128 × 128 cell source. Gated off; those targets fall back to the per-commit recompute. Tracked in #47. - **`median` / `n_unique` / box quantiles:** mathematically not decomposable over a partition — never cube targets, by any source. diff --git a/flexviz/adapters/js/plotly/events.js b/flexviz/adapters/js/plotly/events.js index 118d4175..c7f1dd2f 100644 --- a/flexviz/adapters/js/plotly/events.js +++ b/flexviz/adapters/js/plotly/events.js @@ -540,8 +540,8 @@ function _fvCubeSourceConstraint(figUid) { // key (e.g. map coordinates). Temporal viewports (date-string ranges) // convert to epoch-ms floats — a SELF-CONSISTENT key token, client-local // and never sent to the server (the server parses the original strings via -// the schema dtype, contract G). The token is NOT a snap domain: temporal -// snapping always uses the decoded header's physical domain. A reversed axis +// the schema dtype, contract G). The token is only a key: every gesture +// snaps on the decoded header's grid. A reversed axis // stores its viewport high to low and the server sorts it, so the domain is // sorted too, or the key never matches the cube header. function _fvCubeViewportDomain(figUid, anchor) { @@ -557,14 +557,6 @@ function _fvCubeViewportDomain(figUid, anchor) { return undefined; } -// True when the stored viewport for this axis is a temporal (date-string) -// range — the gesture must then defer its snap grid to the decoded header. -function _fvCubeViewportIsTemporal(figUid, anchor) { - if (!anchor) return false; - const rng = figureViewportRanges(figUid)[anchor]; - return Array.isArray(rng) && rng.some(v => typeof v === 'string'); -} - const _FV_CUBE_AGGS = ['count', 'sum', 'mean', 'min', 'max']; // Canonical store key for one target trace, or null when the trace is not @@ -843,13 +835,14 @@ function _fvCubeGestureSnap(gesture, a, b) { return fvCubeSnap(gesture.snapDomain, gesture.p, a, b); } -// The free-axis bin count a 1-D range source asks for: a histogram brushes its -// own bars, box and line sources a fixed fine grid. A zoom can add one bin, -// because the server snaps the viewport outward to a lattice of width -// (raw span / n). +// The free-axis bin count a source trace asks for: a histogram brushes its +// own bars, a histogram2d its [x, y] cells, box and line sources a fixed fine +// grid. A zoom can add one bin per axis, because the server snaps the +// viewport outward to a lattice of width (raw span / n). function _fvCubeSourceP(traceUid) { const ts = traceSpecByUid[traceUid]; if (ts && ts.trace_type === 'histogram') return ts.params.bins; + if (ts && ts.trace_type === 'histogram2d') return [ts.params.x_bins, ts.params.y_bins]; return _FV_CUBE_P; } @@ -896,40 +889,40 @@ function _fvCubeRangeGestureStart(figUid, figSpec, constraint) { } // Adopt a decoded box2d free header (contract H) as the gesture's per-axis -// snap grid: each axis's physical domain and temporal unit (from the header's -// "units" list). Idempotent — first adoption per axis wins; zoomed numeric -// axes already hold their viewport snapDomain. +// snap grid: each axis's cells ("p"), physical domain, and temporal unit. +// The header is the only grid source: a zoomed axis is the server's snapped +// lattice, which the viewport alone does not give. Idempotent — first +// adoption per axis wins. function _fvCubeAdoptBox2dGrid(gesture, free) { if (!free || free.kind !== 'box2d' || !Array.isArray(free.domains)) return; const units = free.units || [null, null]; for (let a = 0; a < 2; a++) { const ax = gesture.axes[a]; const dom = free.domains[a]; - if (!Array.isArray(dom)) continue; - if (!ax.snapDomain) ax.snapDomain = dom.slice(); - if (units[a] && !ax.unit) ax.unit = units[a]; + if (!Array.isArray(dom) || ax.snapDomain) continue; + ax.snapDomain = dom.slice(); + ax.p = free.p[a]; + ax.unit = units[a] || null; } } -// Snap one box2d axis on its P=128 grid. Returns the fvCubeSnap result. +// Snap one box2d axis on its cell grid. Returns the fvCubeSnap result. function _fvCubeBox2dSnapAxis(ax, a, b) { - return fvCubeSnap(ax.snapDomain, _FV_CUBE_BOX2D_P, a, b); + return fvCubeSnap(ax.snapDomain, ax.p, a, b); } // box2d (hist2d) source gesture (contract H): the free axis is a 2-D box over -// (x_col, y_col) at P₂D=128 per axis. Snaps BOTH axes each frame; the -// per-frame payload is the rectangle of composite free bins. Mirrors +// (x_col, y_col) on the source's own cell grid. Snaps BOTH axes each frame; +// the per-frame payload is the rectangle of composite free bins. Mirrors // _fvCubeRangeGestureStart but with a per-axis structure. function _fvCubeBox2dGestureStart(figUid, figSpec, constraint) { const domX = _fvCubeViewportDomain(figUid, constraint.x.anchor); const domY = _fvCubeViewportDomain(figUid, constraint.y.anchor); if (domX === undefined || domY === undefined) return { inert: true }; - const tempX = _fvCubeViewportIsTemporal(figUid, constraint.x.anchor); - const tempY = _fvCubeViewportIsTemporal(figUid, constraint.y.anchor); const freeDesc = { c: [constraint.x.column, constraint.y.column], k: 'box2d', - p: _FV_CUBE_BOX2D_P, + p: _fvCubeSourceP(constraint.uid), d: [domX, domY], }; const passiveKey = fvCubePassiveKey(DASHBOARD_SPEC.state.selections, figUid); @@ -943,11 +936,10 @@ function _fvCubeBox2dGestureStart(figUid, figSpec, constraint) { sourceName: figSpec.source, cols: [constraint.x.column, constraint.y.column], sourceZoomed: domX != null || domY != null, - // Per-axis snap grids. A zoomed NUMERIC viewport is the snap domain - // directly; a temporal/unzoomed axis defers to the decoded header. + // Per-axis snap grids, adopted from the decoded header. axes: [ - { snapDomain: (!tempX && domX) || null, unit: null }, - { snapDomain: (!tempY && domY) || null, unit: null }, + { snapDomain: null, p: null, unit: null }, + { snapDomain: null, p: null, unit: null }, ], targets: _fvCubeEnumerateTargets(figUid, figSpec.source, freeDesc, passiveKey), live: false, @@ -957,9 +949,7 @@ function _fvCubeBox2dGestureStart(figUid, figSpec, constraint) { savedLayers: null, }; if (!gesture.sourceZoomed) { - _fvCubeAdoptBox2dGrid( - gesture, _fvCubeRememberedBox2dGrid(figSpec.source, gesture.cols) - ); + _fvCubeAdoptBox2dGrid(gesture, _fvCubeRememberedBox2dGrid(constraint.uid)); } const capable = gesture.targets.filter(t => t.capable); if (capable.length && capable.every(t => fvCubeStoreHas(t.key))) { @@ -1072,9 +1062,7 @@ async function _fvCubeFetchForGesture(gesture) { if (gesture.kind === 'categorical') return; if (gesture.kind === 'box2d') { if (!gesture.sourceZoomed) { - _fvCubeRememberBox2dGrid( - gesture.sourceName, gesture.cols, entry.header.free - ); + _fvCubeRememberBox2dGrid(gesture.traceUid, entry.header.free); } _fvCubeAdoptBox2dGrid(gesture, entry.header.free); return; @@ -1774,15 +1762,13 @@ function _fvCubeEchoOfStoredSelection(figUid, range) { // A box2d source's per-axis snap grid is remembered as a box2d free header // (units/domains lists), not a per-column free record; resolve once. const isBox2d = !!constraint && constraint.kind === 'box2d'; - const box2dGrid = isBox2d && figSpec && figSpec.source - ? _fvCubeRememberedBox2dGrid(figSpec.source, [constraint.x.column, constraint.y.column]) - : null; + const box2dGrid = isBox2d ? _fvCubeRememberedBox2dGrid(constraint.uid) : null; const sourceP = constraint ? _fvCubeSourceP(constraint.uid) : null; for (let ci = 0; ci < axisConstraints.length; ci++) { const ac = axisConstraints[ci]; // The nominal bin count: a zoom's extra lattice bin changes the step by // at most 1/n, far below the half-bin tolerance. - const p = isBox2d ? _FV_CUBE_BOX2D_P : sourceP; + const p = isBox2d ? sourceP[ci] : sourceP; // 1-D sources remember a per-(source, column, p) free record; box2d axes // read their {domain, unit} from the remembered box2d header by index. let recDomain = null; diff --git a/flexviz/adapters/js/runtime/cube.js b/flexviz/adapters/js/runtime/cube.js index f22f32e8..3bf4996b 100644 --- a/flexviz/adapters/js/runtime/cube.js +++ b/flexviz/adapters/js/runtime/cube.js @@ -3,9 +3,8 @@ // // Cube live-brush path (see Architecture.md, "Cube Pre-Aggregation & Live Brushing"): // a cube is a pre-aggregation of one target trace's dims × the active brush -// column binned to P bins (a histogram source's own bars, 128 per axis for a -// histogram2d, 2048 for box and line; or its category tuples for a -// categorical source). +// column binned to P bins (a binned source's own grid, 2048 for box and line; +// or its category tuples for a categorical source). // The server builds + encodes it once (FVCube v1); this module decodes the // blob, stores it under a client-local canonical descriptor key, and // re-slices it per drag frame — turning every mousemove of a live brush into @@ -27,7 +26,6 @@ // the existing delta-application path (`setLayerData` + `_fvRenderFigure`). const _FV_CUBE_P = 2048; -const _FV_CUBE_BOX2D_P = 128; // P₂D — the per-axis resolution of a box2d source let _FV_CUBE_STORE_BUDGET_BYTES = 256 * 1024 * 1024; // Test hook: override the store byte budget; returns the previous value. @@ -232,9 +230,10 @@ function fvCubeHeaderMatchesKey(key, header) { if (!keyCols.every((c, i) => c === cols[i])) return false; const keyD = Array.isArray(keyFree.d) ? keyFree.d : [null, null]; const units = free.units || [null, null]; + const p = free.p || []; const doms = free.domains || []; for (let a = 0; a < 2; a++) { - if (!_fvCubeAxisMatchesKey(keyFree.p, keyD[a], free.p, doms[a], !!units[a])) { + if (!_fvCubeAxisMatchesKey(keyFree.p[a], keyD[a], p[a], doms[a], !!units[a])) { return false; } } @@ -339,17 +338,18 @@ function fvCubeFreeDomain(sourceName, column, p) { } // Server-resolved box2d free headers seen in decoded headers (contract H), -// keyed by (source, [cx, cy]). Lets a later unzoomed box2d gesture on the -// same source adopt its per-axis snap grid before its own response lands. +// keyed by the source trace uid: the grid is that trace's own cells, so two +// hist2d traces over the same columns can differ. Lets a later unzoomed box2d +// gesture adopt its per-axis snap grid before its own response lands. const _fvCubeBox2dGrids = new Map(); -function _fvCubeRememberBox2dGrid(sourceName, cols, free) { +function _fvCubeRememberBox2dGrid(traceUid, free) { if (!free || free.kind !== 'box2d' || !Array.isArray(free.domains)) return; - _fvCubeBox2dGrids.set(JSON.stringify([sourceName, cols]), free); + _fvCubeBox2dGrids.set(traceUid, free); } -function _fvCubeRememberedBox2dGrid(sourceName, cols) { - return _fvCubeBox2dGrids.get(JSON.stringify([sourceName, cols])) || null; +function _fvCubeRememberedBox2dGrid(traceUid) { + return _fvCubeBox2dGrids.get(traceUid) || null; } // --------------------------------------------------------------------------- @@ -402,9 +402,15 @@ function decodeFVCube(bytes) { } // CSR offsets over the free_bin-sorted rows: rows of bin b are - // binStart[b] .. binStart[b+1]-1, for every bin 0.._fvCubeMaxBin. + // binStart[b] .. binStart[b+1]-1, for every bin 0.._fvCubeMaxBin. A fine + // histogram2d grid makes this index large even for a small blob, so an entry + // over the store budget is only sized, not built: fvCubeStorePut refuses it. const freeBin = cols.free_bin; - const binStart = new Uint32Array(_fvCubeMaxBin(header) + 2); + const slots = _fvCubeMaxBin(header) + 2; + if (bytesTotal + 4 * slots > _FV_CUBE_STORE_BUDGET_BYTES) { + return { header, bytes: bytesTotal + 4 * slots }; + } + const binStart = new Uint32Array(slots); for (let i = 0; i < freeBin.length; i++) binStart[freeBin[i] + 1]++; for (let b = 1; b < binStart.length; b++) binStart[b] += binStart[b - 1]; bytesTotal += binStart.byteLength; @@ -534,20 +540,20 @@ function fvCubeSnap(domain, p, a, b) { } // The highest valid free-bin index for an entry: the last category for a -// categorical axis, the composite top p*p-1 for box2d (contract H), the top -// bin p-1 otherwise. Used to clamp slice ranges. +// categorical axis, the composite top nx*ny-1 for box2d (contract H), the +// top bin p-1 otherwise. Used to clamp slice ranges. function _fvCubeMaxBin(header) { if (header.free.kind === 'categorical') return header.free.categories.length - 1; - if (header.free.kind === 'box2d') return header.free.p * header.free.p - 1; + if (header.free.kind === 'box2d') return header.free.p[0] * header.free.p[1] - 1; return header.free.p - 1; } // Build the per-row composite free-bin ranges of a snapped 2-D box (contract // H): for each by in [ly..hy], one inclusive range [by*S+lx, by*S+hx]. The // rows of one by form a contiguous CSR block, so the generalized slice walks -// binStart[by*S+lx] .. binStart[by*S+hx+1] exactly. S = p. +// binStart[by*S+lx] .. binStart[by*S+hx+1] exactly. S = nx. function fvCubeRectRanges(header, snap2d) { - const s = header.free.p; + const s = header.free.p[0]; const lx = snap2d.x.loBin, hx = snap2d.x.hiBin; const ly = snap2d.y.loBin, hy = snap2d.y.hiBin; const ranges = []; diff --git a/flexviz/cube.py b/flexviz/cube.py index 1b20432d..de850a8c 100644 --- a/flexviz/cube.py +++ b/flexviz/cube.py @@ -201,17 +201,20 @@ class FreeAxisSpec: A ``"box2d"`` axis (hist2d source, contract H) is a 2-D rectangular brush: ``columns = (x_col, y_col)`` with ``columns[0] == column`` (x is the - primary ``active_source.column`` join key), ``p = P₂D = 128`` per axis, and - the per-axis domains live in ``domains = ((lox,hix),(loy,hiy))`` (the - single-axis ``domain`` stays ``None``). Each axis is binned with the shared - arithmetic; the composite free bin is ``bin_y * p + bin_x``. - ``unit`` is per-axis for box2d — encoded as a 2-tuple ``(unit_x, unit_y)`` - — and is set by the engine from the schema dtypes. + primary ``active_source.column`` join key). It is the source's own cell + grid: ``p = (nx, ny)`` cells per axis over the per-axis domains in + ``domains = ((lox,hix),(loy,hiy))`` (the single-axis ``domain`` stays + ``None``), so a snapped brush edge is a cell edge. Each axis is binned with + the shared arithmetic; the composite free bin is + ``bin_y * nx + bin_x``. ``unit`` is per-axis for box2d — encoded as a + 2-tuple ``(unit_x, unit_y)`` — and is set by the engine from the schema + dtypes. """ column: str kind: FreeAxisKind = "continuous" - p: int = 2048 + # Bins per axis; a (nx, ny) pair for box2d. + p: int | tuple[int, int] = 2048 domain: tuple[float, float] | None = None columns: tuple[str, ...] | None = None # Physical unit for kind="temporal" (contract G); the engine sets it from @@ -246,7 +249,11 @@ def __post_init__(self) -> None: ) if self.domain is not None: raise ValueError("box2d free axis takes no single domain (use domains)") + if not (isinstance(self.p, tuple) and len(self.p) == 2): + raise ValueError("box2d free axis requires p = (nx, ny)") else: + if not isinstance(self.p, int): + raise ValueError(f"free axis kind {self.kind!r} takes an int p") if self.columns is not None: raise ValueError(f"free axis kind {self.kind!r} takes no columns") if self.domains is not None: @@ -432,7 +439,7 @@ def slice_agg_box2d( then combine + finalize like ``slice_agg``. Mirrors the client's rectangle slice (``fvCubeSliceRect``).""" (lx, hx), (ly, hy) = self._snap_box2d(x_lo, x_hi, y_lo, y_hi) - s = self.spec.free.p # the composite stride + s = self.spec.free.p[0] # the composite stride nx codes: list[int] = [] for by in range(ly, hy + 1): row = by * s @@ -448,10 +455,10 @@ def _snap_box2d( self, x_lo: float, x_hi: float, y_lo: float, y_hi: float ) -> tuple[tuple[int, int], tuple[int, int]]: """Per-axis ``snap_brush`` bin pairs of the box's corners, against each - axis's grid.""" + axis's cell grid.""" free = self.spec.free (lox, hix), (loy, hiy) = free.domains # type: ignore[misc] - px = py = free.p + px, py = free.p x = snap_brush(lox, hix, px, x_lo, x_hi) y = snap_brush(loy, hiy, py, y_lo, y_hi) return (x[0], x[1]), (y[0], y[1]) @@ -1115,8 +1122,8 @@ def _build_box2d_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: A range-like build: each of the two axes is binned like the display kernel (``_fixed_hist_bin_expr``, filter-don't-clip via ``is_between`` per axis). - The composite free key is ``free_bin = bin_y * p + bin_x`` (indices - ``0..p*p-1``). Temporal axes run on their physical representation per + The composite free key is ``free_bin = bin_y * nx + bin_x`` (indices + ``0..nx*ny-1``). Temporal axes run on their physical representation per axis (contract G). Reuses ``_target_group_exprs`` / ``_measure_exprs`` (a pure source build has no target dims, but the path stays general). """ @@ -1134,7 +1141,7 @@ def _build_box2d_cube(ldf: pl.LazyFrame, spec: CubeSpec) -> CubeResult: cx, cy = free.columns # type: ignore[misc] (lox, hix), (loy, hiy) = free.domains unit_x, unit_y = _box2d_units(free) - px = py = free.p + px, py = free.p val_x = _box2d_axis(cx, unit_x) val_y = _box2d_axis(cy, unit_y) @@ -1425,8 +1432,8 @@ def encode_fvcube(result: CubeResult, cube_id: str) -> bytes: elif spec.free.kind == "box2d": # The composite free_bin is already a u32 (build cast it to Int32); the # encode below treats it like any range free_bin. The header free block - # carries the per-axis grid ("p" per axis and "domains"); the - # composite stride is p. Temporal axes add "units" = + # carries the per-axis cell grid ("p" = [nx, ny] and "domains"); the + # composite stride is nx. Temporal axes add "units" = # [unit_x|null, unit_y|null]. frame = result.frame.sort(["free_bin", *group_cols]) (lox, hix), (loy, hiy) = spec.free.domains # type: ignore[misc] @@ -1434,7 +1441,7 @@ def encode_fvcube(result: CubeResult, cube_id: str) -> bytes: free_block = { "kind": "box2d", "cols": list(spec.free.columns or ()), - "p": spec.free.p, + "p": list(spec.free.p), "domains": [[lox, hix], [loy, hiy]], } if unit_x is not None or unit_y is not None: @@ -1691,7 +1698,7 @@ def cube_content_key(spec: CubeSpec) -> str: free_payload = { "c": list(spec.free.columns or ()), "k": spec.free.kind, - "p": spec.free.p, + "p": list(spec.free.p), "d": [list(domains[0]), list(domains[1])] if domains else None, } else: diff --git a/flexviz/engine.py b/flexviz/engine.py index 0931208c..f8ff7784 100644 --- a/flexviz/engine.py +++ b/flexviz/engine.py @@ -140,6 +140,24 @@ def _normalize_viewports( } +def _box2d_with_units( + candidate: FreeAxisSpec, schema: pl.Schema | None +) -> FreeAxisSpec | None: + """A box2d (hist2d) free axis with each temporal axis's physical unit from + the schema dtype (contract H); ``None`` gates the cube when an axis is + ``Datetime("ns")`` or ``Time``.""" + units: list[str | None] = [] + for col in candidate.columns or (): + dtype = _dtype_for_col(schema, col) + unit = None + if dtype is not None and dtype.is_temporal(): + unit = temporal_unit(dtype) + if unit is None: + return None + units.append(unit) + return replace(candidate, unit=(units[0], units[1]) if any(units) else None) + + class FlexEngine: """Renderer-agnostic, fully stateless aggregation engine. @@ -546,16 +564,21 @@ def _locate_free_axis( trace = self._scalable_traces.get(ti.uid) if trace is None: return None - anchor = trace.select_axes[0] if trace.select_axes else None - candidate = trace.get_cube_source_spec( + # One viewport per selectable anchor: a 2-D source (hist2d) snaps + # both axes to its own cell grid. A missing anchor is unzoomed. + columns = trace._make_selection_spec().axis_columns + axis_range, y_range = ( self._cube_axis_range( viewports_by_figure, ti.figure_uid, anchor, schema=schema, - column=active_source.column, - ), - schema=schema, + column=columns.get(anchor), + ) + for anchor in (*trace.select_axes, None, None)[:2] + ) + candidate = trace.get_cube_source_spec( + axis_range, schema=schema, y_range=y_range ) if candidate is not None and candidate.column == active_source.column: if candidate.kind == "temporal": @@ -567,66 +590,11 @@ def _locate_free_axis( return None candidate = replace(candidate, unit=unit) elif candidate.kind == "box2d": - candidate = self._locate_box2d_axis( - candidate, ti, viewports_by_figure, schema - ) + candidate = _box2d_with_units(candidate, schema) return candidate return None return None - def _locate_box2d_axis( - self, - candidate: FreeAxisSpec, - ti: TraceInfo, - viewports_by_figure: dict[str, dict[str, Any]], - schema: pl.Schema | None, - ) -> FreeAxisSpec | None: - """Resolve a box2d (hist2d) free axis's per-axis units and viewports - (contract H). ``active_source.column`` is the x column (validated in - ``_locate_free_axis``); the trace's two select anchors map to its two - columns. Each temporal axis takes its physical unit from the schema - dtype (Datetime("ns")/Time gate to no cube); each axis's viewport range - is resolved independently (``None`` = unzoomed → engine-resolved full - domain). Returns ``None`` to gate the whole cube.""" - trace = self._scalable_traces.get(ti.uid) - if trace is None or len(candidate.columns or ()) != 2: - return None - cx, cy = candidate.columns # type: ignore[misc] - # The two select anchors, in (x, y) column order. select_axes is - # (x_anchor, y_anchor) for hist2d; axes[0] is x, axes[1] is y. - anchors = trace.select_axes - if len(anchors) < 2: - return None - anchor_x, anchor_y = anchors[0], anchors[1] - - units: list[str | None] = [] - for col in (cx, cy): - dtype = _dtype_for_col(schema, col) - if dtype is not None and dtype.is_temporal(): - unit = temporal_unit(dtype) - if unit is None: - return None # ns/Time gate - units.append(unit) - else: - units.append(None) - - dom_x = self._cube_axis_range( - viewports_by_figure, ti.figure_uid, anchor_x, schema=schema, column=cx - ) - dom_y = self._cube_axis_range( - viewports_by_figure, ti.figure_uid, anchor_y, schema=schema, column=cy - ) - # Per-axis None viewports are resolved to the full data domain in - # _resolve_cube_domains; carry a partial (x-zoom only / y-zoom only) as - # a domains tuple with one resolved + one None axis. - domains: tuple | None - if dom_x is None and dom_y is None: - domains = None - else: - domains = (dom_x, dom_y) - new_unit = (units[0], units[1]) if any(units) else None - return replace(candidate, unit=new_unit, domains=domains) - @staticmethod def _cube_axis_range( viewports_by_figure: dict[str, dict[str, Any]], diff --git a/flexviz/trace/bar.py b/flexviz/trace/bar.py index 232c5ea0..9f8943d4 100644 --- a/flexviz/trace/bar.py +++ b/flexviz/trace/bar.py @@ -163,6 +163,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A box-drag over bars defines a categorical free axis on the label columns: the free key is the label tuple, ``columns[0]`` the primary diff --git a/flexviz/trace/base.py b/flexviz/trace/base.py index 4f892b56..96766748 100644 --- a/flexviz/trace/base.py +++ b/flexviz/trace/base.py @@ -164,8 +164,8 @@ def select_axes(self) -> tuple[str, ...]: """The anchor ids this trace emits selection clauses on. Computed (not cached) from ``_default_select_axes()``; the cube engine - (``FlexEngine._locate_free_axis`` / ``_locate_box2d_axis``) reads it to - map a brushed source trace onto its free-axis column(s).""" + (``FlexEngine._locate_free_axis``) reads it to map a brushed source + trace onto its free-axis column(s).""" return self._default_select_axes() def _make_selection_spec(self) -> TraceSelectionSpec: @@ -206,11 +206,15 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + *, + y_range: tuple[float, float] | None = None, ) -> FreeAxisSpec | None: """The free axis a brush on this trace defines, or None (not a cube source). - axis_range — the source figure's viewport on this trace's selectable axis - (None = unzoomed; domain resolution happens in the engine). + axis_range — the source figure's viewport on this trace's first + selectable axis, y_range on its second (a 2-D source); None = unzoomed + (the engine resolves the full data domain). A binned trace snaps a + viewport to its own bin lattice here, so it owns its grid. """ return None diff --git a/flexviz/trace/box.py b/flexviz/trace/box.py index 0bc6e274..a252c157 100644 --- a/flexviz/trace/box.py +++ b/flexviz/trace/box.py @@ -111,6 +111,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A brush on a box plot defines a 1-D free axis on its data column. diff --git a/flexviz/trace/hist.py b/flexviz/trace/hist.py index 4686cf4c..6123098d 100644 --- a/flexviz/trace/hist.py +++ b/flexviz/trace/hist.py @@ -245,6 +245,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A brush on a histogram defines a 1-D free axis on its data column. diff --git a/flexviz/trace/hist2d.py b/flexviz/trace/hist2d.py index 97bf1408..f062966b 100644 --- a/flexviz/trace/hist2d.py +++ b/flexviz/trace/hist2d.py @@ -48,7 +48,12 @@ _temporal_dtype_for_col, ) from .batch_fold import hist2d_fold_plan -from .bin_grid import axis_edges, hist2d_count_expr, hist2d_reduce_expr +from .bin_grid import ( + axis_edges, + hist2d_count_expr, + hist2d_reduce_expr, + snapped_domain, +) _DEFAULT_COLOR_SCALE = "Viridis" _DEFAULT_COLOR_RANGE: HeatmapColorRange = "auto" @@ -303,19 +308,21 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + *, + y_range: tuple[float, float] | None = None, ) -> FreeAxisSpec | None: """A box-select on a 2-D histogram defines a **box2d** free axis on its ``(x_col, y_col)`` pair (contract H). ``column`` is the x column (the primary ``active_source.column`` join - key); ``columns = (x_col, y_col)``; ``p = P₂D = 128`` per axis. The - per-axis ``domains`` are resolved by the **engine** (box2d domain - resolution is two-axis: this method's ``axis_range`` is only the - x-anchor viewport, so it cannot fill both), exactly as the 1-D - temporal block has the engine set ``unit``/``domains``. Both columns - must be numeric or temporal; an unsuitable dtype (when a schema is - available) gates to ``None`` — the box2d branch in ``_locate_free_axis`` - validates the per-axis temporal units and resolves the two viewports. + key); ``columns = (x_col, y_col)``. The free axis is this trace's own + cell grid: ``p = (x_bins, y_bins)`` and ``domains=None`` unzoomed (the + engine resolves the full data domain per axis); a zoomed axis + (``axis_range`` for x, ``y_range`` for y) is snapped to the display + lattice like ``axis_edges`` (``snapped_domain``), which can add a + cell. Both columns must be numeric or temporal; an unsuitable dtype + (when a schema is available) gates to ``None``. The engine sets the + per-axis temporal units (``_box2d_with_units``). """ x_col, y_col = self.x_col, self.y_col if not isinstance(x_col, str) or not isinstance(y_col, str): @@ -327,12 +334,15 @@ def get_cube_source_spec( dtype = _dtype_for_col(schema, col) if dtype is not None and not (dtype.is_numeric() or dtype.is_temporal()): return None + dom_x, nx = snapped_domain(axis_range, self.x_bins) + dom_y, ny = snapped_domain(y_range, self.y_bins) return FreeAxisSpec( column=x_col, kind="box2d", - p=128, + p=(nx, ny), columns=(x_col, y_col), - domains=None, + # The engine resolves each None axis to the full data domain. + domains=None if dom_x is None and dom_y is None else (dom_x, dom_y), ) def get_cube_target_spec( diff --git a/flexviz/trace/line.py b/flexviz/trace/line.py index 0cde2849..289369ea 100644 --- a/flexviz/trace/line.py +++ b/flexviz/trace/line.py @@ -507,6 +507,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A brush on a line defines a 1-D free axis on its **x** column. diff --git a/flexviz/trace/pie.py b/flexviz/trace/pie.py index 6d515d87..0a2ada30 100644 --- a/flexviz/trace/pie.py +++ b/flexviz/trace/pie.py @@ -127,6 +127,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A slice click defines a categorical free axis on the label columns (OR'd multi-click = a union of category keys at slice time). diff --git a/flexviz/trace/treemap.py b/flexviz/trace/treemap.py index cbce935d..9d11cb14 100644 --- a/flexviz/trace/treemap.py +++ b/flexviz/trace/treemap.py @@ -74,6 +74,7 @@ def get_cube_source_spec( self, axis_range: tuple[float, float] | None, schema: pl.Schema | None = None, + **_: Any, ) -> FreeAxisSpec | None: """A node click defines a categorical free axis on the full ``path``: the free key is the root-to-leaf tuple; a depth-``k`` click is a diff --git a/tests/test_browser_cube.py b/tests/test_browser_cube.py index 865958b2..dd69c9c6 100644 --- a/tests/test_browser_cube.py +++ b/tests/test_browser_cube.py @@ -2970,6 +2970,52 @@ def test_oversized_entry_refused_smaller_admitted( assert result["hasSmall"] is True assert result["bytesAfterSmall"] == result["before"] + 200 + def test_index_over_budget_is_sized_not_built(self, page: Page, server_port: int): + """A histogram2d source on a fine grid needs a dense bin index of + 4 x (nx x ny + 1) bytes, even for a small blob. An entry whose index + passes the budget is refused before that index is built.""" + import base64 + + df = pl.DataFrame( + {"a": [0.0, 0.5, 1.0], "b": [0.0, 0.5, 1.0], "c": list("xyz")} + ) + spec = CubeSpec( + source_name="s", + free=FreeAxisSpec( + column="a", + kind="box2d", + columns=("a", "b"), + p=(500, 500), + domains=((0.0, 1.0), (0.0, 1.0)), + ), + target_dims=(TargetDimSpec(column="c", kind="categorical"),), + measure=MeasureSpec(agg="count"), + ) + blob = encode_fvcube(build_cube(df.lazy(), spec), "k") + url = _two_hist_dashboard_url(server_port, "_cube_browser_index", "auto") + page.goto(url) + _wait_for_init(page, "plotly") + + result = page.evaluate( + """(b64) => { + const prevBudget = fvCubeStoreSetBudget(65536); + const entry = fvDecodeFVCube(Uint8Array.from(atob(b64), c => c.charCodeAt(0))); + const out = { + built: 'binStart' in entry, + bytes: entry.bytes, + stored: fvCubeStorePut('k-index', entry), + }; + fvCubeStoreSetBudget(prevBudget); + fvCubeStoreReset(); + return out; + }""", + base64.b64encode(blob).decode("ascii"), + ) + assert len(blob) < 1024 + assert result["built"] is False + assert result["bytes"] > 4 * 500 * 500 + assert result["stored"] is False + def test_oversized_cube_degrades_gesture_cleanly( self, page: Page, server_port: int ): @@ -5618,8 +5664,8 @@ def test_spearman_corr_mixed_commit_self_heals(self, page: Page, server_port: in # handleSelecting and so works for box2d unchanged. The tests below prove the # standard path (live updates + one cube_request) and the edit-drag replay. +# The source hist2d's cells per axis: the box2d free axis is this grid. _BOX2D_NX, _BOX2D_NY = 8, 6 -_BOX2D_P = 128 # the fixed per-axis grid of a box2d free axis def _box2d_df() -> pl.DataFrame: @@ -5685,7 +5731,7 @@ def _reference_box2d_hist_counts( free=FreeAxisSpec( column="a", kind="box2d", - p=_BOX2D_P, + p=(_BOX2D_NX, _BOX2D_NY), columns=("a", "b"), domains=((a_lo, a_hi), (b_lo, b_hi)), ), @@ -5700,9 +5746,9 @@ def _reference_box2d_hist_counts( measure=MeasureSpec(agg="count"), ) result = build_cube(df.lazy(), spec) - lx, hx = _committed_bins(ex[0], ex[1], (a_lo, a_hi, _BOX2D_P)) - ly, hy = _committed_bins(ey[0], ey[1], (b_lo, b_hi, _BOX2D_P)) - s = _BOX2D_P + lx, hx = _committed_bins(ex[0], ex[1], (a_lo, a_hi, _BOX2D_NX)) + ly, hy = _committed_bins(ey[0], ey[1], (b_lo, b_hi, _BOX2D_NY)) + s = _BOX2D_NX codes = [] for by in range(ly, hy + 1): codes.extend(range(by * s + lx, by * s + hx + 1)) @@ -5822,8 +5868,8 @@ def test_box2d_commit_skips_post_with_two_clauses( ex = tuple(by_col["a"]["range"]) ey = tuple(by_col["b"]["range"]) - # The edges are the kernel's bin boundaries of the P=128 grid over the - # full domains (_reference_box2d_hist_counts checks them). + # The edges are the kernel's cell boundaries of the source's grid over + # the full domains (_reference_box2d_hist_counts checks them). expected = _reference_box2d_hist_counts(df, ex, ey) assert 0 < sum(expected) < df.height @@ -6690,6 +6736,10 @@ def test_header_guard_rejects_another_traces_grid( const one = (p, d, hp, dom, kind = 'continuous') => fvCubeHeaderMatchesKey( key({c: 'a', k: 'continuous', p, d}), {free: {kind, p: hp, domain: dom}} ); + const two = (p, d, hp, doms) => fvCubeHeaderMatchesKey( + key({c: ['a', 'b'], k: 'box2d', p, d}), + {free: {kind: 'box2d', cols: ['a', 'b'], p: hp, domains: doms}} + ); return { zoomedLattice: one(16, [10, 80], 17, [8.75, 83.125]), zoomedOtherTrace: one(16, [10, 80], 2048, [10, 80.0000000001]), @@ -6698,6 +6748,9 @@ def test_header_guard_rejects_another_traces_grid( zoomedTemporal: one(16, [1, 2], 17, [5, 6], 'temporal'), unzoomed: one(16, null, 16, [0, 99.9]), unzoomedMore: one(16, null, 17, [0, 99.9]), + box2dZoomedX: two([8, 6], [[10, 80], null], [9, 6], [[8.8, 88], [0, 99]]), + box2dOtherTrace: two([8, 6], [[10, 80], null], [2048, 6], [[10, 80], [0, 99]]), + box2dUnzoomedYMore: two([8, 6], [[10, 80], null], [9, 7], [[8.8, 88], [0, 99]]), }; }""") assert got == { @@ -6708,6 +6761,9 @@ def test_header_guard_rejects_another_traces_grid( "zoomedTemporal": True, "unzoomed": True, "unzoomedMore": False, + "box2dZoomedX": True, + "box2dOtherTrace": False, + "box2dUnzoomedYMore": False, } @pytest.mark.parametrize("line_first", [False, True]) @@ -6876,6 +6932,76 @@ def test_constant_column_brush_selects_every_row( assert sum(_target_y(page)) == 10 +def _box2d_zoomed_url(port: int, source_name: str, zoom: dict) -> str: + from flexviz.dashboard import Dashboard + from flexviz.server import register_source + from flexviz.spec import AxisRange, LayoutSpec, encode_spec + + df = _box2d_df() + register_source(source_name, df, cache=True) + dash = Dashboard(df) + dash.add_figure(title="Box2dSource").add_histogram2d( + x="a", y="b", x_bins=_BOX2D_NX, y_bins=_BOX2D_NY + ) + dash.add_figure(title="Hist").add_histogram(x="c", bins=_TGT_BINS) + spec = dash.to_spec(source_name=source_name, layout=LayoutSpec(draggable=False)) + spec.client_state.live_brush = "auto" + for axis, (lo, hi) in zoom.items(): + spec.state.viewport[f"{spec.figures[0].uid}/{axis}"] = AxisRange(min=lo, max=hi) + return f"http://127.0.0.1:{port}/view?spec={encode_spec(spec)}&renderer=plotly" + + +class TestBox2dZoomedSource: + def test_zoomed_box2d_adopts_the_snapped_cell_grid( + self, page: Page, server_port: int + ): + """A zoomed hist2d source: the header grid is the viewport snapped to + the cell lattice (one cell more per axis here), the gesture adopts it + per axis, the store accepts it (local commit), and the commit edges + are cell boundaries of that grid.""" + zoom = {"x": (10.3, 80.7), "y": (5.1, 70.9)} + url = _box2d_zoomed_url(server_port, "_cube_box2d_zoomed", zoom) + bodies = _capture_updates(page) + page.goto(url) + _wait_for_init(page, "plotly") + _enter_select_mode(page) + hist_before = _target_y(page) + coords = _box2d_drag_coords(page) + n_before = len(bodies) + + page.mouse.move(coords["x1"], coords["y1"]) + page.mouse.down() + page.mouse.move(coords["x2"], coords["y2"], steps=8) + page.wait_for_function( + """(yBefore) => { + const gd = document.querySelector('#fv-plot-1'); + const ys = Array.from((gd.data && gd.data[0] && gd.data[0].y) || []); + return ys.length > 0 && JSON.stringify(ys) !== JSON.stringify(yBefore); + }""", + arg=hist_before, + timeout=10_000, + ) + axes = page.evaluate( + "() => Object.values(_fvCubeGestures)[0].axes.map(a => [a.p, a.snapDomain])" + ) + grids = [] + for (lo_v, hi_v), n in ((zoom["x"], _BOX2D_NX), (zoom["y"], _BOX2D_NY)): + lo, hi, cells = snap_range(lo_v, hi_v, n) + assert cells == n + 1 + grids.append((lo, hi, cells)) + assert axes == [[g[2], [g[0], g[1]]] for g in grids] + page.mouse.up() + page.wait_for_timeout(800) + + types = [b.get("event", {}).get("type") for b in bodies[n_before:]] + assert types == ["cube_request"], types # stored, live, local commit + clauses = page.evaluate("DASHBOARD_SPEC.state.selections")[0]["predicates"][0][ + "clauses" + ] + for clause, grid in zip(clauses, grids): + _committed_bins(*clause["range"], grid) + + def _short_date_df() -> pl.DataFrame: """Seven days in 16 bars: bars 0.375 day wide, edges between the days.""" import datetime as dt diff --git a/tests/test_cube.py b/tests/test_cube.py index 574a4ade..8df525f2 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -2032,7 +2032,7 @@ def test_rejects_box2d_free_axis(self, line_df): column="free", columns=("free", "x"), kind="box2d", - p=8, + p=(8, 8), domains=((0.0, 100.0), (0.0, 100.0)), ), target_dims=( @@ -2637,7 +2637,7 @@ def test_build_rejects_box2d_free_axis(self, corr_df): column="free", columns=("free", "x"), kind="box2d", - p=8, + p=(8, 8), domains=((0.0, 100.0), (0.0, 50.0)), ), target_dims=(), @@ -2991,7 +2991,7 @@ def test_non_corr_key_byte_identical_to_before(self): # --------------------------------------------------------------------------- -_BOX2D_PX = _BOX2D_PY = 128 +_BOX2D_PX, _BOX2D_PY = 128, 64 _BOX2D_S = _BOX2D_PX @@ -3000,7 +3000,7 @@ def _box2d_spec( value_col: str | None = None, x_domain: tuple[float, float] = (0.0, 100.0), y_domain: tuple[float, float] = (0.0, 50.0), - p: int = _BOX2D_PX, + p: tuple[int, int] = (_BOX2D_PX, _BOX2D_PY), ) -> CubeSpec: return CubeSpec( source_name="s", @@ -3051,6 +3051,16 @@ def test_box2d_requires_two_columns(self): column="x", kind="box2d", columns=("x",), domains=((0, 1), (0, 1)) ) + def test_box2d_requires_per_axis_p(self): + with pytest.raises(ValueError): + FreeAxisSpec( + column="x", + kind="box2d", + p=128, + columns=("x", "y"), + domains=((0, 1), (0, 1)), + ) + def test_box2d_columns0_must_equal_column(self): with pytest.raises(ValueError): FreeAxisSpec( @@ -3077,6 +3087,10 @@ def test_categorical_rejects_domains(self): column="x", kind="categorical", columns=("x",), domains=((0, 1), (0, 1)) ) + def test_range_kinds_require_int_p(self): + with pytest.raises(ValueError): + FreeAxisSpec(column="x", kind="continuous", p=(8, 8), domain=(0, 1)) + class TestBox2dBuildAndSlice: @pytest.mark.parametrize("agg", ["count", "mean"]) @@ -3171,7 +3185,7 @@ def _reslice_rect(self, blob: bytes, cube: CubeResult, x: tuple, y: tuple) -> in free_bin = _read_u32_col(blob, header, "free_bin") count = _read_u32_col(blob, header, "count") (lx, hx), (ly, hy) = cube._snap_box2d(x[0], x[1], y[0], y[1]) - s = header["free"]["p"] + s = header["free"]["p"][0] codes = set() for by in range(ly, hy + 1): row = by * s @@ -3191,7 +3205,7 @@ def test_header_free_block_shape(self, box2d_df): assert header["free"] == { "kind": "box2d", "cols": ["x", "y"], - "p": 128, + "p": [128, 64], "domains": [[0.0, 100.0], [0.0, 50.0]], } # No target dims for a pure source build; the free_bin and count buffers diff --git a/tests/test_cube_server.py b/tests/test_cube_server.py index a5b45cc5..38b8f8bc 100644 --- a/tests/test_cube_server.py +++ b/tests/test_cube_server.py @@ -1936,7 +1936,7 @@ def _box2d_reslice_count(self, blob, header, x_box, y_box, df): rectangle edges.""" dx = tuple(header["free"]["domains"][0]) dy = tuple(header["free"]["domains"][1]) - px = py = header["free"]["p"] + px, py = header["free"]["p"] lx, hx, ex0, ex1, cx = _snap_axis(dx, x_box[0], x_box[1], px) ly, hy, ey0, ey1, cy = _snap_axis(dy, y_box[0], y_box[1], py) s = px @@ -1978,7 +1978,7 @@ def test_serves_hist_target_with_box2d_header(self, client, df): assert header["free"] == { "kind": "box2d", "cols": ["a", "b"], - "p": 128, + "p": [8, 6], "domains": [[df["a"].min(), df["a"].max()], [df["b"].min(), df["b"].max()]], } @@ -2029,13 +2029,85 @@ def test_zoomed_viewports_resolve_both_axes(self, client, df): body = _cube_body(client.post("/dashboard/update", json=payload)) blob = base64.b64decode(body["cubes"][0]) header = decode_fvcube_header(blob) - assert header["free"]["domains"] == [[10.0, 80.0], [5.0, 70.0]] + # Each zoomed axis snaps to the display lattice of its 8 / 6 cells. + x_lo, x_hi, nx = snap_range(10.0, 80.0, 8) + y_lo, y_hi, ny = snap_range(5.0, 70.0, 6) + assert header["free"]["domains"] == [[x_lo, x_hi], [y_lo, y_hi]] + assert header["free"]["p"] == [nx, ny] sliced, direct = self._box2d_reslice_count( blob, header, (20.0, 65.0), (15.0, 55.0), df ) assert 0 < sum(sliced) < df.height assert sliced == direct + @pytest.mark.parametrize("zoomed", [False, True]) + def test_one_cell_brush_commits_exactly_that_cell(self, zoomed): + """#133: the box2d free axis is the source's own cell grid, so a brush + inside one cell of a 200x150 hist2d commits that cell's edges and + filters exactly its rows. A fixed 128-bin grid is coarser than one + cell and never lands on a cell edge.""" + n = 60_000 + # Low-discrepancy points: dense, deterministic, off any bin lattice. + df = pl.DataFrame( + { + "a": [(i * 0.6180339887498949) % 1.0 * 100.0 for i in range(n)], + "b": [(i * 0.4142135623730951) % 1.0 * 100.0 for i in range(n)], + "c": [float(i % 7) for i in range(n)], + } + ) + register_source(_SRC, df, cache=True) + get_cache().clear() + get_cube_cache().clear() + client = TestClient(app) + dash = Dashboard(df) + dash.add_figure(title="Cells").add_histogram2d( + x="a", y="b", x_bins=200, y_bins=150 + ) + dash.add_figure(title="Hist").add_histogram(x="c", bins=7) + spec = dash.to_spec(source_name=_SRC) + src_fig = spec.figures[0] + if zoomed: + spec.state.viewport[f"{src_fig.uid}/x"] = AxisRange(min=10.3, max=80.7) + spec.state.viewport[f"{src_fig.uid}/y"] = AxisRange(min=5.1, max=70.9) + + (delta,) = client.post("/dashboard/update", json=_init_payload(spec)).json()[ + "figure_deltas" + ][src_fig.uid] + x_lo, x_step, nx = delta["updates"]["x_edges"] + y_lo, y_step, ny = delta["updates"]["y_edges"] + z = delta["updates"]["z"] + + payload = _cube_payload( + spec, src_fig.uid, column="a", trace_uid=src_fig.traces[0].uid + ) + blob = base64.b64decode( + _cube_body(client.post("/dashboard/update", json=payload))["cubes"][0] + ) + header = decode_fvcube_header(blob) + free = header["free"] + assert free["p"] == [nx, ny] + assert free["domains"][0] == pytest.approx([x_lo, x_lo + nx * x_step]) + assert free["domains"][1] == pytest.approx([y_lo, y_lo + ny * y_step]) + + count, kx, ky = max( + (v, i, j) for j, row in enumerate(z) for i, v in enumerate(row) if v + ) + x_box = (x_lo + (kx + 0.25) * x_step, x_lo + (kx + 0.75) * x_step) + y_box = (y_lo + (ky + 0.25) * y_step, y_lo + (ky + 0.75) * y_step) + *_, ex0, ex1, _cx = _snap_axis(tuple(free["domains"][0]), *x_box, nx) + *_, ey0, ey1, _cy = _snap_axis(tuple(free["domains"][1]), *y_box, ny) + assert (ex0, ex1) == pytest.approx( + (x_lo + kx * x_step, x_lo + (kx + 1) * x_step) + ) + assert (ey0, ey1) == pytest.approx( + (y_lo + ky * y_step, y_lo + (ky + 1) * y_step) + ) + sliced, direct = self._box2d_reslice_count(blob, header, x_box, y_box, df) + assert sliced == direct + assert sum(sliced) == count + get_cache().clear() + get_cube_cache().clear() + def test_passive_selection_bakes_into_build(self, client, df): """Contract E: a foreign committed selection bakes into the box2d build frame; the reslice equals the legacy 3-clause recompute (passive @@ -2072,7 +2144,7 @@ def test_passive_selection_bakes_into_build(self, client, df): ] dx = tuple(header["free"]["domains"][0]) dy = tuple(header["free"]["domains"][1]) - px = py = header["free"]["p"] + px, py = header["free"]["p"] _lx, _hx, ex0, ex1, _cx = _snap_axis(dx, 30.0, 60.0, px) _ly, _hy, ey0, ey1, _cy = _snap_axis(dy, 10.0, 50.0, py) passive_expr = pl.col("a").is_between(20.0, 70.0, closed="left") @@ -3060,6 +3132,67 @@ def _sliced_rows(self, lo_bin: int, hi_bin: int) -> int: count = _read_u32_col(self.blob, header, "count") return sum(n for fb, n in zip(free_bin, count) if lo_bin <= fb <= hi_bin) + def test_hist2d_integer_values_on_cell_edges(self): + # 0..100 on both axes with 20 cells: a value on every cell edge. + n = 101 + df = pl.DataFrame( + { + "a": [float(i % n) for i in range(n * n)], + "b": [float(i // n) for i in range(n * n)], + "c": [float(i % 7) for i in range(n * n)], + } + ) + dash = Dashboard(df) + dash.add_figure(title="Cells").add_histogram2d( + x="a", y="b", x_bins=20, y_bins=20 + ) + dash.add_figure(title="Hist").add_histogram(x="c", bins=7) + updates, free = self._grid(df, dash.to_spec(source_name=_GRID_SRC), "a") + z = updates["z"] + (dx, dy), (px, py) = free["domains"], free["p"] + x_step, y_step = (dx[1] - dx[0]) / px, (dy[1] - dy[0]) / py + brushes = [(0, 0, 0, 0), (3, 3, 7, 9), (5, 12, 0, 19), (19, 19, 4, 19)] + for kx, mx, ky, my in brushes: + sx = snap_brush(dx[0], dx[1], px, *_covering(dx[0], x_step, kx, mx)) + sy = snap_brush(dy[0], dy[1], py, *_covering(dy[0], y_step, ky, my)) + assert (sx[0], sx[1], sy[0], sy[1]) == (kx, mx, ky, my) + want = sum( + z[j][i] or 0 for j in range(ky, my + 1) for i in range(kx, mx + 1) + ) + clauses = [_commit_clause("a", sx), _commit_clause("b", sy)] + assert _committed_rows(df, clauses) == want, (kx, mx, ky, my) + + def test_hist2d_date_axis_commits_whole_days(self): + # 365 days in 20 cells: 18.25-day cells with fractional-day edges. + import datetime as dt + + n = 365 * 8 + df = pl.DataFrame( + { + "d": [ + dt.date(2021, 1, 1) + dt.timedelta(days=i % 365) for i in range(n) + ], + "b": [float(i // 365) for i in range(n)], + "c": [float(i % 7) for i in range(n)], + } + ) + dash = Dashboard(df) + dash.add_figure(title="Cells").add_histogram2d( + x="d", y="b", x_bins=20, y_bins=4 + ) + dash.add_figure(title="Hist").add_histogram(x="c", bins=7) + updates, free = self._grid(df, dash.to_spec(source_name=_GRID_SRC), "d") + z = updates["z"] + (dx, dy), (px, py) = free["domains"], free["p"] + assert free["units"] == ["day", None] + step = (dx[1] - dx[0]) / px + sy = snap_brush(dy[0], dy[1], py, dy[0], dy[1]) + for kx, mx in [(k, k) for k in range(px)] + [(3, px - 1)]: + sx = snap_brush(dx[0], dx[1], px, *_covering(dx[0], step, kx, mx)) + want = sum(z[j][i] or 0 for j in range(py) for i in range(kx, mx + 1)) + clauses = [_commit_clause("d", sx, "day"), _commit_clause("b", sy)] + assert _committed_rows(df, clauses) == want, (kx, mx) + # An Int64 column rounds each committed bound inward (_typed_range_bounds). @pytest.mark.parametrize("dtype", [pl.Float64, pl.Int64]) @pytest.mark.parametrize("viewport", [None, {"x": (10.3, 80.7)}]) diff --git a/tests/test_trace_hist2d.py b/tests/test_trace_hist2d.py index 31fa8724..869b2521 100644 --- a/tests/test_trace_hist2d.py +++ b/tests/test_trace_hist2d.py @@ -993,3 +993,31 @@ def test_collect_batches_chunking_semantics(): assert len(batches) == math.ceil(n / chunk) assert sum(b.height for b in batches) == n assert sorted(v for b in batches for v in b["a"].to_list()) == values + + +class TestCubeSourceSpec: + """A hist2d source's free axis is its own cell grid (#133).""" + + def test_unzoomed_is_the_nominal_cell_grid(self): + from flexviz.trace.hist2d import Histogram2D + + spec = Histogram2D(x="a", y="b", x_bins=200, y_bins=150).get_cube_source_spec( + None + ) + assert spec.kind == "box2d" + assert spec.p == (200, 150) + assert spec.domains is None + + def test_zoomed_axes_snap_to_the_cell_lattice(self): + from flexviz.trace.bin_grid import snap_range + from flexviz.trace.hist2d import Histogram2D + + trace = Histogram2D(x="a", y="b", x_bins=8, y_bins=6) + spec = trace.get_cube_source_spec((10.3, 80.7), y_range=None) + lo, hi, n = snap_range(10.3, 80.7, 8) + assert spec.p == (n, 6) == (9, 6) + assert spec.domains == ((lo, hi), None) + spec = trace.get_cube_source_spec(None, y_range=(5.1, 70.9)) + lo, hi, n = snap_range(5.1, 70.9, 6) + assert spec.p == (8, n) == (8, 7) + assert spec.domains == (None, (lo, hi)) From 0a184466c954c5b2e88591dd83f0a2cb7339a5a3 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 16:04:12 +0200 Subject: [PATCH 6/8] fix(trace): round Float32 range bounds without moving a value across them --- CHANGELOG.md | 3 +++ flexviz/trace/base.py | 21 +++++++++++++++++++++ tests/test_cube.py | 16 +++++++++++++++- tests/test_predicates.py | 16 ++++++++++++++++ tests/test_trace_base.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d3f07a..449840b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,9 @@ independently. `flexviz` pins a compatible `flexviz-polars` range. center, and its hover shows the time of day of that center. Before, the center was rounded to a whole day, so a bar could sit up to half a day off its bin, and bars narrower than a day stacked on one day. +- A selection or a zoom on a `Float32` column keeps exactly the values inside + its range. Before, each bound was rounded to the nearest `Float32`, which + could move a value next to the bound to the other side. ## [0.1.0b5] - 2026-09-27 diff --git a/flexviz/trace/base.py b/flexviz/trace/base.py index 96766748..f84cd993 100644 --- a/flexviz/trace/base.py +++ b/flexviz/trace/base.py @@ -22,6 +22,7 @@ from typing import Any, ClassVar, get_args from uuid import uuid4 +import numpy as np import polars as pl from ..cube import CubeTargetSpec, FreeAxisSpec, MeasureAgg, MeasureSpec, temporal_unit @@ -777,11 +778,31 @@ def _typed_range_bounds( pl.lit(lo).cast(dtype, strict=False), pl.lit(hi).cast(dtype, strict=False), ) + if dtype == pl.Float32: + # Round like an integer bound, so no Float32 value crosses it: a + # closed bound toward the interior, an open bound away from it. The + # nearest Float32 can land on the far side of a value next to the bound. + # A whole number arrives from JSON as an int; above 2**24 it rounds too. + lo_closed = closed in ("both", "left") + hi_closed = closed in ("both", "right") + if isinstance(lo, (int, float)): + lo = _float32_toward(lo, up=lo_closed) + if isinstance(hi, (int, float)): + hi = _float32_toward(hi, up=not hi_closed) # Cast to match column dtype (e.g. f32) to avoid implicit column upcast # to f64 (measured ~5x slower at 5M rows). return (pl.lit(lo).cast(dtype, strict=False), pl.lit(hi).cast(dtype, strict=False)) +def _float32_toward(x: float, up: bool) -> float: + """The nearest Float32 at or above ``x`` (``up``), or at or below it.""" + # float(): NumPy would compare a Float32 with x cast to Float32 (NEP 50). + f = np.float32(x) + if (float(f) < x) if up else (float(f) > x): + f = np.nextafter(f, np.float32(np.inf if up else -np.inf)) + return float(f) + + def _range_filter_expr( col_name: str, range_: tuple[Any, Any] | None, diff --git a/tests/test_cube.py b/tests/test_cube.py index 8df525f2..6ca309ff 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -349,7 +349,7 @@ def kernel_bin(v: float) -> int: assert kernel_bin(e_lo) == 10 assert kernel_bin(math.nextafter(e_lo, -math.inf)) == 9 - @pytest.mark.parametrize("dtype", [pl.Float64, pl.Int64]) + @pytest.mark.parametrize("dtype", [pl.Float64, pl.Float32, pl.Int64]) def test_committed_edges_keep_the_kernel_rows(self, dtype): # Values packed around every edge, on the column's own value grid: a # committed range keeps exactly the rows the kernel counts in its bars. @@ -368,12 +368,26 @@ def test_committed_edges_keep_the_kernel_rows(self, dtype): if dtype == pl.Int64: lo, span = float(math.floor(lo)), float(math.floor(span) + p) hi = lo + span + if dtype == pl.Float32: + lo, hi = float(np.float32(lo)), float(np.float32(hi)) + if not hi > lo: + continue step = (hi - lo) / p vals = [lo, hi] for k in range(1, min(p, 50)): for e in (lo + k * step, lo + (k - 1e-9) * step): if dtype == pl.Int64: vals += [float(math.floor(e) + d) for d in (-1, 0, 1)] + elif dtype == pl.Float32: + f = np.float32(e) + vals += [ + float(v) + for v in ( + f, + np.nextafter(f, np.float32(-np.inf)), + np.nextafter(f, np.float32(np.inf)), + ) + ] else: vals += [ e, diff --git a/tests/test_predicates.py b/tests/test_predicates.py index a8fb5507..4efbfffa 100644 --- a/tests/test_predicates.py +++ b/tests/test_predicates.py @@ -26,6 +26,22 @@ def test_empty_returns_true(self, df: pl.DataFrame): expr = predicates_to_expr([], df.schema) assert df.filter(expr).height == df.height + def test_float32_whole_number_bound_from_json_rounds(self): + # The browser sends a whole-number edge as a JSON integer. Above 2**24 + # a Float32 column cannot hold every integer, so that bound must round + # without moving a value across it too. + from flexviz.predicates import predicates_to_expr + + df = pl.DataFrame( + {"a": pl.Series([16777216.0 + 2 * i for i in range(5)], dtype=pl.Float32)} + ) + pred = SelectionPredicate.model_validate_json( + '{"clauses": [{"column": "a", "range": [16777217, 16777219],' + ' "closed": "left"}]}' + ) + got = df.filter(predicates_to_expr([pred], df.schema))["a"].to_list() + assert got == [16777218.0] + def test_single_categorical_clause(self, df: pl.DataFrame): from flexviz.predicates import predicates_to_expr diff --git a/tests/test_trace_base.py b/tests/test_trace_base.py index f7036876..f42e2b01 100644 --- a/tests/test_trace_base.py +++ b/tests/test_trace_base.py @@ -214,6 +214,36 @@ def test_naive_datetime_epoch_ms_bounds_filter(self): def test_none_range_returns_none(self): assert _typed_range_bounds("x", None, schema=None) is None + # A whole-number bound arrives from JSON as an int: above 2**24 it sits + # between two Float32 values as well. + @pytest.mark.parametrize("bounds", [(100 / 7, 600 / 7), (16777217, 16777219)]) + @pytest.mark.parametrize("closed", ["both", "left", "right", "none"]) + def test_float32_bounds_keep_the_real_membership(self, closed, bounds): + """A bound between two Float32 values: the nearest Float32 can land + on the far side of the value next to it. Each side must select the + Float32 values that the real bound selects.""" + import numpy as np + + lo, hi = bounds + vals = [] + for bound in (lo, hi): + f = np.float32(bound) + vals += [ + np.nextafter(f, np.float32(-np.inf)), + f, + np.nextafter(f, np.float32(np.inf)), + ] + df = pl.DataFrame({"x": pl.Series([float(v) for v in vals], dtype=pl.Float32)}) + b_lo, b_hi = _typed_range_bounds("x", (lo, hi), df.schema, closed) + got = df.select(pl.col("x").is_between(b_lo, b_hi, closed=closed))["x"] + in_lo = ( + (lambda v: v >= lo) if closed in ("both", "left") else (lambda v: v > lo) + ) + in_hi = ( + (lambda v: v <= hi) if closed in ("both", "right") else (lambda v: v < hi) + ) + assert got.to_list() == [in_lo(v) and in_hi(v) for v in df["x"].to_list()] + def test_float_bounds_on_int64_column(self): """Float bounds use ceil(lo)/floor(hi) so that is_between matches the viewport. lo=99.9 → ceil → 100, hi=100.1 → floor → 100: only x=100 included. From 7e2520271c03e78c2b1ce2dca06d94dead3bfce5 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 21:07:48 +0200 Subject: [PATCH 7/8] fix(predicates): round temporal selection bounds without moving a value across them --- Architecture.md | 2 ++ CHANGELOG.md | 4 ++++ flexviz/predicates.py | 19 ++++++++++++++-- flexviz/trace/base.py | 21 ++++++++++++++++++ tests/test_predicates.py | 48 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/Architecture.md b/Architecture.md index b04c1106..ecc9e580 100644 --- a/Architecture.md +++ b/Architecture.md @@ -725,6 +725,8 @@ _range_filter_expr(col, lo, hi, schema) → pl.Expr # is_between with typed lit All traces call these for consistent datetime, integer, and float casting in `is_between` expressions. +`_typed_range_bounds` rounds an integer or `Float32` bound so that no value of the column crosses it: a closed bound toward the interior of the range, an open bound away from it. A selection predicate applies the same rule to a temporal bound that is finer than its column (`_temporal_bound_toward`, for `Date` and `Datetime("ms")`), so a brush from 13:00 does not keep that day. A viewport mask still truncates a temporal bound: rounding there would change which points a line keeps at the edge of a zoom. + --- ## Plugin Layer diff --git a/CHANGELOG.md b/CHANGELOG.md index 449840b3..ed8e3f86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,10 @@ independently. `flexviz` pins a compatible `flexviz-polars` range. - A selection or a zoom on a `Float32` column keeps exactly the values inside its range. Before, each bound was rounded to the nearest `Float32`, which could move a value next to the bound to the other side. +- A selection on a `Date` or `Datetime("ms")` column keeps exactly the values + inside its range. Before, a bound with a time of day (or a fraction of a + millisecond) was cut to the day (or millisecond), so a brush that started at + 13:00 still kept that day. ## [0.1.0b5] - 2026-09-27 diff --git a/flexviz/predicates.py b/flexviz/predicates.py index 1cc4c675..94899884 100644 --- a/flexviz/predicates.py +++ b/flexviz/predicates.py @@ -20,7 +20,7 @@ import polars as pl from .spec import ClauseFilter, SelectionPredicate, SelectionState -from .trace.base import _dtype_for_col, _typed_range_bounds +from .trace.base import _dtype_for_col, _temporal_bound_toward, _typed_range_bounds def _values_to_typed_series( @@ -77,7 +77,22 @@ def _clause_to_expr( ) return pl.col(clause.column).is_in(typed.implode()) - bounds = _typed_range_bounds(clause.column, clause.range, schema, clause.closed) + dtype = _dtype_for_col(schema, clause.column) + if clause.range is not None and dtype is not None and dtype.is_temporal(): + # A selection keeps exactly the instants in its range: each bound + # rounds to a whole unit of the column without moving a value across + # it, like an integer bound. A viewport mask still truncates + # (_typed_range_bounds): rounding there would change which points a + # line keeps at the edge of a zoom. + lo, hi = clause.range + bounds = ( + _temporal_bound_toward(lo, dtype, up=clause.closed in ("both", "left")), + _temporal_bound_toward( + hi, dtype, up=clause.closed not in ("both", "right") + ), + ) + else: + bounds = _typed_range_bounds(clause.column, clause.range, schema, clause.closed) if bounds is None: # range was None — should never happen because the model_validator # rejects it, but treat as a no-op for safety. diff --git a/flexviz/trace/base.py b/flexviz/trace/base.py index f84cd993..7daf095c 100644 --- a/flexviz/trace/base.py +++ b/flexviz/trace/base.py @@ -680,6 +680,27 @@ def _typed_temporal_lit(value: Any, dtype: pl.DataType | None) -> pl.Expr: return pl.lit(value, dtype=dtype) +def _temporal_bound_toward(value: Any, dtype: pl.DataType, up: bool) -> pl.Expr: + """A range bound as a whole unit of a ``Date`` or ``Datetime("ms")`` + column, rounded so no value of the column crosses it: ``up`` to the first + unit at or after the bound, else the last unit at or before it. + + ``_typed_temporal_lit`` truncates a finer bound, so a bound at 13:00 would + keep that day's value at 00:00. A bound carries at most microseconds, so a + finer column takes it as it is. + """ + if dtype == pl.Date: + unit_us = 86_400_000_000 + elif isinstance(dtype, pl.Datetime) and dtype.time_unit == "ms": + unit_us = 1_000 + else: + return _typed_temporal_lit(value, dtype) + tz = getattr(dtype, "time_zone", None) + us = _typed_temporal_lit(value, pl.Datetime("us", tz)).to_physical() + units = -(-us // unit_us) if up else us // unit_us + return units.cast(pl.Int32 if dtype == pl.Date else pl.Int64).cast(dtype) + + # --------------------------------------------------------------------------- # Temporal ↔ physical helpers for the numeric histogram kernels. # diff --git a/tests/test_predicates.py b/tests/test_predicates.py index 4efbfffa..e0d020f7 100644 --- a/tests/test_predicates.py +++ b/tests/test_predicates.py @@ -329,6 +329,54 @@ def test_viewport_range_filter_keeps_closed_both_rounding(self): assert df.filter(expr)["i"].to_list() == [1, 2, 3] +class TestTemporalBoundRounding: + """A temporal bound finer than its column rounds to a whole unit without + moving a value across it: a closed bound toward the interior of the range, + an open bound away from it, as for an integer column.""" + + @staticmethod + def _select(df: pl.DataFrame, rng: list, closed: str) -> list: + from flexviz.predicates import predicates_to_expr + + pred = SelectionPredicate.model_validate( + {"clauses": [{"column": "t", "range": rng, "closed": closed}]} + ) + return df.filter(predicates_to_expr([pred], df.schema))["t"].to_list() + + @pytest.mark.parametrize( + "rng,closed,days", + [ + # Jan 1 is at 00:00, before a bound at 13:00. + (["2020-01-01 13:00:00", "2020-01-05 07:00:00"], "both", [2, 3, 4, 5]), + ( + ["2020-01-01 22:30:00.000000", "2020-01-04 07:30:00.000000"], + "left", + [2, 3, 4], + ), + (["2020-01-02", "2020-01-04"], "both", [2, 3, 4]), + (["2020-01-02", "2020-01-04"], "left", [2, 3]), + # Epoch-ms bounds at noon on Jan 1 and Jan 3. + ([1577880000000, 1578052800000], "both", [2, 3]), + ], + ) + def test_date_column(self, rng, closed, days): + import datetime as dt + + df = pl.DataFrame({"t": [dt.date(2020, 1, d) for d in range(1, 7)]}) + assert self._select(df, rng, closed) == [dt.date(2020, 1, d) for d in days] + + def test_millisecond_column(self): + import datetime as dt + + base = dt.datetime(2020, 1, 1) + stamps = [base + dt.timedelta(milliseconds=i) for i in range(4)] + df = pl.DataFrame({"t": pl.Series(stamps, dtype=pl.Datetime("ms"))}) + got = self._select( + df, ["2020-01-01 00:00:00.000500", "2020-01-01 00:00:00.002500"], "left" + ) + assert got == stamps[1:3] + + class TestBooleanCoercion: def test_string_true_false_coerced_to_bool(self): from flexviz.predicates import predicates_to_expr From 0b51914b99b057b72c47765aa688597f56b676b2 Mon Sep 17 00:00:00 2001 From: Jeroen Van Der Donckt Date: Wed, 30 Sep 2026 21:15:18 +0200 Subject: [PATCH 8/8] fix(cube): commit Date bar edges so a restored selection box keeps them --- Architecture.md | 16 +++++---- flexviz/adapters/js/plotly/events.js | 49 ++++++++++++---------------- tests/test_browser_cube.py | 39 ++++++++++++++++------ tests/test_cube.py | 17 +++++++--- tests/test_cube_server.py | 13 ++++---- 5 files changed, 78 insertions(+), 56 deletions(-) diff --git a/Architecture.md b/Architecture.md index ecc9e580..d7e233bc 100644 --- a/Architecture.md +++ b/Architecture.md @@ -1314,13 +1314,15 @@ viewports key as self-consistent epoch-ms tokens (never sent to the server — t the original date strings via the schema dtype in `_cube_axis_range`); the snap grid is adopted from the decoded header; drag ranges convert through `fvTemporalToPhysical` (manual UTC parse, never bare `Date.parse`); commits emit snapped **string** ranges rendered by -`fvPhysicalToTemporal`, which ceils: the ceil of an exact edge is the first whole unit the -kernel puts in its bin, and a closed upper edge (the top bin) rounds down to the last unit. So -an integer row stays in the bar the display counts it in, and the string parses back exactly -through `_typed_range_bounds`. The selection box keeps each -bin edge's time of day instead (a µs string): a whole-day edge can pass the center of a bar -narrower than two days. The `plotly_selected` echo guard converts both sides to physical before -its half-bin comparison (half the source step). +`fvPhysicalToTemporal`, which ceils (`_fvCubeCommitEdges`). On a ms or µs axis an edge is a +whole unit: the ceil of an exact edge is the first unit the kernel puts in its bin, and a closed +upper edge (the top bin) rounds down to the last unit. On a Date axis an edge keeps its time of +day (a µs string), because a whole-day edge can pass the center of a bar narrower than two days; +the server rounds each Date bound of a selection to whole days without moving a day across it +(`_temporal_bound_toward`). So an integer row stays in the bar the display counts it in. The +stored selection box sits on the committed edges, so a restore, which rebuilds the box from the +predicate, draws the same box. The `plotly_selected` echo guard converts both sides to physical +before its half-bin comparison (half the source step). **Temporal binned *target* dims.** A binned target dim over a temporal column is built on the column's physical representation (epoch µs/ms, day index) and the header ships its `unit`. Most diff --git a/flexviz/adapters/js/plotly/events.js b/flexviz/adapters/js/plotly/events.js index c7f1dd2f..223a5ad6 100644 --- a/flexviz/adapters/js/plotly/events.js +++ b/flexviz/adapters/js/plotly/events.js @@ -1552,8 +1552,7 @@ function _fvCubeCommitOverride(figUid, range, box) { if (!Array.isArray(r) || r.length !== 2) return null; // Temporal sources: the drag range arrives as date strings — convert to // the cube's physical unit, snap, then render the snapped edges back as - // strings (contract G; the server's _typed_range_bounds parses them back - // to the exact integral-unit bounds). + // strings (contract G, _fvCubeCommitEdges). if (gesture.unit) r = r.map(v => fvTemporalToPhysical(v, gesture.unit)); if (!r.every(Number.isFinite)) return null; const snap = _fvCubeGestureSnap(gesture, r[0], r[1]); @@ -1566,15 +1565,14 @@ function _fvCubeCommitOverride(figUid, range, box) { closed: snap.closed, }], }]; - const [boxLo, boxHi] = _fvCubeBoxEdges(snap, gesture.unit); const snappedBox = { ...(box || {}) }; if (gesture.role === 'x') { - snappedBox.x0 = boxLo; - snappedBox.x1 = boxHi; + snappedBox.x0 = edgeLoOut; + snappedBox.x1 = edgeHiOut; snappedBox.xref = snappedBox.xref || 'x'; } else { - snappedBox.y0 = boxLo; - snappedBox.y1 = boxHi; + snappedBox.y0 = edgeLoOut; + snappedBox.y1 = edgeHiOut; snappedBox.yref = snappedBox.yref || 'y'; } if (gesture.live) { @@ -1585,27 +1583,24 @@ function _fvCubeCommitOverride(figUid, range, box) { return { predicates, box: snappedBox, skipPost: _fvCubeSkipPost(gesture) }; } -// A snap's committed edges in the clause's value space. A temporal edge is a -// string of whole units: fvPhysicalToTemporal ceils, which gives the first -// unit in the bin; a closed upper edge (the top bin) rounds down instead. +// A snap's committed edges in the clause's value space. The stored selection +// box sits on them too, so a restored selection draws the same box. A temporal +// edge is a string that fvPhysicalToTemporal ceils. On a ms or µs axis it is a +// whole unit, and a closed upper edge (the top bin) rounds down. On a Date axis +// it keeps its time of day (µs): a whole-day edge can pass the center of a bar +// narrower than two days, and the server rounds each bound to whole days +// without moving a day across it (_temporal_bound_toward). function _fvCubeCommitEdges(snap, unit) { if (!unit) return [snap.edgeLo, snap.edgeHi]; + if (unit === 'day') { + return [snap.edgeLo, snap.edgeHi].map( + v => fvPhysicalToTemporal(fvPhysicalToEpochMs(v, 'day') * 1000, 'us') + ); + } const hi = snap.closed === 'both' ? Math.floor(snap.edgeHi) : snap.edgeHi; return [fvPhysicalToTemporal(snap.edgeLo, unit), fvPhysicalToTemporal(hi, unit)]; } -// A snap's edges for the stored and rendered selection box: the bin edges -// themselves, in the axis's value space. Plotly highlights a bar when its -// center is inside the box, so the highlight stays on the committed bins. A -// whole-day commit edge can pass the center of a bar that is less than two -// days wide, so a temporal box edge keeps its time of day (µs string). -function _fvCubeBoxEdges(snap, unit) { - if (!unit) return [snap.edgeLo, snap.edgeHi]; - return [snap.edgeLo, snap.edgeHi].map( - v => fvPhysicalToTemporal(fvPhysicalToEpochMs(v, unit) * 1000, 'us') - ); -} - // Conditional commit predicate (shared by every range/box2d gesture): local // only when EVERY trace in EVERY other figure was cube-served this gesture. // Any non-capable target (box plot, median bar, a server-demoted target) @@ -1625,8 +1620,8 @@ function _fvCubeSkipPost(gesture) { // Commit a box2d gesture (contract H): ONE predicate with TWO snapped // clauses (x and y) — today's two-clause hist2d selection shape, snapped to -// the cube grid. The stored/rendered box sits on the bin edges of both axes -// (_fvCubeBoxEdges). Conditional-commit skipPost is shared +// the cube grid. The stored/rendered box sits on the committed edges of both +// axes (_fvCubeCommitEdges). Conditional-commit skipPost is shared // (_fvCubeSkipPost). Returns null if either axis has no snap domain (cold // degrade) or the range is malformed. function _fvCubeBox2dCommitOverride(gesture, range, box) { @@ -1644,12 +1639,10 @@ function _fvCubeBox2dCommitOverride(gesture, range, box) { { column: gesture.cols[1], range: [eyLo, eyHi], closed: snap.y.closed }, ], }]; - const [bxLo, bxHi] = _fvCubeBoxEdges(snap.x, gesture.axes[0].unit); - const [byLo, byHi] = _fvCubeBoxEdges(snap.y, gesture.axes[1].unit); const snappedBox = { ...(box || {}), - x0: bxLo, x1: bxHi, xref: (box && box.xref) || 'x', - y0: byLo, y1: byHi, yref: (box && box.yref) || 'y', + x0: exLo, x1: exHi, xref: (box && box.xref) || 'x', + y0: eyLo, y1: eyHi, yref: (box && box.yref) || 'y', }; if (gesture.live) _fvCubeApplyBox2d(gesture, range); return { predicates, box: snappedBox, skipPost: _fvCubeSkipPost(gesture) }; diff --git a/tests/test_browser_cube.py b/tests/test_browser_cube.py index dd69c9c6..29f1822e 100644 --- a/tests/test_browser_cube.py +++ b/tests/test_browser_cube.py @@ -4187,9 +4187,9 @@ def test_zoomed_temporal_source_gesture(self, page: Page, server_port: int): assert lo >= dt.datetime(2020, 1, 5) assert hi <= dt.datetime(2020, 1, 25) - def test_date_source_commits_integer_day_edges(self, page: Page, server_port: int): - """A Date-typed source brush commits YYYY-MM-DD edges (whole days, - ceil-ed from the kernel's bar boundaries) with full parity.""" + def test_date_source_commits_bar_edges(self, page: Page, server_port: int): + """A Date-typed source brush commits the kernel's bar boundaries as µs + strings, which the server rounds to whole days, with full parity.""" df = _temporal_browser_df("date") url = _temporal_dashboard_url(server_port, "_cube_browser_tsday", kind="date") bodies = _capture_updates(page) @@ -4202,9 +4202,9 @@ def test_date_source_commits_integer_day_edges(self, page: Page, server_port: in types = [b["event"]["type"] for b in bodies[n0:]] assert types == ["cube_request"], types clause = _assert_temporal_commit_and_parity(page, df) - # Integer-day edges: plain dates, no time component. + # Bar edges keep their time of day: YYYY-MM-DD HH:MM:SS.ffffff. for v in clause["range"]: - assert len(v) == 10, v # YYYY-MM-DD + assert len(v) == 26, v # --------------------------------------------------------------------------- @@ -7045,10 +7045,12 @@ class TestDateHistSource: def test_box_on_bar_edges_highlights_the_committed_bars( self, page: Page, server_port: int, k: int, m: int ): - """On bars narrower than a day, a brush over bars k..m commits whole - days that hold exactly their rows, and its box sits on the bar edges, - so Plotly highlights bars k..m. A brush up to the top bar commits - closed="both" with the last day. The echo guard tolerates half a bar.""" + """On bars narrower than a day, a brush over bars k..m commits the bar + edges, which the server rounds to the whole days that hold exactly + their rows. The box sits on the same edges, also when a restore + rebuilds it from the predicate, so Plotly highlights bars k..m. A brush + up to the top bar commits closed="both" up to the last day. The echo + guard tolerates half a bar.""" df, spec = _short_date_page(page, server_port) counts = _hist_y(page, "#fv-plot-0") day_ms = 86_400_000 @@ -7063,8 +7065,25 @@ def test_box_on_bar_edges_highlights_the_committed_bars( top = m == bars - 1 assert clause["closed"] == ("both" if top else "left") if top: - assert clause["range"][1] == "2020-01-07" + assert clause["range"][1] == "2020-01-07 00:00:00.000000" assert df.filter(_selection_expr(df, sels[0])).height == sum(counts[k : m + 1]) + # /share keeps only the model fields, so a restore rebuilds the box + # from the predicate: it must land on the same bar edges. + live, rebuilt = page.evaluate( + """(fig) => { + const sel = fvFigureSelection(fig, DASHBOARD_SPEC.state.selections); + const shared = { ...sel }; + delete shared._plotly_selection_box; + const kept = DASHBOARD_SPEC.state.selections; + DASHBOARD_SPEC.state.selections = [shared]; + const [box] = selectionBoxesForFigure(fig); + DASHBOARD_SPEC.state.selections = kept; + const b = sel._plotly_selection_box; + return [[b.x0, b.x1], [box.x0, box.x1]]; + }""", + spec.figures[0].uid, + ) + assert rebuilt == live (highlighted,) = page.eval_on_selector( "#fv-plot-0", "gd => gd.data.map(t => t.selectedpoints ? Array.from(t.selectedpoints) : null)" diff --git a/tests/test_cube.py b/tests/test_cube.py index 6ca309ff..27d07773 100644 --- a/tests/test_cube.py +++ b/tests/test_cube.py @@ -1822,11 +1822,18 @@ def test_slice_membership_equals_string_predicate(self, unit, span_days, p): ].sum() or 0 ) - # Mirrors _fvCubeCommitEdges: a closed top edge rounds down. - str_lo = _physical_to_temporal_str(e_lo, unit) - str_hi = _physical_to_temporal_str( - math.floor(e_hi) if closed == "both" else e_hi, unit - ) + # Mirrors _fvCubeCommitEdges: a Date edge keeps its time of day + # (µs); on a ms or µs axis a closed top edge rounds down. + if unit == "day": + str_lo, str_hi = ( + _physical_to_temporal_str(v * 86_400_000 * 1000, "us") + for v in (e_lo, e_hi) + ) + else: + str_lo = _physical_to_temporal_str(e_lo, unit) + str_hi = _physical_to_temporal_str( + math.floor(e_hi) if closed == "both" else e_hi, unit + ) pred = SelectionPredicate( clauses=[ ClauseFilter(column="t", range=(str_lo, str_hi), closed=closed) diff --git a/tests/test_cube_server.py b/tests/test_cube_server.py index 38b8f8bc..42666f1b 100644 --- a/tests/test_cube_server.py +++ b/tests/test_cube_server.py @@ -3066,20 +3066,21 @@ def test_grouped_line_target_served(self, cat_client, cat_df): def _commit_clause(column: str, snap: tuple, unit: str | None = None): """Mirror of the client commit (``_fvCubeCommitEdges``): one clause from a - ``snap_brush`` result, a Date edge rendered as a whole-day string.""" + ``snap_brush`` result. A Date edge keeps its time of day as a µs string; + the server rounds it to whole days.""" import datetime as dt from flexviz.spec import ClauseFilter _lo_bin, _hi_bin, lo, hi, closed = snap if unit == "day": - if closed == "both": - hi = math.floor(hi) - def _day(v: float) -> str: - return (dt.date(1970, 1, 1) + dt.timedelta(days=math.ceil(v))).isoformat() + def _us(v: float) -> str: + us = math.ceil(v * 86_400_000 * 1000) + stamp = dt.datetime(1970, 1, 1) + dt.timedelta(microseconds=us) + return stamp.strftime("%Y-%m-%d %H:%M:%S.%f") - lo, hi = _day(lo), _day(hi) + lo, hi = _us(lo), _us(hi) return ClauseFilter(column=column, range=(lo, hi), closed=closed)