diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d3cd7cf..568a10a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: python3 scripts/verify_benchmark_report.py kernel.json --kind kernel-native python3 scripts/verify_benchmark_report.py transport.json --kind transport-loopback python3 scripts/check_regressions.py --scatter scatter.json --kernel kernel.json \ - --transport transport.json --emit-md docs/engineering/benchmark_metrics.md + --transport transport.json --emit-md spec/benchmarks/metrics.md - name: Upload regression benchmark report if: always() @@ -125,7 +125,7 @@ jobs: name: regression-benchmark-report if-no-files-found: warn path: | - docs/engineering/benchmark_metrics.md + spec/benchmarks/metrics.md scatter.json kernel.json transport.json @@ -254,7 +254,7 @@ jobs: - name: Python 3.11 tests (native core) run: .venv/bin/python -m pytest -q - # Cross-library scatter benchmark incl. time-to-first-render (docs/engineering/benchmark.md). + # Cross-library scatter benchmark incl. time-to-first-render (spec/benchmarks/results.md). # Each isolated group uses the same deterministic input sizes and runs in # parallel. The report job below merges the partial results. benchmark_vs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71d49ed5..2233de4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -123,7 +123,7 @@ jobs: # Pyodide / Emscripten WASM wheel for in-browser CPython. Pyodide's platform # ABI is compiler-sensitive, so keep Rust, Emscripten, the wheel tag, and the - # runtime probe pinned as one tested set (see docs/engineering/production-readiness.md). + # runtime probe pinned as one tested set (see spec/process/production-readiness.md). wasm: name: Wheel Pyodide (runtime verified) runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae0d4d22..b117ea21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,5 +20,4 @@ repos: language: system types: [text] files: ^docs/ - exclude: ^docs/engineering/ require_serial: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 586c2b54..4b80166a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,7 +119,7 @@ in the README). values. - `xy.pyplot`: a matplotlib-flavored shim over the composition API (`import xy.pyplot as plt`). Corpus-defined compatibility — - see `docs/engineering/matplotlib-compat.md`; fully contained in + see `spec/matplotlib/compat.md`; fully contained in `python/xy/pyplot/` with boundary guardrails. - **Statistical and density chart breadth.** Added first-class `errorbar`/ `error_band`, `box`, `violin`, `ecdf`, `hexbin`, and `contour` marks plus @@ -299,7 +299,7 @@ in the README). as a ~58 KB, resolution-independent SVG); covers every chart kind including density/heatmap rasters, and the full mark styling surface (gradients, dashes, symbols, rounded bars, smooth curves as exact cubic Béziers). -- **Mark-level styling** (both APIs; `docs/engineering/styling.md#styling-the-marks`): +- **Mark-level styling** (both APIs; `spec/api/styling.md#styling-the-marks`): - `fill="linear-gradient(...)"` on `area`/`bar`/`column`/`histogram` — real CSS gradient syntax (2–8 stops, `%` positions, `currentColor` = the mark's own resolved color, hue-preserving fades to `transparent`); mark-space by @@ -362,7 +362,7 @@ in the README). Linux glibc **and** musl/Alpine (x86-64, aarch64, armv7), macOS (x86-64, Apple Silicon), and Windows (x86, x64, arm64). An experimental Pyodide/Emscripten WASM wheel is built but does not yet load in-browser - (`docs/engineering/production-readiness.md` documents the exact linker failure and fix + (`spec/process/production-readiness.md` documents the exact linker failure and fix direction). - Release workflow `workflow_dispatch` dry-run mode: builds and verifies the full artifact matrix without publishing to PyPI (default for manual runs). diff --git a/CLAUDE.md b/CLAUDE.md index 823a89e4..ccd66108 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,9 +1,16 @@ # xy / xy A high-performance charting engine. The authoritative design is -`docs/engineering/design-dossier.md` — **read the relevant § before changing behavior**; +`spec/design-dossier.md` — **read the relevant § before changing behavior**; code comments cite dossier sections (e.g. §16 = deep-zoom re-centering). +The entire `spec/` directory is the source of truth for intended behavior, +architecture, compatibility, benchmarks, release readiness, and contributor +contracts. Keep it current with every relevant code, configuration, build, and +release change. A change is incomplete while its affected specification is +missing, stale, or inconsistent with the implementation; resolve discrepancies +instead of treating the implementation alone as authoritative. + ## Layout - `src/` — Rust core, **minimal external crates** (C ABI; one cdylib per @@ -31,9 +38,9 @@ code comments cite dossier sections (e.g. §16 = deep-zoom re-centering). - `python/xy/pyplot/` — the matplotlib shim, fully contained (one-way dependency onto the public composition API; guardrails in `tests/pyplot/test_boundaries.py`). Corpus-defined compatibility: - `tests/pyplot/corpus/` + `docs/engineering/matplotlib-compat.md`. + `tests/pyplot/corpus/` + `spec/matplotlib/compat.md`. - `python/reflex-xy/` — the Reflex adapter, a separate distributable - package (`reflex_xy`; design: `docs/engineering/design/reflex-integration.md`). Chart + package (`reflex_xy`; design: `spec/design/reflex-integration.md`). Chart data rides the app's own websocket as a second socket.io namespace; figures live in a per-process registry rebuilt from Reflex state on miss. Depends on `xy` + `reflex`; `xy` itself must never import diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb2b3d60..9c12a93d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ The full contributor guide — PR checklist, local gate commands, and the chart-type contribution walkthrough — lives at -[`docs/engineering/contributing.md`](docs/engineering/contributing.md). +[`spec/process/contributing.md`](spec/process/contributing.md). Quick start: @@ -26,6 +26,6 @@ python -c "import xy.kernels as k; print(k.BACKEND)" `BACKEND` is always `native`; an unavailable native core raises `ImportError` with remediation instead of silently degrading. -Design questions are settled by [`docs/engineering/design-dossier.md`](docs/engineering/design-dossier.md) +Design questions are settled by [`spec/design-dossier.md`](spec/design-dossier.md) — code comments cite its §-numbers. Read the relevant section before changing behavior, and don't regress the invariants listed in `CLAUDE.md`. diff --git a/README.md b/README.md index 4c9f5d95..68abc23d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@

- Grouped horizontal bar-chart comparison of xy, Matplotlib, and Plotly cold-render times at 10 million points; xy has the lowest measured time in all three output modes in this recorded run. + Grouped horizontal bar-chart comparison of xy, Matplotlib, and Plotly cold-render times at 10 million points; xy has the lowest measured time in all three output modes in this recorded run.

xy is an experimental Python charting library for large, interactive datasets. @@ -73,7 +73,7 @@ The same chart can be exported without changing how it is built. xy currently includes line, scatter, area, histogram, bar and column, heatmap, error bar and band, box, violin, ECDF, hexbin, contour, step, stairs, stem, triangle mesh, and faceted charts. See the -[copyable examples](docs/engineering/api-examples.md) for the complete surface. +[copyable examples](spec/api/api-examples.md) for the complete surface. ### Coming from matplotlib @@ -91,7 +91,7 @@ plt.show() ``` The shim intentionally covers common plotting workflows rather than every -matplotlib feature. See the [compatibility guide](docs/engineering/matplotlib-compat.md). +matplotlib feature. See the [compatibility guide](spec/matplotlib/compat.md). ## Benchmarks @@ -163,7 +163,7 @@ hover and selection can still return original rows. For benchmark methodology and measured results, see the [benchmark runbook](benchmarks/README.md) and the committed [launch report](benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro/report.md). -For the full design, see the [design dossier](docs/engineering/design-dossier.md). +For the full design, see the [design dossier](spec/design-dossier.md). ## Stable vs. Experimental @@ -202,12 +202,12 @@ needed for browser behavior. Engineering references: -- [API examples](docs/engineering/api-examples.md) -- [Styling](docs/engineering/styling.md) +- [API examples](spec/api/api-examples.md) +- [Styling](spec/api/styling.md) - [Benchmarks](benchmarks/README.md) -- [Matplotlib compatibility](docs/engineering/matplotlib-compat.md) -- [Architecture and design](docs/engineering/design-dossier.md) -- [Production readiness](docs/engineering/production-readiness.md) +- [Matplotlib compatibility](spec/matplotlib/compat.md) +- [Architecture and design](spec/design-dossier.md) +- [Production readiness](spec/process/production-readiness.md) - [Security](SECURITY.md) - [Changelog](CHANGELOG.md) diff --git a/SECURITY.md b/SECURITY.md index cfcd2749..3fd2f01b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,4 +33,4 @@ metadata is rejected. default; `sandbox=False` is an explicit caller opt-out for trusted HTML. - The native core is a local in-process C-ABI library; it processes only data already in the caller's process and performs no I/O or network access. -- The audit trail lives in `docs/engineering/security-audit-2026-07-06.md`. +- The audit trail lives in `spec/process/security-audit-2026-07-06.md`. diff --git a/docs/api-reference/contributing.md b/docs/api-reference/contributing.md index 3221b1e6..63d3a743 100644 --- a/docs/api-reference/contributing.md +++ b/docs/api-reference/contributing.md @@ -8,7 +8,8 @@ description: Set up XY development, run the right gates, and prepare a focused c XY welcomes focused bug fixes, documentation improvements, tests, and chart surface work. The canonical contributor guide is [CONTRIBUTING.md](https://github.com/reflex-dev/xy/blob/main/CONTRIBUTING.md); -the engineering version contains the complete release and browser checklists. +the internal [contributor specification](https://github.com/reflex-dev/xy/blob/main/spec/process/contributing.md) +contains the complete release and browser checklists. ## Local Setup diff --git a/docs/app/AGENTS.md b/docs/app/AGENTS.md index bb8bf4e7..b1218c1a 100644 --- a/docs/app/AGENTS.md +++ b/docs/app/AGENTS.md @@ -72,7 +72,7 @@ xy_docs/ # App entry point and XY-specific site composition scripts/ # Post-build sitemap, Markdown, and HTML validators tests/ # Docs, links, live-preview, and integration tests ../ # Public Markdown sources -../engineering/ # Internal project docs; never published +../../spec/ # Internal project docs; never published ``` ## Commands diff --git a/docs/app/README.md b/docs/app/README.md index fb3a3000..bc60a43d 100644 --- a/docs/app/README.md +++ b/docs/app/README.md @@ -30,7 +30,7 @@ development. Public pages live in the parent `docs/` directory. Page routes and sidebar order are declared centrally in `xy_docs/config.py`; do not add ordering -frontmatter to individual Markdown files. The `docs/engineering/` tree is +frontmatter to individual Markdown files. The `spec/` tree is internal project documentation and is intentionally excluded from the public site. diff --git a/docs/app/xy_docs/config.py b/docs/app/xy_docs/config.py index 43e20d9b..af46e09e 100644 --- a/docs/app/xy_docs/config.py +++ b/docs/app/xy_docs/config.py @@ -199,7 +199,6 @@ exclude=( "app/**", "assets/**", - "engineering/**", "styling/chrome-slots.md", "styling/component-variations.md", "styling/mark-styles.md", diff --git a/docs/engineering/README.md b/docs/engineering/README.md deleted file mode 100644 index 387fcc6f..00000000 --- a/docs/engineering/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Engineering Documentation - -This directory contains XY's implementation dossiers, benchmarks, compatibility -matrices, release-readiness records, and contributor internals. The public -documentation lives directly under `docs/`, while the Reflex application that -renders it lives in `docs/app/`. diff --git a/docs/engineering/benchmark_metrics.md b/docs/engineering/benchmark_metrics.md deleted file mode 100644 index 0dac09f8..00000000 --- a/docs/engineering/benchmark_metrics.md +++ /dev/null @@ -1,43 +0,0 @@ -### Auto-generated benchmark metrics - -Regenerated from the CI benchmark run; do not hand-edit. - -| metric | value | -|---|---:| -| kernel.bin_2d_mpts_s.1000000 | 207.85 | -| kernel.bin_2d_mpts_s.10000000 | 559.86 | -| kernel.bin_2d_ms.1000000 | 4.81 | -| kernel.bin_2d_ms.10000000 | 17.86 | -| kernel.encode_mpts_s.1000000 | 1,645.58 | -| kernel.encode_mpts_s.10000000 | 2,739.71 | -| kernel.histogram_mpts_s.1000000 | 622.92 | -| kernel.histogram_mpts_s.10000000 | 1,037.61 | -| kernel.histogram_ms.1000000 | 1.61 | -| kernel.histogram_ms.10000000 | 9.64 | -| kernel.local_density_mpts_s.1000000 | 72.83 | -| kernel.local_density_mpts_s.10000000 | 71.12 | -| kernel.local_density_ms.1000000 | 2.75 | -| kernel.local_density_ms.10000000 | 2.81 | -| kernel.local_density_n.1000000 | 200,000 | -| kernel.local_density_n.10000000 | 200,000 | -| kernel.m4_full_mpts_s.1000000 | 712.93 | -| kernel.m4_full_mpts_s.10000000 | 982.22 | -| kernel.normalize_mpts_s.1000000 | 1,602.35 | -| kernel.normalize_mpts_s.10000000 | 1,906.09 | -| kernel.range_mpts_s.1000000 | 1,750.32 | -| kernel.range_mpts_s.10000000 | 1,472.45 | -| kernel.range_ms.1000000 | 0.57 | -| kernel.range_ms.10000000 | 6.79 | -| kernel.zone_maps_mpts_s.1000000 | 810.47 | -| kernel.zone_maps_mpts_s.10000000 | 1,310.92 | -| kernel.zoom_redecimate_ms.1000000 | 0.05 | -| kernel.zoom_redecimate_ms.10000000 | 0.38 | -| scatter.tier.100000 | direct | -| scatter.tier.1000000 | density | -| scatter.tier.10000000 | density | -| scatter.wire_bytes.100000 | 800,983 | -| scatter.wire_bytes.1000000 | 855,778 | -| scatter.wire_bytes.10000000 | 853,850 | -| scatter.wire_bytes_per_point.100000 | 8.01 | -| scatter.wire_bytes_per_point.1000000 | 0.86 | -| scatter.wire_bytes_per_point.10000000 | 0.09 | diff --git a/docs/engineering/design/benchmark-methodology.md b/docs/engineering/design/benchmark-methodology.md deleted file mode 100644 index 1267107d..00000000 --- a/docs/engineering/design/benchmark-methodology.md +++ /dev/null @@ -1,149 +0,0 @@ -# Benchmark Methodology — Defensible Numbers - -**Status:** methodology spec. Extends the shipped harness (`benchmarks/ -bench_vs.py` adapters, `categories.py` registry, `_browser.py` FCP-based -TTFR, `bench_scatter_native.py`, CI `benchmark` job feeding -`docs/engineering/benchmark.md`). Written to survive a hostile Hacker News thread: every -number is mode-scoped, reproducible, oracle-checked, and the cases we *lose* -are published. - -## 0. The three rules - -1. **Mode-scoped claims only** (§2 of the dossier, already policy): "10M - density in X ms" is a different claim from "10M individually-styled - markers." Output labels every row `direct | decimated | density | sampled` - — a reader can never mistake an aggregate result for a raw-marker result. -2. **Same-work comparisons.** Each competitor renders the *same visual - contract*, not the same API call: if XY aggregates at 10M, the - fair Plotly comparison is Plotly failing/succeeding at raw markers AND - Plotly+Datashader doing aggregation — both are reported. We never - benchmark our fast path against a competitor's wrong tool. -3. **Truthfulness is a benchmark axis, not a footnote** (§6). Fast-but-wrong - fails the suite. - -## 1. Metric definitions (each with a precise probe) - -| Metric | Definition | Probe | -|---|---|---| -| payload prep | wall time: canonical arrays → wire payload (spec+blob) | `perf_counter` around `build_payload`; competitors: their figure→HTML/JSON serialize | -| wire size | bytes crossing to the client | `len(blob)+len(spec_json)`; competitors: HTML/JSON size | -| browser TTFR | figure build + interactive HTML serialization + navigation → visible chart surface | shipped `_browser.py` chart-ready probe; page FCP is diagnostic only | -| loopback transport | `channel.handle_message()` dispatch → HTTP envelope → client decode; raw/gzip bytes, Python allocation, p50/p95, browser heap, next frame | `benchmarks/bench_transport.py`; binary arm uses the production versioned frame and shipped JS decoder | -| interaction-ready | navigation → first successful synthetic wheel-zoom applied (view actually changes) | extend page probe: dispatch wheel, RAF-poll view/transform change | -| pan/zoom latency | p50/p95/p99 of input→pixels for repeated gestures | dispatched DOM events + draw + WebGL readback in `bench_interaction.py`; standalone-client scope is explicit | -| memory (kernel) | peak RSS delta + tracemalloc peak during prep | shipped psutil/tracemalloc pattern | -| memory (browser) | precise JS heap at chart-ready/dashboard settle; GPU memory remains unavailable | Chromium `performance.memory` with precise-memory flag; GPU numbers are not claimed | -| install size | `pip install` into a fresh venv: total site-packages bytes + wheel bytes + transitive dep count | scripted venv; competitors measured identically (Plotly+kaleido vs plotly alone reported separately) | -| cold import | `python -c "import lib"` best-of-5 fresh interpreters | subprocess timing (already the §33 import-budget concern) | -| small-data | full pipeline at N=1k/10k: prep+TTFR+interaction-ready | the "performance library must not lose the everyday case" check | -| large-data | N=1M/10M/100M per mode ladder | scatter+line+heatmap scenarios | - -## 2. Truthfulness/exactness checks (the novel part) - -Every performance scenario carries an **oracle assertion**; a run that fails -its oracle produces no number (it produces a bug): - -- **Extrema preservation (lines):** decimated polyline's per-pixel-column - min/max equals NumPy oracle min/max (M4's contract). Global max/min pixels - are lit within ±1px column. -- **Count conservation (density/histogram):** sum(grid) == in-window row - count (already the pattern in kernel tests; promoted to the bench harness - at 10M/100M). -- **Channel honesty (after LOD phase 1):** per-cell mean grid equals NumPy - groupby-mean oracle within f32 eps; majority cells match; purity ∈ [0,1]. -- **Drill exactness:** zoom to ≤ budget ⇒ rendered point count == oracle - window count; hover row values equal source f64 exactly. -- **Determinism:** two identical runs produce byte-identical payloads and - (SwiftShader) identical pixel hashes — the anti-shimmer guarantee, and - it's what makes all other numbers reproducible. -- **Reduction disclosure:** spec records tier/mode/visible for every - reduced trace (assert present) — "no silent quality changes" as a test. - -Competitors get the same oracles where the claim applies (e.g. Datashader -count conservation — it passes; Plotly scattergl marker dropout at high N — -documented finding, with repro). - -## 3. Competitor matrix - -Shipped adapters: xy, Plotly, matplotlib, seaborn, Bokeh, Altair, -Datashader, hvPlot/HoloViews, and **plotly-resampler** ✅ (`bench_line.py` — -the honest line competitor, same decimation thesis, so comparing against -vanilla Plotly alone on lines would be a strawman; both run, with the M4 -extrema oracle on the xy row). Still to add: **PyGWalker** (adapter: -programmatic `walk()` export path; if headless render proves unstable, report -prep+payload only and say so). Every adapter: `unavailable` rows rather than -silent omission (harness behavior). Two adjacent metric harnesses now ship -beside the scatter comparison: `bench_install.py` (cold import + install -footprint, §1) and the PNG-export path (`Figure.to_png`) that makes the -static-export size row a real, non-raster-only comparison. - -Per-competitor fairness notes ship in the report: Plotly measured both via -kaleido-PNG (their static path) and browser-HTML (their interactive path); -matplotlib is Agg (it's not interactive — its interaction rows are `n/a`, -not zero); Datashader is prep-only unless embedded in HoloViews (both -reported). - -## 4. Environment & disclosure protocol - -- **Two tiers of numbers:** (a) CI numbers — GitHub Actions ubuntu runner, - SwiftShader software GL; reproducible by anyone from the repo, labeled - "CI (software GL)"; (b) reference-hardware numbers — one pinned desktop - spec (documented CPU/GPU/driver/browser build), labeled as such. Never mix - tiers in one table. HN's first attack is "benchmarked on a potato/cherry - machine" — pre-empt by publishing both. -- Every table header carries: dataset generator + seed, library versions, - timestamp, harness commit. `benchmark.json` (already emitted by CI) is the - canonical artifact; `docs/engineering/benchmark.md` renders from it — numbers in prose - that don't exist in the artifact are banned (existing policy, kept). -- **Warm/cold discipline:** every timing reports which it is; first-run - (cold cache) and steady-state are separate rows for TTFR and import. -- **Losses ship.** The report has a standing "where XY loses" table - (e.g., tiny-data static PNG export vs matplotlib; ecosystem breadth). - Nothing buys credibility like published losses. - -## 5. Scenario set (the public story) - -1. `small_startup`: 1k line + 10k scatter — TTFR/interaction-ready vs all. -2. `line_10M`: decimated line vs plotly-resampler vs Bokeh — extrema oracle. -3. `scatter_10M_plain` and `scatter_10M_channels`: density path vs - Datashader/HoloViews; channel-honesty oracle; Plotly raw as the - documented-failure row. -4. `scatter_100M_pan`: pyramid pan/zoom percentiles (post LOD phase 3) — - the headline; includes never-blank check (frame sampling: no frame - without chart pixels). -5. `drilldown_truth`: CodSpeed native cycle covers density overview → exact - points → density zoom-out; the 10M+ exact-hover oracle remains the larger - browser/widget follow-up. -6. `core_2d_payloads`: CodSpeed tracks native histogram, area, bar, heatmap, - and composed/layered `xy.chart(...)` payload prep; - `benchmarks/bench_2d_charts.py` stays the Plotly/Seaborn chart-to-pixels - comparison. -7. `dashboard_scale`: `benchmarks/bench_dashboard.py` attempts 10/20/50 mixed - charts, checks every canvas initially and while scrolling, and records payload - prep, navigation readiness, JS heap, redraw-submission p95, per-chart context - loss/restoration events, and the stable loss-free chart-count ceiling. Partial - dashboards remain successful measurement rows rather than losing their metrics. - CI hard-gates the 10-chart row as loss-free/nonblank and applies deliberately - loose catastrophic budgets to its render, scroll, and redraw timings. -8. `install_import`: lower-bound distribution size plus opt-in fresh-venv total - site-packages, transitive distribution count, install time, and cold import. -9. `public_workflows`: `benchmarks/bench_workflows.py` tracks ingestion shapes, - streaming refresh/incremental pyramid update, and HTML/SVG/native-PNG/Chromium-PNG export independently. - -## 6. Remaining implementation plan - -1. Extend the shipped count-conservation and per-pixel line extrema oracles with - channel-aggregation and cross-library pixel-dropout oracles. -2. Add the final widget/comm and Reflex request-to-pixels probes. The shipped - loopback transport diagnostic now covers `channel.handle_message()` through - real HTTP plus browser decode/next-frame, but deliberately excludes widget - comm internals, production framing, GPU upload, and visible-pixel readback. -3. PyGWalker adapter and Plotly/Bokeh equivalents for the - `dashboard_20` scenario. -4. Reference-hardware runbook (`benchmarks/README`): exact pins + one-command - repro; publish both tiers on the next README refresh. - -Timing regression policy is two-level: movement beyond 2x is advisory on shared -runners, while movement beyond 4x is a hard failure. Interaction and visual -budgets are capped in the report verifier so a benchmark change cannot silently -make its own gate easier. diff --git a/docs/engineering/design/chart-grammar.md b/docs/engineering/design/chart-grammar.md deleted file mode 100644 index d3b42ca6..00000000 --- a/docs/engineering/design/chart-grammar.md +++ /dev/null @@ -1,191 +0,0 @@ -# Core Declarative Chart Grammar - -**Status:** API design proposal. Goal: fix the composition model **before** -the catalog grows to 40 kinds, so nothing here forces a public-API rewrite -later. Grounded in what ships today (`components.py`: `Chart` + `Mark` + -`Axis` + `Legend`, `_MARK_APPLIERS` registry; `figure.py`: fluent builders + -`_emit_` dispatch) — the proposal is an *extension* of that shape, not -a replacement. - -Related: `reflex-shaped-api.md` covers the public API/styling proposal for a -Reflex-like component surface without making Reflex a core dependency. - -## 1. The model in one paragraph - -A **Figure** is a grid of **Panels** (1×1 today; faceting/subplots later). -A Panel owns **Scales** (x, y, and later y2) and a z-ordered list of -**Marks**. A Mark = kind + data bindings + style props + optional channel -encodings. Axes, legend, and title are Panel/Figure chrome that *read* scales -and marks — they never own data. Everything is a plain declarative node -(dataclass), composable as children, and compiles to the existing internal -engine figure + wire spec. One public front door over that engine: - -- **Compositional (Reflex-flavored):** `xy.chart(xy.scatter(...), xy.line(...), - xy.x_axis(...))` — declarative, component-tree shaped, what a Reflex wrapper - serializes naturally. (The internal `_figure.Figure` fluent methods share the - same mark implementations, so the vocabulary cannot fork.) - -Rule G0: **the public composition API and the internal mark core are the same -vocabulary** (same mark names, same prop names, same defaults — one -implementation in `marks.py`). This is already true and is the thing we must -not break. - -## 2. The layering/overlay rules (what makes composition sound) - -- **G1 — Marks layer by order.** Children render in listed order (painter's - model). No z-index prop until a real need appears. -- **G2 — One shared coordinate space per panel.** All marks in a panel share - x and y scales; autorange is the union of every mark's `range_for(axis)` - contribution (already the per-kind hook in `Trace.range_for`). A mark never - gets a private scale — the escape hatch is a second *panel*, or (later, - explicit) `y_axis(secondary=True)` creating a named y2 scale that marks - opt into with `axis="y2"`. Silent dual axes are how charts lie; y2 must be - loud. -- **G3 — Scale type is a panel decision** (`linear | time | log | category`), - auto-inferred from marks (time columns → time; bar categories → category) - but overridable on the axis node. Mixing marks whose natural scales - conflict (bar-category + scatter-linear x) is a build-time error with a - fix-it message, not a coercion. -- **G4 — Chrome reads, never owns.** Legend derives entries from mark - channel modes (already true); axes derive from scales; tooltips derive - from the hovered mark's readout row. Adding a mark kind never edits chrome - code (contract already enforces this via capabilities). -- **G5 — Declarative all the way down.** Every node is data (kind + props). - No callbacks in the tree except the event props (`on_hover`, `on_select`, - later `on_view_change`), which is exactly what a Reflex component can - serialize + wire to server events without escape hatches. - -## 3. The 10 common charts (all expressible today or with planned nodes) - -```python -import xy - -# 1. line -xy.chart(xy.line(x="date", y="close", data=df), title="Price") -# 2. multi-series line (wide → long handled by repeated marks) -xy.chart(xy.line(x="date", y="aapl", data=df, name="AAPL"), - xy.line(x="date", y="msft", data=df, name="MSFT"), xy.legend()) -# 3. scatter with channels -xy.chart(xy.scatter(x="gdp", y="life", color="continent", size="pop", data=df)) -# 4. big scatter (auto density tier — same call, no special API) -xy.chart(xy.scatter(x="x", y="y", data=ten_million_rows)) -# 5. area -xy.chart(xy.area(x="date", y="active_users", data=df)) -# 6. histogram -xy.chart(xy.histogram(values="latency_ms", bins=256, data=df)) -# 7. bar -xy.chart(xy.bar(x="region", y="revenue", data=df)) -# 8. horizontal bar -xy.chart(xy.bar(y="region", x="revenue", data=df, orientation="h")) -# 9. heatmap -xy.chart(xy.heatmap(z=matrix, colormap="viridis")) -# 10. time series with unit-scale y -xy.chart(xy.line(x="ts", y="value", data=df), xy.y_axis(label="watts")) -``` - -(`xy.chart` is the kind-neutral container; the existing `scatter_chart`/ -`line_chart`/… wrappers remain as readable aliases — they already just tag -`Chart(kind_str, children)`.) - -## 4. The 5 complex overlays (the composition stress tests) - -```python -# A. line-on-scatter (regression overlay) — shared scales, order = layering -xy.chart( - xy.scatter(x="x", y="y", data=df, opacity=0.4), - xy.line(x=xs_fit, y=ys_fit, color="var(--accent)", width=2), -) - -# B. area-under-line (band + emphasis line) -xy.chart( - xy.area(x="date", y="p95", base="p5", data=df, opacity=0.25, name="p5-p95"), - xy.line(x="date", y="median", data=df, name="median"), -) - -# C. histogram + KDE-style line -xy.chart( - xy.histogram(values="dur", bins=200, density=True, data=df), - xy.line(x=kde_x, y=kde_y, width=2), -) - -# D. volume-under-candles (finance pane pair) — PANELS, not one panel: -xy.figure( - xy.panel(xy.candlestick(x="t", open="o", high="h", low="l", close="c", data=df), - height=3), - xy.panel(xy.bar(x="t", y="volume", data=df), height=1), - link_x=True, # shared x scale + synced pan/zoom across panels -) - -# E. threshold rule + annotated scatter -xy.chart( - xy.scatter(x="x", y="y", color="cluster", data=df), - xy.rule(y=0.8, color="#ef5350", dash=True), # planned: rule mark - xy.label(x=3.2, y=0.85, text="SLA"), # planned: label mark -) -``` - -D is the deliberate boundary case: volume-under-candles is **not** an overlay -(different y meaning) — the grammar answers it with linked panels, matching -the "chart owns rendering, app owns chrome" decision already taken. `link_x` -is the chart-level primitive (one shared x scale + view sync); arranging the -panes is layout the Figure grid owns. - -## 5. Node vocabulary (target; ★ = exists today) - -- Containers: `figure` (grid of panels; today implicit 1×1), `panel` - (planned), `chart` ★ (sugar: figure with one panel). -- Marks ★: `scatter, line, area, histogram, bar/column, heatmap, errorbar, - error_band, box, violin, ecdf, hexbin, contour, step, stairs, stem` (+ every - future kind — one `Mark` node each, registry-dispatched). -- Annotation marks (planned, same Mark machinery, tiny data): `rule`, - `band`, `label`. -- Chrome: `x_axis`/`y_axis` ★ (grows `type_="log"|"category"` ★partial, - `secondary=True` later), `legend` ★, `title` (prop today, node later - if styling demands). -- Events ★: `on_hover`, `on_select`; planned `on_view_change` (viewport for - server-driven cross-filtering in Reflex apps). - -## 6. Reflex integration (no escape hatches) - -The tree above is precisely a Reflex component tree's shape: snake_case -props, children composition, `data=` + column-name resolution (`data_key` -idiom), event props. It remains a XY-owned tree, not a Reflex object, -so the core package keeps zero Reflex dependencies. A future Reflex wrapper is -therefore a *thin* codegen layer: - -1. Each `xy.*` factory maps 1:1 to a Reflex component; props serialize as-is - (they're plain scalars/strings/arrays). -2. Data flows as the existing binary payload through a Reflex asset/endpoint - (the 100M live-drilldown demo already proves the comm shape: a `comm` - object with `send`/`onMessage` over any transport). -3. Events (`on_hover/on_select/on_view_change`) map to Reflex event handlers; - payloads are the same dicts the widget path already emits. -4. State-driven restyle = re-render with new props; data identity is kept by - the column-store handles so unchanged columns don't re-ship (buffer-diff - updates, dossier §6 — planned, and this is the API that needs it). - -Anti-goals, stated so they stay out: no `raw_plotly_json`-style passthrough -node, no per-mark imperative `draw(ctx)` callback, no CSS-in-props for mark -geometry (theming stays on the `--chart-*` token path). - -## 7. Migration & compatibility - -- Everything shipped keeps working: `chart(...)` is additive sugar; - `panel`/`figure` introduce multi-pane without touching single-pane specs - (spec gains `panels: [...]` only when >1 — protocol bump at that point, - bundled with the view-message unification the contract already schedules). -- The wire spec keeps its current per-trace shape; panels reference traces by - id. Scales become explicit spec objects (`scales: {x: {...}, y: {...}}`) - in the same bump — currently implicit in `x_axis/y_axis` ranges. -- The `Chart(kind_str, ...)` wrappers never encoded behavior (kind string is - cosmetic) — safe to keep forever. - -## 8. Implementation order - -1. `xy.chart` neutral container + `rule`/`band`/`label` annotation marks - (small, high leverage for real dashboards). -2. Explicit scale objects in the spec + `category`/`log` axis completion. -3. `panel`/`figure` grid + `link_x` view sync (enables finance pair, subplot - grids; carries the protocol bump). -4. `y2` secondary scale (loud opt-in). -5. Faceting sugar (`facet(col="…")`) compiling to the panel grid. diff --git a/docs/engineering/design/renderer-architecture.md b/docs/engineering/design/renderer-architecture.md deleted file mode 100644 index 1f1ba04d..00000000 --- a/docs/engineering/design/renderer-architecture.md +++ /dev/null @@ -1,144 +0,0 @@ -# Renderer Architecture — Audit & Target Design - -**Status:** audit of the shipped WebGL2 client (`js/src/40_gl.js`, -`45_lod.js`, `50_chartview.js`, `55_marks.js`, `60_entries.js`) + the -architecture it should converge to before ~20 more chart kinds land. Findings -below are verified against the source, not hypothetical. - -## 1. What's structurally right (keep) - -- **Mark registry** (`MARK_KINDS` + capabilities `pointPick/retainCpu/ - refreshColor` + `markOf` fallback): the render loop is kind-blind; new - kinds are entries. This is the load-bearing decision — preserve it. -- **LOD module** (`45_lod.js`): tier orchestration/fades/caches are outside - ChartView and call back through `view._draw*`, so tests intercept the - renderer and future kinds can swap mark drawing. Keep the callback seam. -- **Three-surface layering**: GL canvas (marks) + 2D chrome canvas - (grid/axes) + DOM (labels/tooltip/legend/modebar/crosshair). Crisp text, - selectable tooltips, zero GL cost for chrome — correct division (§7). -- **Uniform-only pan/zoom**: geometry is static offset-encoded f32; view - changes touch two vec2 uniforms per mark (`_map`). This is why interaction - is cheap; nothing below may regress it. -- **Screen-bounded inputs**: the client never receives O(N) buffers (§29). -- **DPR correctness**: backing stores sized `css × devicePixelRatio` - everywhere including the pick FBO and its lazy resize-realloc; line widths - and point sizes multiply by dpr. Audited clean. (Gap: DPR *changes* — see - R7.) - -## 2. Audit findings — must-fix-before-20-charts list - -Ordered by how much each compounds as kinds multiply. - -- **R1 — Per-draw uniform lookups.** ✅ **Done.** `makeProgram` attaches a - per-program memo and all draw sites go through `uniformOf()`; each name - hits the driver once at first use. Verified by the pixel-determinism - smoke probe (bitwise-identical frames through the cached path). -- **R2 — No VAOs.** Every draw re-binds attributes via `_bindScalarAttr` - (verified: zero `createVertexArray` in the bundle). **Deferred with a - stated trigger:** at ~6 kinds the per-frame rebinding is unmeasurable and - the refactor risks exactly the stale-attribute bugs it would prevent. - Adopt together with the first rect-family pick geometry (the R9 trigger), - when draw+pick sharing VAO layouts pays twice. -- **R3 — Draw-state discipline.** Re-audit: the discipline is currently - sound — BLEND is enabled once at init and toggled in exactly one place - (the pick pass, correctly paired), and every texture bind sets its unit - explicitly. A `withState` helper today would be ~40 lines guarding one - pair — over-engineering by this codebase's own standard. **Deferred:** - adopt when a second state-toggling pass lands (e.g. scissored panels or - an additive-blend mark). -- **R4 — No GL context-loss handling.** ✅ **Done.** - `_initContextLossRecovery` listens for loss, prevents default eviction - handling, quiesces draw/animation/re-bin work, and increments the request - sequence so pre-loss kernel/worker replies cannot mutate the rebuilt state. - Streaming appends still replace the retained canonical payload while the - context is down. Restore drops dead handles, re-runs `_initGl` from that - payload, and re-fires the view request to re-sync live tiers; a failed - restore remains explicitly failed instead of throwing from the event - handler. The dependency-free smoke forces three `WEBGL_lose_context` - cycles, checks pixel-identical frames after each, and zooms after recovery. -- **R5 — Shader source conventions are informal.** ✅ **Done.** `build.mjs` - lints every shader at build time: `#version 300 es` first line, every FS - declares `precision highp float;`, every VS references a `u_*map` uniform - (quad shaders exempted by name), uniforms `u_`-prefixed, attributes - `a`-prefixed. Violations fail the build (negative-tested). -- **R6 — Instancing is per-mark bespoke.** Line uses 4-corner strip + - divisor-1 endpoints; rects likewise; points use POINTS. Fine at 5 kinds, - but each new rect-family kind re-writes the same corner-expansion VS - preamble. *Fix:* extract shared VS chunks (corner tables, px↔clip - helpers) via string includes in `40_gl.js` (`GLSL_COMMON`), not a shader - framework. Deliberately stop there — a "shader graph" is over-engineering - at this scale. -- **R7 — DPR/zoom *changes* aren't observed.** ✅ **Done.** `_armDprWatch` - re-arms a one-shot `matchMedia('(resolution: Ndppx)')` per dpr value; - `_resize` re-reads devicePixelRatio so a pure-DPR change re-derives - backing stores with no container resize. Smoke `dprw` flag covers it. -- **R8 — Lifecycle cleanup.** ✅ **Already complete** (re-audit): `destroy()` - → `_destroyGlResources` frees per-trace static + drill buffers, density/ - heatmap/LUT textures (dedup via `texSeen`), pick FBO/texture, the quad, and - all programs. The original finding is stale. Remaining nicety only: move - per-kind buffer-name lists into a registry `dispose` hook when a kind - arrives whose resources don't fit the shared name list. -- **R9 — Picking model won't stretch as-is.** GPU ID pass is point-only - (`pointPick`); rect-family picking (bars/candles) is planned as a - registry `pick` step (contract). Two additions when that lands: - slot >255 guard (u8 alpha channel caps 255 pickable traces — assert, - it's fine) and shared pick-VS chunks per R6 so each mark's ID pass reuses - its draw geometry. -- **R10 — Tooltip data path**: solid (local approx row → kernel-exact - replacement, seq-guarded, drill_seq-versioned, XSS-safe text nodes). Only - gap: the row schema is implicit per kind. Write it down in the chart-kind - contract (`{x, y?, ohlc?, color_*?}` + `_dist` for non-point hover) so new - kinds emit compatible rows — doc task, no code. - -## 3. Target architecture (converged form) - -``` -60_entries mount/unmount, comm plumbing (unchanged) -50_chartview ChartView = surfaces, scales/view state, - interaction, chrome, pick orchestration (shrinks) -55_marks MARK_KINDS: build/draw/pick/hover/dispose/ - refreshColor per kind (grows, stays declarative) -45_lod tier orchestration (kind-blind) (unchanged) -40_gl programs, GLSL_COMMON chunks, state - helper, VAO + location-cache utilities (gains R1/R2/R3/R5/R6) -``` - -ChartView's steady-state responsibilities: own the GL context + view rect + -event wiring + chrome; delegate everything mark-shaped to the registry and -everything tier-shaped to lod. It should *lose* lines as kinds are added, not -gain them — that's the health metric (it's 1707 lines today; the R-fixes and -candle-era extractions should hold it under ~1500 while kinds double). - -## 4. WebGPU migration path - -Position: **WebGL2 remains the shipping target** (universal, and our loads -are instancing-friendly); WebGPU is an *additive backend*, attractive for -Tier-2 compute (atomic-add binning in a compute shader, dossier §5) and -storage-buffer picking. The architecture above is deliberately the portable -subset: - -1. Everything GPU-API-specific already lives in `40_gl.js` + the `_draw*`/ - `_upload*` methods; marks talk to buffers/uniform-maps, not raw GL, once - R1-R3 land. That interface (`createProgram/uploadBuffer/draw(markState)`) - is implementable on WebGPU nearly 1:1 (uniform maps → bind groups, - VAOs → vertex-buffer layouts). -2. GLSL→WGSL is mechanical for our shader inventory (no derivatives beyond - fwidth — supported; no geometry tricks). Keep shaders tiny and - convention-bound (R5) so a one-time port stays a week, not a rewrite. -3. Migration trigger, stated so we don't drift: adopt WebGPU **only when** - (a) a measured Tier-2/3 client-side re-bin path needs compute, or - (b) WebGL2 fill-rate demonstrably caps a real workload the pyramid can't - absorb. Not before — two backends is a tax. - -## 5. Sequencing - -1. R1 + R2 + R3 together (one PR: `makeProgram` location cache, VAO helper, - state helper) — pure refactor, byte-identical smoke expected. -2. R4 context-loss + R8 dispose (one PR: lifecycle) + smoke probes - (lose/restore renders again; destroy leaves zero live GL objects via - `WEBGL_debug`-style accounting where available). -3. R5 build-time shader lint + R6 GLSL_COMMON extraction (with the next new - rect-family kind, not before). -4. R7 dpr watch (fold into `_resize`). -5. R9 pick generalization lands with the first pickable rect mark, per - contract trigger. diff --git a/docs/engineering/design/rust-engine.md b/docs/engineering/design/rust-engine.md deleted file mode 100644 index 30cb5e15..00000000 --- a/docs/engineering/design/rust-engine.md +++ /dev/null @@ -1,194 +0,0 @@ -# Rust Data Engine — Module Boundaries & FFI Protocol - -**Status:** design. Decides what lives in Rust vs Python and how the FFI seam -evolves without rewrites. Grounded in the shipped engine: zero-crate cdylib -(`src/lib.rs`, ABI v31), ctypes binding (`_native.py`), and -dispatch in `kernels.py`. The native core is required — `kernels.py` raises a -clear ImportError when it can't load, with no pure-Python fallback. - -## 1. The placement rule - -**Rust owns row-scan loops; Python owns decisions.** Precisely: - -- If work is O(N) over data rows and sits on an interaction path (build, - zoom, pan, drill) → Rust kernel. -- If work is O(traces), O(screen), or policy (tier choice, budget math, spec - assembly, validation, warnings) → Python. Moving policy into Rust buys - nothing (it's microseconds) and costs iteration speed — policy is what - changes weekly. -- The client (JS) owns nothing O(N): it receives screen-bounded buffers only - (§29). This boundary is what keeps the browser safe at 1B rows and it never - moves. - -### Current placement audit - -| Concern | Today | Verdict | -|---|---|---| -| zone maps, encode_f32, m4, bin_2d, min_max, histogram_uniform, normalize_f32, range/validity indices, local_log_density | Rust (ABI v31) | correct — new equal-length x/y columns use a paired zone-map call with bit-identical per-column reductions; full-domain density first paint fuses binning with uniform or counted-u8 overlay sampling while retaining exact standalone outputs; mesh/rectangle validity scans consume only columns not already proven finite by zone metadata | -| fixed-width string/bytes/bool factorization | Rust (ABI v31) | correct — compact palettes use a bounded L1-resident codebook with full-record collision checks and emit exact counts; U1 uses a direct Unicode-scalar table with endian support; ≥512k rows probe a prefix then encode disjoint chunks in parallel, merging late labels by canonical first-row order before any retry; Python sees only unique labels and retains display-label ordering policy | -| static display-list raster, row-banded polyline/point/segment paint, batched fill+stroke triangle meshes, affine scatter projection plus typed color/size resolution, density/heatmap colormap and sampling | Rust (ABI v31) | correct — commands borrow f32/u8 payload or canonical spans synchronously; compact stratified sampling reuses factorization counts; batched/banded output is byte-identical | -| ohlc_decimate (when finance returns) | was NumPy-in-kernels.py | acceptable stopgap **only** because candles decimate to ≤px buckets; promote to Rust with the pyramid work | -| tier decisions, hysteresis, drill_seq, spec/emitters, channel resolution | Python | correct — keep | -| visible-count mask for drill | NumPy expression in `lod.visible_mask` | promote: it's O(N) per zoom step at 100M — fold into `xy_range_indices` (already exists) so count+indices come from one pass | - -### Target Rust ownership (matches the priority list) - -binning (1D/2D/channel-aware) ✅/plan · decimation (M4 ✅, OHLC plan) · -range filtering (`xy_range_indices` ✅) · implicit uniform and compact-u8 -stratified sampling (`xy_sample_range_indices` / `xy_stratified_sample_range_u8` ✅) · grouping/category encoding -(`xy_factorize_fixed` ✅ for contiguous fixed-width values; defensive Python -label canonicalization for mixed objects) · histogram stats ✅ · quantiles (plan: -`xy_quantiles`, needed by box/violin) · box/violin stats (thin composition -over quantiles — stats in Rust, assembly in Python) · multi-resolution tile -generation (`tiles.rs` ✅, including stable-domain incremental updates) · -Rust-owned streaming column buffers (plan: `stream.rs`, §5 below). - -## 2. Module boundaries (crate layout) - -``` -src/ - lib.rs # C ABI shell ONLY: extern fns, pointer/len marshaling, - # ABI_VERSION. No math. Every fn: null/len checks → slice - # → call kernels::* → write out-params. - kernels.rs # pure safe Rust, today's 9 kernels. No unsafe, no I/O. - tiles.rs # pyramid build/compose/incremental append. Owns tile memory; - # handles are opaque u64 ids passed over the ABI (§3.3). - stream.rs # (plan) Rust-owned canonical append buffers. - stats.rs # (plan) quantiles/box/violin/factorize. -``` - -Rules: `lib.rs` is the only file with `unsafe`; kernel modules are pure -functions over slices (fuzzable, testable without FFI); **dependencies are -minimized, not prohibited** (policy 2026-07-05): a crate may be added when it -pays for itself — measured win, small dependency tree, well-maintained — and -the C-ABI/one-cdylib-per-platform property is preserved. Note the dev sandbox -cannot reach crates.io, so required crates must be vendored or the sandbox -loses local build/test; prefer feature-gated optional deps (e.g. SIMD -argminmax, tsdownsample-class speed) with the lean build as default. - -## 3. FFI protocol — how it evolves without rewrites - -### 3.1 What's already right (keep as law) - -- **C ABI + ctypes, no PyO3**: no per-CPython builds; consumers install the - compiled wheel without a Rust or crate-registry dependency. -- **Caller-allocated buffers**: Python (NumPy) allocates outputs; Rust writes - into them and returns counts. No cross-language ownership for array data. -- **f64 in, f32 out** for geometry (offset-encoded, §16); u32 for indices. -- **Lockstep `ABI_VERSION`** in `src/lib.rs` + `_native.py`, checked at load, - hard error on mismatch — an old wheel never mis-calls a new lib. -- **The native core is the single implementation.** There is no pure-Python - fallback; every kernel is tested directly against the Rust core, and a - platform that can't load the core gets a clear ImportError, not a degrade. - -### 3.2 Evolution rules (the anti-rewrite discipline) - -- **E1 — additive by default**: new capability = new `xy_*` symbol. Existing - signatures are immutable once shipped in a release; changing one means a - new name (`xy_bin_2d_v2`) + ABI bump, old symbol kept for one minor cycle. -- **E2 — flags-struct for growth-prone kernels**: kernels we *know* will grow - options (tile fetch, channel binning) take a final `*const FcOpts` pointer - to a versioned plain-C struct (`{u32 size; ...}`); size-checked so old - callers pass smaller structs safely. This is how we avoid v2/v3 name churn - for the pyramid API specifically. -- **E3 — no callbacks across the ABI**: Rust never calls back into Python. - Progress/streaming = polling functions. Keeps the seam re-entrant and GIL- - trivial (all kernels release the GIL implicitly since ctypes calls do). -- **E4 — errors are return codes** (0 ok, negative = documented error enum), - never panics across FFI: `lib.rs` wraps kernel calls in `catch_unwind` → - error code (work item — today kernels are panic-free by construction, but - the belt goes on with the next ABI bump). -- **E5 — threading stays inside**: parallel kernels use `std::thread::scope` - inside the call; the ABI stays synchronous. General row scans cross over at - 512k values and scale to at most 18 workers. Zone maps cross earlier, at two - complete 65,536-row chunks, because chunks are independent and require no - merge; worker count is also capped by actual chunks. CodSpeed stays serial - because its simulator sums thread instructions rather than wall time. - Async/incremental build = handle + - `xy_pyramid_poll` (E3). - -### 3.3 Opaque handles (for tiles/streams) - -Arrays-in/arrays-out stops working when Rust must own long-lived state -(pyramid, append buffers). Pattern: - -```c -u64 xy_pyramid_build(const f64* x, const f64* y, /*channels…*/, FcOpts*); -i32 xy_pyramid_tile(u64 h, u32 level, u32 tx, u32 ty, f32* out_count, ...); -void xy_pyramid_free(u64 h); -``` - -Handles are indices into a Rust-side registry (mutex-guarded slab), not raw -pointers — a stale/double-freed handle is an error code, not UB. Python wraps -each in an object with `__del__`/weakref finalizer. - -### 3.4 SIMD (contained in `src/simd.rs`) - -The cdylib builds for baseline x86-64 (SSE2), so hot scans never see 256-bit -registers unless we ask. `simd.rs` holds branch-free clones of selected -kernels compiled under `#[target_feature(enable = "avx2")]` (LLVM -autovectorizes them — no hand-written intrinsics unless a loop demonstrably -fails to vectorize) with runtime `is_x86_feature_detected!` dispatch and a -`XY_SIMD=0` kill switch. - -Rules, in priority order: - -1. **Bitwise parity is non-negotiable.** Only order-independent kernels are - eligible: integer counts, exact comparisons, truncation casts, min/max. - Float *accumulation* (zone-map sum/sum_sq) must stay scalar — vector - reassociation changes the result. Parity is enforced by fuzz tests in - `simd.rs` comparing SIMD vs scalar on hostile data. -2. **Only measured wins ship.** Every dispatch is justified by a before/after - number (via the kill switch); a kernel where the two-phase restructure - loses (M4's sequential bucket state machine, histogram's scatter-dominated - loop) is documented at its scalar definition and NOT dispatched. -3. **`unsafe` containment.** This module is the one exception to "unsafe only - in `lib.rs`": the `#[target_feature]` wrappers are unsafe to call, and - every call site sits behind a safe `try_*` fn that checks detection first. - Kernels code never writes `unsafe` — it calls `simd::try_*` and falls back. -4. **aarch64 needs no twin.** NEON is part of the aarch64 baseline, so the - scalar kernels already autovectorize at full width there. - -## 4. What Python keeps forever - -Ingest normalization (pandas/arrow/dtype coercion — ecosystem glue), -`ColumnStore`/zone-map bookkeeping (thin, O(chunks)), all spec emission, -tier/budget policy, channel *resolution* (mode inference, palette, warnings), -validation and error messages (the new bounds/bool hardening lives at this -layer and belongs there), widget/comm transport. This layer is the product's -personality; keeping it in Python is a feature. - -## 5. Streaming append (Phase-0 landed Python-side; Rust `stream.rs` later) - -**Landed (Phase-0, Python-side canonical):** `Column.append` grows an -amortized capacity buffer and extends zone maps incrementally (only chunks -at/after the old length recompute — the splice is bitwise identical to a -from-scratch ingest). `Figure.append(trace_id, x, y, color=, size=)` -validates atomically (line appends must continue the sorted series; -categorical channels and shared columns are rejected for now), frees the -trace's pyramid for lazy rebuild, exits any drill, and returns an `append` -message carrying a complete fresh payload — screen-bounded by construction -(§29), so the wire never needs deltas. The client rebuilds only the traces -named in `affected`, applies the follow policy (refit when at home, slide -when pinned to the live right edge, hold when inspecting history), and -refines tiered traces to its current window through the normal -stale-while-revalidate request path (§17). - -**Still future (`stream.rs`):** Rust-owned chunked append buffers with -zone maps computed on seal, and — the important one — appends marking -intersecting pyramid tiles dirty with lazy per-tile rebuild (bounded: a -stream touching one region rebuilds ~1 tile/level). Phase-0 instead frees -the whole pyramid on append, so a >2M-point stream pays a full pyramid -rebuild on its next far-out view — recorded, not hidden. ABI: -`xy_stream_new/append/seal/free` + the pyramid fetch reading through the -stream handle. - -## 6. Implementation order - -1. E4 panic-shield + error enum (with next ABI bump, cheap insurance). -2. `xy_bin_2d_channels` (LOD doc phase 1) — first FcOpts-style kernel. -3. Fold drill visible-count into `xy_range_indices` (one pass, count+idx). -4. `stats.rs`: `xy_quantiles` (+ box/violin composition) — unblocks rank-8 - box plots with Rust-grade interaction. -5. `tiles.rs` pyramid handles (LOD phases 3-4). -6. `stream.rs` append (after Arrow ingest lands). diff --git a/docs/integrations/matplotlib.md b/docs/integrations/matplotlib.md index 00a2d5de..6cfa777d 100644 --- a/docs/integrations/matplotlib.md +++ b/docs/integrations/matplotlib.md @@ -47,7 +47,7 @@ Artist graphs, clipping/transform graphs, and material options that XY cannot honor fail with an actionable error instead of being silently ignored. Consult the repository's -[generated compatibility matrix](https://github.com/reflex-dev/xy/blob/main/docs/engineering/matplotlib-compat-matrix.md) +[generated compatibility matrix](https://github.com/reflex-dev/xy/blob/main/spec/matplotlib/compat-matrix.md) when a workflow depends on a specific option. Compatibility shims remain experimental and can change before XY 1.0. diff --git a/examples/pdsh/README.md b/examples/pdsh/README.md index 38b78ee0..0c15ac3e 100644 --- a/examples/pdsh/README.md +++ b/examples/pdsh/README.md @@ -74,7 +74,7 @@ cannot download the Olivetti faces; the previously recorded failure was the `subplots_adjust` rejection itself, raised before any drawing). ¹ 3D projections are outside xy's 2-D chart-method compatibility target -(see [docs/engineering/matplotlib-compat.md](../../docs/engineering/matplotlib-compat.md)); +(see [spec/matplotlib/compat.md](../../spec/matplotlib/compat.md)); `plt.axes(projection='3d')` fails loudly rather than silently returning a 2-D axes, so only this notebook's 2-D cells pass. ² Soft evidence: seaborn draws through real matplotlib internally, so @@ -95,7 +95,7 @@ graded pairs) stands at **34 match / 38 minor / 8 major** — majors down from 33 on those notebooks, with the stylesheet-legend, color-cycle, colorbar-domain, and colormap-fidelity classes fully cleared. The audit findings, fixes, and remaining boundaries are recorded in -[docs/engineering/matplotlib-compat-changelog.md](../../docs/engineering/matplotlib-compat-changelog.md). +[spec/matplotlib/compat-changelog.md](../../spec/matplotlib/compat-changelog.md). One caveat inflates the scorecard above: with seaborn (which draws through real matplotlib) holding the current figure, module-level diff --git a/js/src/00_header.js b/js/src/00_header.js index fbf353f1..53f7e958 100644 --- a/js/src/00_header.js +++ b/js/src/00_header.js @@ -24,7 +24,8 @@ const PROTOCOL = 3; -// HTTP binary frame v1 (reflex-integration §3.2). The chart spec's PROTOCOL +// HTTP binary frame v1 (spec/design/wire-protocol.md §7; Python side in +// python/xy/_framing.py). The chart spec's PROTOCOL // above versions renderer semantics; this separately versions the transport // envelope so either layer can fail loudly without coupling their evolution. const XY_FRAME_MAGIC = [0x58, 0x59, 0x42, 0x46]; // "XYBF" diff --git a/js/src/40_gl.js b/js/src/40_gl.js index 751e9686..7475250a 100644 --- a/js/src/40_gl.js +++ b/js/src/40_gl.js @@ -556,7 +556,7 @@ void main() { } }`; -// Mark-fill gradients (docs/engineering/styling.md#styling-the-marks): up to 8 stops, +// Mark-fill gradients (spec/api/styling.md#styling-the-marks): up to 8 stops, // interpolated in premultiplied alpha so a fade to `transparent` keeps the hue // (no dark fringe). u_gradMode: 0=off, 1=mark space (t runs along the mark's // value axis, 0 at the base, 1 at the tip/line), 2=plot space (screen axes — diff --git a/js/src/55_marks.js b/js/src/55_marks.js index 8360e4e6..ff66fd17 100644 --- a/js/src/55_marks.js +++ b/js/src/55_marks.js @@ -17,7 +17,7 @@ // Tiering is orthogonal: a density-tier trace is handled by 45_lod.js before // this registry is consulted (its drilled marks still render as points today). // Picking and legend swatches are separate extension points documented in -// docs/engineering/chart-kind-contract.md — generalize them when a pickable non-point +// spec/api/chart-kind-contract.md — generalize them when a pickable non-point // mark actually lands, not preemptively. // --------------------------------------------------------------------------- @@ -25,7 +25,7 @@ // ChartView branches: // pointPick — marks are point-shaped and participate in the GPU ID pick // pass (§17). A future rect/candle mark supplies its own pick -// step instead (see docs/engineering/chart-kind-contract.md). +// step instead (see spec/api/chart-kind-contract.md). // retainCpu — standalone export keeps CPU f32 copies of x/y so hover can // read approximate values with no kernel attached (§37). // refreshColor — re-resolve CSS-expressed constant colors on theme change diff --git a/pyproject.toml b/pyproject.toml index de13ab06..71001721 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ include = [ "js/src", "benchmarks", "docs", + "spec", "examples/reflex/README.md", "examples/reflex/requirements.txt", "examples/reflex/rxconfig.py", diff --git a/python/reflex-xy/README.md b/python/reflex-xy/README.md index 8ef29728..540ca6fe 100644 --- a/python/reflex-xy/README.md +++ b/python/reflex-xy/README.md @@ -5,7 +5,7 @@ interactivity, and streaming updates — with chart data riding the app's **existing websocket**, not a sidecar API. -Status: **prototype** implementing `docs/engineering/design/reflex-integration.md`. +Status: **prototype** implementing `spec/design/reflex-integration.md`. ## How it works diff --git a/python/reflex-xy/pyproject.toml b/python/reflex-xy/pyproject.toml index f775c8cb..73a72474 100644 --- a/python/reflex-xy/pyproject.toml +++ b/python/reflex-xy/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ # Prototype dependency surface. The adapter needs: Component/Var/EventHandler # (reflex-base), plus App/State/state_manager access, which today live in the # full `reflex` distribution. Revisit when the reflex-base split grows a - # public state/app surface (docs/engineering/design/reflex-integration.md §4). + # public state/app surface (spec/design/reflex-integration.md §4). "reflex>=0.9.6", ] diff --git a/python/reflex-xy/reflex_xy/__init__.py b/python/reflex-xy/reflex_xy/__init__.py index c4635ed7..a07c636b 100644 --- a/python/reflex-xy/reflex_xy/__init__.py +++ b/python/reflex-xy/reflex_xy/__init__.py @@ -1,7 +1,7 @@ """reflex-xy: xy figures as first-class Reflex components. The integration in one paragraph (full design: -docs/engineering/design/reflex-integration.md in the xy repo): chart data rides +spec/design/reflex-integration.md in the xy repo): chart data rides the app's *existing* websocket as a second socket.io namespace — binary columns, no JSON numbers, no extra endpoints to proxy. Figures live in a per-process registry keyed by tokens; the tokens live in Reflex state. A diff --git a/python/reflex-xy/reflex_xy/assets/XYChart.jsx b/python/reflex-xy/reflex_xy/assets/XYChart.jsx index 285d4b67..3b19afde 100644 --- a/python/reflex-xy/reflex_xy/assets/XYChart.jsx +++ b/python/reflex-xy/reflex_xy/assets/XYChart.jsx @@ -1,6 +1,6 @@ // XYChart: mount a xy figure inside a Reflex app. // -// Two modes, one prop apart (docs/engineering/design/reflex-integration.md): +// Two modes, one prop apart (spec/design/reflex-integration.md): // // `token` (live) — this component does NOT open its own connection. // socket.io multiplexing reuses the app's engine.io websocket when the diff --git a/python/reflex-xy/reflex_xy/component.py b/python/reflex-xy/reflex_xy/component.py index c483533b..7d9001c5 100644 --- a/python/reflex-xy/reflex_xy/component.py +++ b/python/reflex-xy/reflex_xy/component.py @@ -1,6 +1,6 @@ """The Reflex component: `reflex_xy.chart(...)`. -One factory, three chart sources (docs/engineering/design/reflex-integration.md §5): +One factory, three chart sources (spec/design/reflex-integration.md §5): reflex_xy.chart(Dash.chart) # @reflex_xy.figure state var (live) reflex_xy.chart(some_token_string) # register()/inline() token (live) diff --git a/python/reflex-xy/reflex_xy/namespace.py b/python/reflex-xy/reflex_xy/namespace.py index 6689b041..0974dfdd 100644 --- a/python/reflex-xy/reflex_xy/namespace.py +++ b/python/reflex-xy/reflex_xy/namespace.py @@ -1,6 +1,6 @@ """The xy data plane as a second socket.io namespace on Reflex's server. -Transport decision (docs/engineering/design/reflex-integration.md): instead of new HTTP +Transport decision (spec/design/reflex-integration.md): instead of new HTTP endpoints, the data plane multiplexes onto the app's existing engine.io websocket as its own namespace (`/_xy`). socket.io multiplexing means the browser keeps ONE physical connection for app state and chart data; this diff --git a/python/reflex-xy/reflex_xy/payload_asset.py b/python/reflex-xy/reflex_xy/payload_asset.py index beb6aa47..4854f2a8 100644 --- a/python/reflex-xy/reflex_xy/payload_asset.py +++ b/python/reflex-xy/reflex_xy/payload_asset.py @@ -11,7 +11,7 @@ (deep drilldown, server picks, streaming) deliberately out of scope. Reach for `reflex_xy.inline` or `@reflex_xy.figure` when those matter. -Why this works from any context (docs/engineering/design/reflex-integration.md): +Why this works from any context (spec/design/reflex-integration.md): - **Page bodies** run in the process that compiles the frontend, *before* the compiler copies ``assets/`` into ``.web/public`` — so a file written diff --git a/python/reflex-xy/reflex_xy/registry.py b/python/reflex-xy/reflex_xy/registry.py index d43770bb..2a911a9e 100644 --- a/python/reflex-xy/reflex_xy/registry.py +++ b/python/reflex-xy/reflex_xy/registry.py @@ -1,7 +1,7 @@ """Per-process figure registry: tokens in Reflex state, figures in here. The registry is deliberately NOT a distributed store (see -docs/engineering/design/reflex-integration.md §4): Reflex state is the durable, +spec/design/reflex-integration.md §4): Reflex state is the durable, already-distributed source of truth, and every registered figure is a rebuildable cache of it — the same rule the dossier applies to GPU buffers (§27). A registry miss (worker restart, reconnect landing on another node) diff --git a/python/reflex-xy/reflex_xy/tokens.py b/python/reflex-xy/reflex_xy/tokens.py index d2607489..049dad5f 100644 --- a/python/reflex-xy/reflex_xy/tokens.py +++ b/python/reflex-xy/reflex_xy/tokens.py @@ -10,7 +10,7 @@ Reflex state is the pantry). - **Opaque tokens** (`xyfig-`) come from imperative `reflex_xy.register(...)`. They cannot be rebuilt elsewhere — dev-tier by - design, documented in docs/engineering/design/reflex-integration.md. + design, documented in spec/design/reflex-integration.md. Tokens are visible to their own client (they ride through state deltas), so they must not carry anything the client doesn't already know: the client diff --git a/python/reflex-xy/reflex_xy/vars.py b/python/reflex-xy/reflex_xy/vars.py index a643e848..c21a7ad3 100644 --- a/python/reflex-xy/reflex_xy/vars.py +++ b/python/reflex-xy/reflex_xy/vars.py @@ -1,6 +1,6 @@ """`@reflex_xy.figure`: a computed var that *is* the chart registration. -The pattern (docs/engineering/design/reflex-integration.md): the state method builds the +The pattern (spec/design/reflex-integration.md): the state method builds the chart from state, the computed var's value is only the figure *token*, and evaluating the var is what (re)registers the figure in the per-process registry. Reflex's own dependency tracking decides when that happens: diff --git a/python/xy/__init__.py b/python/xy/__init__.py index 8135e205..5b8852fe 100644 --- a/python/xy/__init__.py +++ b/python/xy/__init__.py @@ -2,7 +2,7 @@ Cost scales with pixels on screen, not points in the dataset: native Rust core in the Python process, offset-encoded f32 binary transport, M4 decimation, GPU -density aggregation, and a WebGL2 render client. See docs/engineering/design-dossier.md. +density aggregation, and a WebGL2 render client. See spec/design-dossier.md. One declarative API over one engine — Reflex-flavored composition with `on_*` event props: diff --git a/python/xy/_svg.py b/python/xy/_svg.py index be4d3a4c..a135e758 100644 --- a/python/xy/_svg.py +++ b/python/xy/_svg.py @@ -9,7 +9,7 @@ Layout, tick math, colormaps, and mark styling mirror the JS client (`30_ticks.js`, `10_colormaps.js`, `50_chartview.js`); tests assert the ported tables stay in sync with the JS parts. Known static-export -approximations, documented in docs/engineering/styling.md: area mark-space gradients use +approximations, documented in spec/api/styling.md: area mark-space gradients use the area's bounding box (SVG has no per-column gradient); complete chart color tokens resolve statically, while nested browser-only expressions remain browser-dependent in SVG and use the native PNG fallback. diff --git a/python/xy/channel.py b/python/xy/channel.py index 9b158fbf..89d8cbf5 100644 --- a/python/xy/channel.py +++ b/python/xy/channel.py @@ -1,4 +1,4 @@ -"""Transport-agnostic message dispatcher (docs/engineering/design/reflex-integration.md §3.1). +"""Transport-agnostic message dispatcher (spec/design/reflex-integration.md §3.1). One kernel-side dispatcher for every transport: the anywidget comm today, and any future host (the planned Reflex adapter's HTTP routes) tomorrow. A diff --git a/python/xy/components.py b/python/xy/components.py index ef0a9182..6c7e37c6 100644 --- a/python/xy/components.py +++ b/python/xy/components.py @@ -3299,7 +3299,7 @@ def _mark_style_dict(value: Any, label: str) -> dict[str, StyleValue]: def _slot_styles_dict(value: Any, label: str) -> dict[str, dict[str, StyleValue]]: - """Per-slot inline styles (`styles={slot: {...}}` — docs/engineering/styling.md's + """Per-slot inline styles (`styles={slot: {...}}` — spec/api/styling.md's fourth mechanism): slot names validate against `CHART_DOM_SLOTS`, each inner dict through the same CSS-declaration gate as `style=`.""" if value is None: diff --git a/python/xy/marks.py b/python/xy/marks.py index 9fc476b5..f64bcb36 100644 --- a/python/xy/marks.py +++ b/python/xy/marks.py @@ -549,7 +549,7 @@ def area( `base` may be a scalar or a length-N array, which covers both the common zero-baseline area chart and future stacked-area construction. - `fill` accepts a CSS `linear-gradient(...)` (see docs/engineering/styling.md); + `fill` accepts a CSS `linear-gradient(...)` (see spec/api/styling.md); `curve="smooth"` renders a monotone cubic through the points; `dash` dashes the outline. """ @@ -1838,7 +1838,7 @@ def bar( `corner_radius`/`stroke`/`stroke_width` are the CSS border analogues rendered into the mark; `fill` accepts a CSS `linear-gradient(...)` - (docs/engineering/styling.md#styling-the-marks).""" + (spec/api/styling.md#styling-the-marks).""" css = styles.compile_mark_style("bar", style) color = css.get("color", color) opacity = css.get("opacity", opacity) diff --git a/python/xy/pyplot/__init__.py b/python/xy/pyplot/__init__.py index a5d94473..5afbf7fc 100644 --- a/python/xy/pyplot/__init__.py +++ b/python/xy/pyplot/__init__.py @@ -9,7 +9,7 @@ Every call translates onto the public declarative API (`xy.chart` and friends); the engine never knows this module exists. Coverage is -corpus-defined — see docs/engineering/matplotlib-compat.md for the supported surface +corpus-defined — see spec/matplotlib/compat.md for the supported surface and the loud `NotImplementedError` list. """ @@ -722,7 +722,7 @@ def plot( ``marker``, ``markersize``/``ms``, ``markerfacecolor``/``mfc``, ``markeredgecolor``/``mec``, ``markeredgewidth``/``mew``, ``markevery``, ``drawstyle``, and ``transform``. Anything else - raises a loud error (see docs/matplotlib-compat.md). + raises a loud error (see spec/matplotlib/compat.md). Returns the list of `Line2D` handles, one per plotted series. """ @@ -2840,9 +2840,7 @@ def __getattr__(self, name: str) -> Any: if name.lower() in CMAPS: return name - raise AttributeError( - f"colormap {name!r} is not supported; see docs/engineering/matplotlib-compat.md" - ) + raise AttributeError(f"colormap {name!r} is not supported; see spec/matplotlib/compat.md") cm = _CmapNamespace() diff --git a/python/xy/pyplot/_axes.py b/python/xy/pyplot/_axes.py index 21283a44..3f4b3ee8 100644 --- a/python/xy/pyplot/_axes.py +++ b/python/xy/pyplot/_axes.py @@ -4958,7 +4958,7 @@ def _bbox_label_style(bbox: dict[str, Any], font_size: float = 11.0) -> dict[str """matplotlib text ``bbox`` patch → annotation-label box styles. A CSS approximation drawn by the render client's DOM label; the static - exporters keep the plain label (recorded in docs/engineering/matplotlib-compat.md). + exporters keep the plain label (recorded in spec/matplotlib/compat.md). """ style: dict[str, Any] = {} face = bbox.get("fc", bbox.get("facecolor", "C0")) diff --git a/python/xy/pyplot/_translate.py b/python/xy/pyplot/_translate.py index fad58880..ebacb4b0 100644 --- a/python/xy/pyplot/_translate.py +++ b/python/xy/pyplot/_translate.py @@ -12,7 +12,7 @@ from ._colors import resolve_color -COMPAT_URL = "https://github.com/reflex-dev/xy/blob/main/docs/engineering/matplotlib-compat.md" +COMPAT_URL = "https://github.com/reflex-dev/xy/blob/main/spec/matplotlib/compat.md" # Matplotlib's unscaled named dash patterns, in points (rcParams # lines.{dashed,dotted,dashdot}_pattern). Matplotlib multiplies these by the diff --git a/scripts/check_claim_guardrails.py b/scripts/check_claim_guardrails.py index a5e811d0..f1dea905 100644 --- a/scripts/check_claim_guardrails.py +++ b/scripts/check_claim_guardrails.py @@ -21,11 +21,11 @@ "README.md", "SECURITY.md", "CONTRIBUTING.md", - "docs/engineering/api-examples.md", - "docs/engineering/benchmark.md", - "docs/engineering/chart-roadmap.md", - "docs/engineering/contributing.md", - "docs/engineering/production-readiness.md", + "spec/api/api-examples.md", + "spec/benchmarks/results.md", + "spec/api/chart-roadmap.md", + "spec/process/contributing.md", + "spec/process/production-readiness.md", "examples/reflex/README.md", ) @@ -82,7 +82,7 @@ re.compile( r"\b(?:benchmark|measured|documented|ttfr|payload|memory|ms|mb|gb|artifact)\b", re.I ), - re.compile(r"\b(?:chart type|data size|mode|row|table|docs/engineering/benchmark\.md)\b", re.I), + re.compile(r"\b(?:chart type|data size|mode|row|table|spec/benchmarks/results\.md)\b", re.I), ) @@ -193,7 +193,7 @@ def _default_paths() -> list[Path]: public_docs = ( path for path in sorted((ROOT / "docs").rglob("*.md")) - if not {"app", "engineering"}.intersection(path.relative_to(ROOT / "docs").parts) + if "app" not in path.relative_to(ROOT / "docs").parts ) return list(dict.fromkeys((*paths, *public_docs))) diff --git a/scripts/check_python_floor.py b/scripts/check_python_floor.py index 2091945a..3600fc6c 100644 --- a/scripts/check_python_floor.py +++ b/scripts/check_python_floor.py @@ -42,7 +42,7 @@ DECLARATION_DOCS = ( Path("README.md"), - Path("docs/engineering/production-readiness.md"), + Path("spec/process/production-readiness.md"), ) diff --git a/scripts/check_regressions.py b/scripts/check_regressions.py index 279b3b05..05a36142 100644 --- a/scripts/check_regressions.py +++ b/scripts/check_regressions.py @@ -143,6 +143,31 @@ def _fmt(v) -> str: return str(v) +def _report_provenance(*reports: object) -> str | None: + commits: set[str] = set() + timestamps: set[str] = set() + for report in reports: + if not isinstance(report, dict): + continue + environment = report.get("environment") + if not isinstance(environment, dict): + continue + timestamp = environment.get("generated_at_utc") + if isinstance(timestamp, str) and timestamp: + timestamps.add(timestamp) + git = environment.get("git") + if isinstance(git, dict): + commit = git.get("commit") + if isinstance(commit, str) and commit: + commits.add(commit) + details: list[str] = [] + if commits: + details.append("commit " + ", ".join(f"`{commit}`" for commit in sorted(commits))) + if timestamps: + details.append(f"latest measurement `{max(timestamps)}`") + return "Source CI reports: " + "; ".join(details) + "." if details else None + + def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--scatter", default=None, help="bench_scatter_native.py --json output") @@ -215,10 +240,12 @@ def main() -> None: print(f" - {mid}: {_fmt(b)} -> {_fmt(c)}") if args.emit_md: + provenance = _report_provenance(scatter, kernel, transport) lines = [ "### Auto-generated benchmark metrics", "", "Regenerated from the CI benchmark run; do not hand-edit.", + *(["", provenance] if provenance else []), "", "| metric | value |", "|---|---:|", diff --git a/scripts/rename_fc_to_xy.py b/scripts/rename_fc_to_xy.py index 26375fd0..716943a3 100644 --- a/scripts/rename_fc_to_xy.py +++ b/scripts/rename_fc_to_xy.py @@ -96,7 +96,12 @@ # Audit whitelist: occurrences of fc/fastchart that are correct to keep. # - "fc"/fc= is matplotlib's facecolor alias (quotes may be JSON-escaped in .ipynb) # - Fc is matplotlib's center-frequency parameter (psd/*_spectrum signatures) -MPL_PATHS = ("python/xy/pyplot/", "tests/pyplot/", "examples/pdsh/", "docs/matplotlib-compat.md") +MPL_PATHS = ( + "python/xy/pyplot/", + "tests/pyplot/", + "examples/pdsh/", + "spec/matplotlib/compat.md", +) MPL_FC = re.compile(r"""["']fc["']|\bfc=\\?["'0-9]|\bFc\b""") AUDIT = re.compile(r"(?i:\bfc\b|fastchart)|\bfc(?=[A-Z])") # base64 blobs / minified vendor payloads live on very long lines diff --git a/scripts/render_smoke_nonumpy.py b/scripts/render_smoke_nonumpy.py index 81e3c773..97ae3281 100644 --- a/scripts/render_smoke_nonumpy.py +++ b/scripts/render_smoke_nonumpy.py @@ -930,7 +930,7 @@ def main() -> None: vS._drawNow(); vS.destroy();holderS.remove(); const stream=(okGrow&&okHome&&okHoldS&&okPin&&okPayload)?1:0; - // Mark styling (docs/engineering/styling.md#styling-the-marks): gradient fills, + // Mark styling (spec/api/styling.md#styling-the-marks): gradient fills, // rounded corners, and stroke borders on BOTH rect-family programs // (histogram uses RECT, compact bar uses BAR), plus curve:"smooth" // monotone-cubic densification for line/area. diff --git a/scripts/sync_matplotlib_compat.py b/scripts/sync_matplotlib_compat.py index 8aa2f66e..c8abdd8d 100644 --- a/scripts/sync_matplotlib_compat.py +++ b/scripts/sync_matplotlib_compat.py @@ -15,7 +15,7 @@ SNAPSHOT = ROOT / "tests/pyplot/matplotlib_311_plotting.json" METADATA = ROOT / "tests/pyplot/compatibility.json" CORPUS = ROOT / "tests/pyplot/corpus" -OUTPUT = ROOT / "docs/engineering/matplotlib-compat-matrix.md" +OUTPUT = ROOT / "spec/matplotlib/compat-matrix.md" def _load(path: Path) -> dict: diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index 8b5d8d7c..5ef4aece 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -220,13 +220,13 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]: '--browser-reps 12 --chromium "$CHROME" --require-browser --json transport.json', "scripts/verify_benchmark_report.py transport.json --kind transport-loopback", "scripts/check_regressions.py --scatter scatter.json --kernel kernel.json", - "--transport transport.json --emit-md docs/engineering/benchmark_metrics.md", + "--transport transport.json --emit-md spec/benchmarks/metrics.md", "Upload regression benchmark report", "if: always()", "actions/upload-artifact@", "regression-benchmark-report", "if-no-files-found: warn", - "docs/engineering/benchmark_metrics.md", + "spec/benchmarks/metrics.md", "transport.json", ) _require_job_contains( diff --git a/scripts/verify_sdist.py b/scripts/verify_sdist.py index c520df3d..b0447a17 100644 --- a/scripts/verify_sdist.py +++ b/scripts/verify_sdist.py @@ -52,11 +52,17 @@ "benchmarks/bench_workflows.py", "benchmarks/categories.py", "benchmarks/environment.py", - "docs/engineering/api-examples.md", - "docs/engineering/benchmark.md", - "docs/engineering/chart-roadmap.md", - "docs/engineering/contributing.md", - "docs/engineering/production-readiness.md", + "spec/README.md", + "spec/design-dossier.md", + "spec/api/api-examples.md", + "spec/api/chart-roadmap.md", + "spec/benchmarks/results.md", + "spec/design/renderer-architecture.md", + "spec/matplotlib/compat.md", + "spec/process/contributing.md", + "spec/process/production-readiness.md", + "spec/assets/benchmark-snapshot.svg", + "spec/assets/launch-benchmark-comparison.svg", "hatch_build.py", "pyproject.toml", "js/src/00_header.js", @@ -290,11 +296,32 @@ def _require_baseline_json(path: str, root: str) -> None: raise AssertionError("benchmarks/baseline.json must contain a non-empty metrics object") +# Every grouped subdirectory of spec/ must survive packaging. Pinning +# individual files alone would let a whole group be dropped as long as the +# pinned member stayed, so require each group to be non-empty in its own right. +SPEC_SUBDIRS = ("api", "benchmarks", "design", "matplotlib", "process") + + +def _require_spec_layout(files: set[str]) -> None: + empty = [ + name + for name in SPEC_SUBDIRS + if not any(f.startswith(f"spec/{name}/") and f.endswith(".md") for f in files) + ] + if empty: + raise AssertionError(f"sdist has no markdown under spec/ subdirectories: {empty}") + + svgs = sorted(f for f in files if f.startswith("spec/assets/") and f.endswith(".svg")) + if len(svgs) < 2: + raise AssertionError(f"sdist is missing spec/assets SVG evidence snapshots: {svgs}") + + def verify_sdist(path: str) -> None: root, files = _normalized_files(path) missing = sorted(REQUIRED_FILES - files) if missing: raise AssertionError(f"sdist missing required files: {missing}") + _require_spec_layout(files) forbidden = sorted( name @@ -332,14 +359,14 @@ def verify_sdist(path: str) -> None: { "Stable vs. Experimental", "Python 3.11+", - "docs/engineering/api-examples.md", + "spec/api/api-examples.md", "make check-examples", }, ) _require_file_contains( path, root, - "docs/engineering/api-examples.md", + "spec/api/api-examples.md", { "Chart Family Quick Reference", "Small Business Chart", @@ -351,11 +378,11 @@ def verify_sdist(path: str) -> None: _require_file_contains( path, root, - "docs/engineering/benchmark.md", + "spec/benchmarks/results.md", { "benchmark-report", "regression-benchmark-report", - "docs/engineering/benchmark_metrics.md", + "spec/benchmarks/metrics.md", "scatter.json", "kernel.json", }, @@ -363,7 +390,7 @@ def verify_sdist(path: str) -> None: _require_file_contains( path, root, - "docs/engineering/production-readiness.md", + "spec/process/production-readiness.md", { "Release-Blocking Gates", "make check-artifacts", @@ -379,7 +406,7 @@ def verify_sdist(path: str) -> None: _require_file_contains( path, root, - "docs/engineering/contributing.md", + "spec/process/contributing.md", { "Pull Request Checklist", "make check-full", diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 00000000..f3fb4e04 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,100 @@ +# Specification + +The root-level `spec/` directory is XY's engineering source of truth: intended +behavior, architecture, compatibility, benchmarks, release readiness, and +contributor contracts. The public documentation lives directly under `docs/`, +while the Reflex application that renders it lives in `docs/app/`. + +Keep this tree current with every relevant code, configuration, build, and +release change. A change is incomplete while its affected specification is +missing, stale, or inconsistent with the implementation; resolve discrepancies +instead of treating the implementation alone as authoritative. + +[`design-dossier.md`](design-dossier.md) is the entry point — the single +compiled record of the design, the competitive research behind it, the +performance estimates, and the audit trail. Code comments cite its sections by +number (e.g. §16 = deep-zoom re-centering). + +## api/ + +The public surface: what callers can build, style, export, and interact with. + +- [`api-examples.md`](api/api-examples.md) — copyable examples for every + implemented 2D chart family; the Python snippets are executed by + `tests/test_docs_examples.py`, so API drift fails the suite. +- [`chart-kind-contract.md`](api/chart-kind-contract.md) — how to add a 2D chart + type: what the shared machinery provides and what a new kind must supply. +- [`chart-roadmap.md`](api/chart-roadmap.md) — the single 2D-first chart-type + coverage backlog, prioritized by popularity and primitive reuse. +- [`export.md`](api/export.md) — how a figure becomes bytes: one entry point + across five image formats, deterministic engine choice, browser-free default. +- [`interaction.md`](api/interaction.md) — the authority on which browser + interactions exist, which are configurable, and every event payload. +- [`styling.md`](api/styling.md) — the implementation contract for CSS-addressable + chrome, per-slot styles, and the documented static-render approximations. + +## design/ + +Internal architecture: how the engine is built and why. + +- [`chart-grammar.md`](design/chart-grammar.md) — the declarative composition + model (`Chart` + `Mark` + `Axis` + `Legend`), fixed before the catalog grows. +- [`lod-architecture.md`](design/lod-architecture.md) — the Tier 0/1/2/3 + LOD/drilldown design that keeps large data truthful and interactive. +- [`reflex-integration.md`](design/reflex-integration.md) — the `reflex-xy` + adapter design: figures as first-class Reflex components over a second + socket.io namespace. +- [`reflex-shaped-api.md`](design/reflex-shaped-api.md) — how the core package + feels Reflex-shaped while keeping no Reflex dependency. +- [`renderer-architecture.md`](design/renderer-architecture.md) — audit of the + shipped WebGL2 render path plus the architecture it converges to. +- [`rust-engine.md`](design/rust-engine.md) — what lives in Rust vs Python and + how the C-ABI/FFI seam evolves without rewrites. +- [`wire-protocol.md`](design/wire-protocol.md) — the client↔Python message + catalog, first-paint buffer layouts, and the version handshake. + +## matplotlib/ + +The `xy.pyplot` shim and its corpus-defined compatibility contract. + +- [`compat.md`](matplotlib/compat.md) — the supported `xy.pyplot` surface and + what the one-line import change does and does not promise. +- [`compat-matrix.md`](matplotlib/compat-matrix.md) — method-by-method + compatibility against the pinned upstream revision. Generated by + `scripts/sync_matplotlib_compat.py`; do not hand-edit. +- [`compat-changelog.md`](matplotlib/compat-changelog.md) — changes to the + upstream compatibility target and to the meaning of the compatibility levels. +- [`shim-todo.md`](matplotlib/shim-todo.md) — the audit and remaining work to + make the shim reliable, separating the supported target from full parity. + +## benchmarks/ + +Measured numbers and the rules that make them defensible. + +- [`results.md`](benchmarks/results.md) — the cross-library scatter comparison: + point ceiling, speed, memory, and payload size per library. +- [`metrics.md`](benchmarks/metrics.md) — the current regression metric table. + Emitted by `scripts/check_regressions.py --emit-md` in CI; do not hand-edit. +- [`methodology.md`](benchmarks/methodology.md) — how numbers are produced: + mode-scoped, reproducible, oracle-checked, and publishing the cases we lose. + +## process/ + +Release bar, contribution rules, and audit trail. + +- [`contributing.md`](process/contributing.md) — the contribution bar and the + production invariants a change must not lose. +- [`production-readiness.md`](process/production-readiness.md) — the release + bar, separating hard gates from advisory measurements. +- [`security-audit-2026-07-06.md`](process/security-audit-2026-07-06.md) — + scope, findings, and status of the 2026-07-06 source audit. + +## assets/ + +Files under `assets/` are dated evidence snapshots; their recorded dates are +intentional and do not represent the freshness of the current regression gates. + +- [CodSpeed benchmark snapshot](assets/benchmark-snapshot.svg) records the + historical 2026-07-09 run described in its caption. +- [Launch benchmark comparison](assets/launch-benchmark-comparison.svg) is the + comparison graphic used by the repository README. diff --git a/docs/engineering/api-examples.md b/spec/api/api-examples.md similarity index 84% rename from docs/engineering/api-examples.md rename to spec/api/api-examples.md index dbc904c9..8744046d 100644 --- a/docs/engineering/api-examples.md +++ b/spec/api/api-examples.md @@ -23,20 +23,30 @@ children, layered marks, axes, annotations, built-in or custom legend/tooltip chrome, and CSS/Tailwind-friendly DOM hooks. Callback payload details and future adapter packages may still evolve before 1.0. -Composed `Chart` objects handle standalone HTML export, PNG/SVG export, -notebook display, and memory reporting directly. The internal engine object a -chart compiles to is reachable via `chart.figure()` as an advanced escape -hatch, but it is not part of the public chart-building surface. +Composed `Chart` objects handle standalone HTML export, static image export +(PNG/JPEG/WebP/SVG/PDF), notebook display, and memory reporting directly. The +internal engine object a chart compiles to is reachable via `chart.figure()` as +an advanced escape hatch, but it is not part of the public chart-building +surface. Charts accept `width="100%"` and/or `height="100%"` for responsive layouts. -Standalone `to_html(...)` needs no browser dependency, and `to_png(...)` defaults -to a browser-free native rasterizer (`Engine.default`). -`to_png(..., engine=Engine.chromium)` uses an -installed Chrome, Chromium, Edge, or `chrome-headless-shell` executable because -it screenshots the same standalone HTML document for browser CSS/WebGL fidelity. -Automatic discovery can be overridden with `XY_BROWSER`. Pass `custom_css=` to -that engine when the screenshot also needs author CSS; native PNG intentionally -rejects browser-only stylesheets. +Standalone `to_html(...)` needs no browser dependency. Every static image +format — PNG, JPEG, WebP, SVG, PDF — is produced natively, without a browser. + +`Engine` has three members — `auto`, `default`, `chromium` — and the entry +points split on their default. `to_png(...)` defaults to `Engine.default`, the +browser-free native rasterizer; `to_svg(...)` takes no `engine` argument at all +and is always native. The unified `to_image(...)` / `write_image(...)` entry +points default to `Engine.auto`, which resolves deterministically per request: +native for every format, chromium only when `custom_css=` is passed, since +author CSS needs a real CSS engine. `to_png(..., engine=Engine.chromium)` uses +an installed Chrome, Chromium, Edge, or `chrome-headless-shell` executable +because it screenshots +the same standalone HTML document for browser CSS/WebGL fidelity. Automatic +discovery can be overridden with `XY_BROWSER`. Native raster export +intentionally rejects browser-only stylesheets, and SVG is native-only: +resolving to a browser engine for SVG — by `engine=Engine.chromium` or by +`custom_css=` — raises, because a screenshot cannot emit vector SVG. ## Chart Family Quick Reference @@ -439,8 +449,23 @@ chart ``` Composed `Chart` objects expose `to_html(...)`, `html(...)`, `_repr_html_()`, -`to_png(...)`, `to_svg(...)`, `widget()`, `show()`, and `memory_report()` -readout methods directly. +`to_png(...)`, `to_svg(...)`, `to_image(...)`, `write_image(...)`, `widget()`, +`show()`, and `memory_report()` readout methods directly. +`to_image(format="png", ...)` returns bytes and `write_image(path, ...)` writes +one file atomically with the format inferred from its extension; together they +are the unified export surface across PNG/JPEG/WebP/SVG/PDF (and standalone +HTML for `write_image`). `xy.write_images(figs, paths)` is the batch form: it +accepts composed charts directly and shares one persistent Chromium session +across every browser-resolved file instead of paying browser startup per +figure. + +Export defaults travel with the chart. `xy.export_config(...)` is a chart child +taking `formats`, `filename`, `width`, `height`, `scale`, `background`, and +`quality`; it sets the browser modebar's download menu (the client-safe subset +png/jpeg/webp/svg/csv, in the given order — an empty list hides the menu) and +supplies the `width`/`height`/`scale`/`background`/`quality` defaults for any +of those arguments omitted from `to_image`, `write_image`, or that chart's +files in `write_images`. Explicit arguments override them. ## Live Data On A Composed Chart diff --git a/docs/engineering/chart-kind-contract.md b/spec/api/chart-kind-contract.md similarity index 55% rename from docs/engineering/chart-kind-contract.md rename to spec/api/chart-kind-contract.md index 9b78b0cc..0db57af0 100644 --- a/docs/engineering/chart-kind-contract.md +++ b/spec/api/chart-kind-contract.md @@ -15,13 +15,22 @@ reduces to a few GPU primitives on top of the shared infrastructure. |---|---|---| | Points | built (`scatter`) | scatter, bubble | | Lines | built (`line`) | line, spline, step/stairs, ECDF, error-band outlines | -| Segments | built (`errorbar`/`stem`/`contour`) | error bars, stems, box whiskers, contour isolines | -| Rectangles | built (`histogram`; compact-bar variant for `bar`/`column`) | bar, histogram, box, violin, candlestick/OHLC, waterfall | -| Filled polygons | built (`area`) | area fill, confidence bands, stacked area | -| Grid texture | built (`density` tier, `heatmap`) | heatmap, image, 2D histogram, hexbin, filled contour | +| Segments | built (`errorbar`/`stem`/`contour`/`segments`/`box_whisker`/`box_median`) | error bars, stems, box whiskers, contour isolines | +| Rectangles | built (`histogram`/`box`/`violin`; compact-bar variant for `bar`/`column`) | bar, histogram, box, violin, candlestick/OHLC, waterfall | +| Filled polygons | built (`area`/`error_band`) | area fill, confidence bands, stacked area | +| Grid texture | built (`density` tier, `heatmap`) | heatmap, image, 2D histogram, filled contour | +| Triangle mesh | built (`triangle_mesh`, `hexbin`) | hexbin, arbitrary polygon fills, quiver/vector glyphs | Establish the primitive once; the charts sharing it are mostly wiring. +The registry is the authority on what exists. `MARK_KINDS` (`js/src/55_marks.js`) +holds eighteen kinds today — `area`, `bar`, `box`, `box_median`, `box_whisker`, +`column`, `contour`, `error_band`, `errorbar`, `heatmap`, `hexbin`, `histogram`, +`line`, `scatter`, `segments`, `stem`, `triangle_mesh`, `violin` — each with a +matching `_emit_` in `_payload.py`. `density` is a *tier* of `scatter`, not a +kind. Public builders that reuse an existing kind add no registry entry: +`hist` → `histogram`, and `step`/`stairs`/`ecdf` → `line`. + ## The two seams (and the dispatch that ties them) A chart kind `K` is defined by a kernel emitter and a client renderer, matched @@ -41,9 +50,93 @@ by the string `K` on the wire (`trace.kind`). into the `ColumnStore` and appends a `Trace`. Reuse `_ingest_xy` for the equal-length (x,y) contract; a non-xy mark ingests its own columns. - **Channels** (optional): if the mark has per-mark color/size, reuse - `channels.ship_channels(trace, sel, ship_scalar, palette)` — the same wire - shape scatter and heatmap use, so continuous/categorical color and size come - from one path. + `channels.ship_channels(trace, sel, ship_scalar, ship_u8, palette)` — the same + wire shape scatter and heatmap use, so continuous/categorical color and size + come from one path. Most kernels should call the `Figure._ship_channels(t, sel, + pw.ship_scalar, pw.ship_u8)` wrapper in `_payload.py`, which supplies + `DEFAULT_PALETTE`. + +#### Shared-geometry marks: the hexbin centers-only contract + +A mark whose cells all share one geometry ships **centers plus channels**, not +expanded vertices. `_emit_hexbin` (`python/xy/_payload.py:420-447`) ships one +(x, y) center and one scalar color value per cell; the hexagon itself is a style +constant (`hex_dx`/`hex_dy`). Each renderer expands the six-triangle fan locally, +so the wire cost stays O(cells) instead of O(cells × vertices × channels). + +Three renderers expand it today and must agree exactly: `_buildHexbinMark` +(`js/src/50_chartview.js:2038`, WebGL), `HEX_RING`/`hexbin_ring()` +(`python/xy/_svg.py:2074`, the reference ring), and `_emit_hexbin` +(`python/xy/_raster.py:1413`, raster export, consuming `hexbin_ring`). Only code +comments bind them; changing one without the others silently desynchronizes +exports from the live chart, which has already produced a CI-red payload +regression once. The rest of this section is normative — a fourth renderer +implements it without reading the other three. + +**On the wire.** The trace entry carries this and nothing else the geometry +needs: + +| Field | Meaning | +| --- | --- | +| `kind` | `"hexbin"` | +| `tier` | always `"direct"`; hexbin aggregates at build time and is never re-tiered, decimated, or view-updated — `Trace.use_density()` returns `False` for every non-`scatter` kind (`_trace.py:74-77`), and the view-update path returns no traces without it (`interaction.py:456`) | +| `x`, `y` | column indices for the cell centers, one entry per occupied cell, offset-encoded f32 (§4/§16). Shipped via `pw.ship_values`, not `pw.ship` — the centers are derived geometry with no canonical `Column` behind them, so the offset is the midpoint of their own bounds | +| `n_marks` | occupied cell count — the length of `x`/`y` | +| `n_points` | input row count before binning; reporting only | +| `color` | channel record from `_ship_channels`: `constant`, `continuous` (`buf` + `colormap`), or `categorical` (`buf` + `palette`), one value per **cell** | +| `style.hex_dx`, `style.hex_dy` | data-space cell pitch — the x/y range divided by `gridsize` (`python/xy/marks.py`) | + +There is no size channel (`_emit_hexbin` resolves one and discards it), and no +vertex, index, or per-triangle column ever ships. + +**What each renderer reconstructs.** For cell `i`, a hexagon centered on +`(x[i], y[i])` with six vertices at `center + (rx_k · hex_dx, ry_k · hex_dy)`, +where the ring fractions are `HEX_RING`: + +``` +k: 0 1 2 3 4 5 +rx: 0 +1/2 +1/2 0 -1/2 -1/2 +ry: -1/3 -1/6 +1/6 +1/3 +1/6 -1/6 +``` + +The ring is closed — vertex 5 connects back to vertex 0 — and runs +counter-clockwise in data space (y up). + +**Geometry convention.** Pointy-top hexagons: one vertex directly above and one +directly below the center, and two vertical sides at `x = center ± hex_dx/2`. +Full width is `hex_dx`; full height is `2·hex_dy/3`. Centers lie on two +interleaved lattices of pitch (`hex_dx`, `hex_dy`) offset by half a pitch on both +axes (the matplotlib hexbin lattice), but centers ship as absolute coordinates, +so a renderer never reconstructs the lattice or needs to know which of the two a +cell came from. + +**Fill.** One color per cell, applied to the whole hexagon, with fill alpha +`style.opacity × style.fill_opacity` — the same product in all three +(`_svg._fill_opacity`, `_raster._fill_opacity`, `50_chartview.js:1880`). + +Cells are unstroked, but the three renderers reach that outcome differently and +only the style keeps them agreeing: `marks.hexbin` builds its style from +`color`/`opacity`/`hex_dx`/`hex_dy` plus `styles._opacity_channels(css)`, which +whitelists `fill_opacity` and `stroke_opacity` only — so `stroke` and +`stroke_width` can never reach a hexbin trace. The exporters hardcode that +(`_raster._emit_hexbin` passes width `0.0` and a transparent stroke; +`_svg._hexbin_marks` emits no stroke attribute at all), while +`_buildHexbinMark` reads `style.stroke_width`/`style.stroke` and *would* stroke +if either appeared. Widening the style whitelist therefore diverges the live +chart from both exports; adding a stroke to hexbin means teaching the two +exporters first. A triangle renderer emits six triangles +per cell — `(center, v_k, v_(k+1 mod 6))` — replicating the cell's color across +all six; a path renderer may emit the six vertices as one closed polygon instead +(what `_svg.py` does). The coverage is identical. + +**Coordinate space.** The two exporters expand in data space and then apply the +axis scale. The WebGL client instead expands in the centers' *encoded* space: +stored = `(value − offset) × scale`, so a data-space delta scales by the column's +`scale` alone (the offset cancels) and the center columns' metas serve every +derived vertex — the vertices inherit the centers' precision center (§16). + +A new shared-geometry mark should follow the same split: constant geometry in the +style, per-cell data on the wire, one ring definition the other renderers cite. ### 2. Client — `js/src/` @@ -77,6 +170,10 @@ benchmark tracks this as part of the core 2D payload budget. - **Interaction**: pan, wheel zoom (cursor-anchored), box-zoom, modebar, reset, dblclick — all operate on the view rectangle and `_map` uniforms, mark-blind. + A new kind inherits them without writing any interaction code, but they are + not unconditional: `navigation`/`pan`/`zoom` default to on and can be turned + off per figure. [interaction.md](interaction.md) is the authority on the + switches, defaults, gesture map, and event payloads. - **Responsive sizing**: `width/height:"100%"` + ResizeObserver. - **Precision**: canonical f64 CPU-side; f32 upload offset-encoded and re-centered on deep zoom (§16). Never send f64 through the GPU path. diff --git a/docs/engineering/chart-roadmap.md b/spec/api/chart-roadmap.md similarity index 94% rename from docs/engineering/chart-roadmap.md rename to spec/api/chart-roadmap.md index a92fba08..575669d7 100644 --- a/docs/engineering/chart-roadmap.md +++ b/spec/api/chart-roadmap.md @@ -41,7 +41,7 @@ Beyond the mark set, four capability layers now ship on `main`: (hex/`rgb()`/`hsl()`/named colors, lengths, numbers) parse strictly, browser-resolved forms (`var()`/`oklch()`/`calc()`) pass through, and a malformed value raises instead of rendering a silently wrong chart (see - `docs/engineering/styling.md` § Validation). + `spec/api/styling.md` § Validation). - **Static export:** `fig.to_svg(...)` (pure-Python, screen-bounded vector — a 10M-point line exports in ~4 ms / ~58 KB) and `fig.to_png(...)` (a browser-free native Rust rasterizer by default, ~50× faster than the @@ -58,7 +58,7 @@ Beyond the mark set, four capability layers now ship on `main`: `@reflex_xy.figure` computed-var decorator binds figure builders to Reflex state so dependency tracking drives rebuilds and republishes, and a static payload tier compiles zero-backend charts to asset files. See - `docs/integrations/reflex.md` and `docs/engineering/design/reflex-integration.md`. + `docs/integrations/reflex.md` and `spec/design/reflex-integration.md`. **Prototyped (not on `main`):** candlestick + OHLC marks and a finance overlay/indicator layer (volume pane, SMA, VWAP, Bollinger, RSI, MACD, @@ -131,7 +131,7 @@ not fall out of sight. | 24 | Categorical distributions | strip, swarm, beeswarm, boxen, rug | Planned | Seaborn-style stats breadth; requires jitter/packing and distribution summaries. | | 25 | Step/stairs/stem/lollipop | stairs, step area, stem, lollipop | Implemented core | Step/stairs remain compact line inputs; stems reuse instanced segments and points. | | 26 | Slope/bump/dumbbell | slopegraph, bump chart, dumbbell, connected dot plot | Planned | Popular editorial/business comparisons; composition of line + point + labels. | -| 27 | Timeline/Gantt/event charts | event timeline, bar range, Gantt, milestone chart | Planned later | Useful for operations/product; more UI/axis polish than rendering complexity. | +| 27 | Timeline/Gantt/event charts | event timeline, bar range, Gantt, milestone chart | Partially implemented in `xy.pyplot` | `eventplot(positions, orientation=, lineoffsets=, linelengths=, linewidths=, colors=, linestyles=)` renders event timelines through instanced segments; `broken_barh(xranges, yrange)` renders bar ranges as horizontal `bar` marks with a per-range base and categorical y support. Gantt/milestone composition and date-axis polish remain planned later. | | 28 | Calendar charts | calendar heatmap, contribution graph, daily cohort grid | Planned later | Product analytics and ops compatibility; grid + date-axis specialization. | | 29 | Parallel coordinate/category | parallel coordinates, parallel categories, alluvial-lite | Planned later | Present in Plotly/ECharts; useful for high-dimensional EDA. | | 30 | Sankey / alluvial | Sankey, alluvial, dependency wheel | Planned later | Important flow chart, but requires layout and interaction work. | @@ -144,7 +144,7 @@ not fall out of sight. | 37 | Statistical evaluation | ROC, precision-recall, lift, calibration, Manhattan, volcano | Planned later | Mostly composed line/scatter variants with domain helpers. | | 38 | Composition/part-to-whole extras | waffle, mosaic/Mekko, variwide, packed bubble, Venn/Euler | Planned later | Useful compatibility charts; mostly layout algorithms plus rectangles/circles. | | 39 | Decorative/compatibility series | pictorial bar, item chart, text marks, image markers, custom symbol scatter | Planned later | ECharts/Highcharts-style polish and compatibility after core performance work. | -| 40 | Tables and table-adjacent views | table, pivot-like summary, annotated table | Deferred dashboard surface | Plotly includes tables, but they are dashboard UI, not core chart rendering. | +| 40 | Tables and table-adjacent views | table, pivot-like summary, annotated table | Partially implemented in `xy.pyplot` | `table(cellText=, cellColours=, cellLoc=, colWidths=, rowLabels=, colLabels=, loc=, bbox=)` tessellates cells into `triangle_mesh` geometry with per-cell colors, rules, and text, so it renders through the core mark path rather than as dashboard DOM. Pivot-like summaries and annotated-table depth remain deferred. | ### P1 - Add next @@ -195,7 +195,7 @@ depth: strip/swarm/boxen/rug distributions, regression diagnostics, richer | 23 | Sankey / alluvial / dependency wheel | Common flow visualization in BI and systems analysis. | Layout and interaction are the hard parts. | | 24 | Network / tree / org / dendrogram / arc | Graph and hierarchy compatibility. | Requires graph layout algorithms and selection semantics. | | 25 | Gauge / bullet / indicator | Dashboard/KPI compatibility. | Mostly chrome and layout, not large-array rendering. | -| 26 | Table-adjacent views | Table, pivot-like summary, annotated table. | Useful in dashboards, but not a chart-rendering primitive. | +| 26 | Table-adjacent views | Table, pivot-like summary, annotated table. | Basic `table` is implemented through `xy.pyplot` as tessellated cell geometry; pivot-like summary and annotated-table depth remain deferred. | ### P5 - Scientific, coordinate-system, and obscure compatibility @@ -311,7 +311,7 @@ pattern); line/area take `curve="smooth"` (monotone-cubic); scatter takes 17 renderer-backed `symbol` glyphs plus point strokes. Mark colors flow through the same `--chart-*` tokens as the chrome, so a theme change re-resolves marks and chrome together. The full matrix and per-mark support -table live in [`docs/engineering/styling.md`](styling.md). Static SVG export reproduces all +table live in [`spec/api/styling.md`](styling.md). Static SVG export reproduces all of it (gradients → ``, smooth curves → exact cubic Béziers, etc.) with two documented approximations (area mark-space gradient uses the bbox; `var()` colors fall back to the mark color — no DOM). @@ -378,14 +378,20 @@ not re-implementing shipped primitives. as a durable release asset (#97). The docs site is live, and the reflex-xy adapter is on `main` — so release/distribution correctness and the post-launch bug backlog are the current gate, not shipping. The interaction - correctness issues (heatmap hover kernel crash, box-zoom view collapse, - double-click blanking a dense Reflex scatter, reflex-xy static chart crash, - FacetChart CSS leak) come before any new chart family. + correctness issues (box-zoom view collapse, double-click blanking a dense + Reflex scatter, reflex-xy static chart crash, FacetChart CSS leak) come + before any new chart family. The heatmap hover kernel crash is fixed + (PR #106): the pick handler takes a dedicated grid-trace branch that returns + row/col plus the cell value instead of indexing the edge-only x/y arrays. 2. **Reflex-first reactive API — shipped** (PR #55, `python/reflex-xy/`; see - the capability-layer summary near the top of this doc). Remaining depth is - adapter polish: install/packaging docs from a clean environment, linked - facet interactions, and first-class pan/zoom controls surfaced through the - component API. + the capability-layer summary near the top of this doc). Linked facet + interactions shipped in PR #105: `xy.facet_chart(...)` takes + `link=True|"both"|"x"|"y"` for runtime-linked panel axes and + `link_select=True` to echo data-space selections across panels. First-class + pan/zoom controls shipped in PR #103: `xy.interaction_config(...)` exposes + `pan` and `zoom` switches alongside the existing `navigation` flag. + Remaining depth is adapter polish: install/packaging docs from a clean + environment. 3. **Statistical compatibility depth.** Add strip/swarm/boxen/rug, regression/diagnostic helpers, richer density hover/readout, and scatter-matrix/joint-plot composition on the shipped primitives. diff --git a/spec/api/export.md b/spec/api/export.md new file mode 100644 index 00000000..1a8538fa --- /dev/null +++ b/spec/api/export.md @@ -0,0 +1,218 @@ +# Static export + +How a figure becomes bytes. One unified entry point across five image formats, +a deterministic engine choice, and a browser-free default path. The renderers +consume the same decimated wire payload the browser client consumes +(`build_payload`), so export is **screen-bounded**: source point count does not +enter the output size. Styling fidelity and the documented static-render +approximations live in [`styling.md`](styling.md); this document is the API and +routing contract. + +## 1. Public surface + +| Entry point | Location | Notes | +|---|---|---| +| `Figure.to_image(format="png", ...)` | `python/xy/_figure.py:1310` | bytes | +| `Figure.write_image(path, format=None, ...)` | `python/xy/_figure.py:1346` | atomic write, extension-inferred | +| `xy.export.to_image(fig, ...)` | `python/xy/export.py:1006` | the implementation both methods delegate to | +| `xy.export.write_image(fig, path, ...)` | `python/xy/export.py:1069` | | +| `xy.write_images(figs, paths, ...)` | `python/xy/export.py:533` | batch, §8 | +| `xy.export_config(...)` | `python/xy/components.py:2301` | declarative defaults, no I/O | +| `xy.Engine` | `python/xy/export.py:27` | §3 | + +`to_image` / `write_image` are **methods**, not package-level functions: +`xy.__init__` re-exports only `Engine`, `export_config`, and `write_images` +from the export module. Reach the free functions as `xy.export.to_image(fig, +...)`. + +`Chart` mirrors both methods (`components.py:3014`, `components.py:3054`) and +adds one behavior the `Figure` methods do not have: omitted +`width`/`height`/`scale`/`background`/`quality` fall back to the chart's +`export_config` via `_export_defaults` (`components.py:2981`). `FacetChart` +(`components.py:3740`, `components.py:3767`) and `FacetGrid` (`facets.py:354`, +`facets.py:430`) mirror the same format matrix but expose no `width`/`height` +and no `export_config` defaulting — grid geometry is fixed by its panels. + +Legacy per-format surfaces remain: `to_html` (interactive standalone document), +`Figure.to_png` (`engine=Engine.default` by default), `_svg.to_svg` (carries +`id_prefix=` for composing several exports into one document). `to_png` routes +through `_png_engine`, which has no `auto` case — it maps any `Engine` that is +not `Engine.default` to the browser, so `Engine.auto` there means Chromium, not +"choose". Use `to_image` when you want the `auto` policy. + +`ExportConfig` is pure description. Beyond seeding the Python defaults above it +governs the client modebar's download menu (formats and order, filename +basename); `pdf` and `html` are accepted in `formats=` but are Python-side only +and are skipped in the browser menu (`js/src/53_interaction.js:1020-1038`, where +`EXPORT_ITEMS` enumerates only png/jpeg/webp/svg/csv and the build loop drops any +configured name with no entry). With no `export` spec the menu falls back to +`["png", "svg", "csv"]`; an explicit empty list hides the download items. + +There is no Reflex-side export API. `python/reflex-xy/` renders kernel-lessly in +the browser; export runs on the composed chart object in Python. + +## 2. Formats and backends + +`IMAGE_FORMATS = ("png", "jpeg", "webp", "svg", "pdf")`, with `jpg` aliased to +`jpeg`. HTML is deliberately not an image format: `to_image("html")` raises, +while `write_image("chart.html")` routes to `to_html`, rejecting every +raster-only option that was passed non-default. + +| Format | Native backend | Chromium backend | +|---|---|---| +| PNG | `_raster.to_png` → Rust rasterizer (`src/raster.rs`), encoded by the fused Rust path or `_png.encode` | `Page.captureScreenshot` | +| JPEG | `_raster.to_rgba` → `_jpeg.encode` (pure numpy/stdlib baseline JFIF, 4:4:4) | `Page.captureScreenshot` | +| WebP | `_raster.to_rgba` → `_webp.encode` (pure numpy/stdlib VP8L, **lossless only**) | `Page.captureScreenshot` (lossy) | +| SVG | `_svg.to_svg` (pure Python renderer over the wire payload) | none — SVG is native-only | +| PDF | `_svg.to_svg` → `_pdf.svg_to_pdf` (closed SVG subset → single-page vector PDF) | `Page.printToPDF` | + +`_png.encode` auto-selects an indexed-palette PNG (color type 3 + `tRNS`) when +the image has ≤256 distinct RGBA colors. `optimize=True` selects this +size-oriented path; `optimize=False` (default) takes the fused Rust encode. + +`_pdf.py` accepts only the SVG that `_svg.py` emits and raises +`ValueError("unsupported SVG feature: ...")` on anything else, so generator +drift fails loudly rather than rendering wrong. Text stays text (base-14 +Helvetica, WinAnsiEncoding, `?` for out-of-range characters); density and +heatmap layers embed as bounded rasters. + +`quality` (1–100, default 90) applies to JPEG and to **Chromium** WebP. +Requesting `quality` for native WebP is an error, not a silent no-op — the +native encoder is lossless by policy (`_validated_quality`). + +## 3. Engine selection + +``` +Engine.auto # default for to_image/write_image/write_images +Engine.default # native (browser-free) +Engine.chromium # installed Chrome/Chromium/Edge/chrome-headless-shell +``` + +`_resolve_image_engine` (`export.py:800`) resolves to `"native"` or `"browser"`: + +- `auto` → **native for every format**. Every format is natively supported, so + the browser-free path is always the fast path. Size, point count, and trace + count do not enter this decision. +- `auto` + `custom_css is not None` → **browser**. Utility-class CSS needs a + real cascade; the native renderers have no CSS engine. +- `native` + `custom_css` → `ValueError`. +- `browser` + `svg` → `ValueError` (a screenshot cannot produce vector SVG). +- Deprecated string values `"native"`, `"chromium"`, `"browser"` still resolve, + with a `DeprecationWarning` (`_png_engine`). + +Browser discovery (`find_browser`) searches `XY_BROWSER`, the legacy +`XY_CHROMIUM`, `PATH` over nine known executable names, then platform install +locations. An explicit non-`"auto"` value is treated as a path or executable +name and is never silently replaced with a different browser. With no browser +found, browser-resolved export raises `RuntimeError` naming `XY_BROWSER` — it +does **not** fall back to native, because the caller asked for browser fidelity +(or passed `custom_css`, which native cannot honor). + +## 4. Determinism + +Native export is deterministic: `to_image(fig, fmt) == to_image(fig, fmt)` for +all five formats (`tests/test_image_export.py:80`). No wall-clock stamps enter +any output — `_svg.py`'s `datetime` use is time-axis tick math on data values in +UTC, and `_pdf.py` writes no timestamps or generated ids, with stable object +numbering and a byte-accurate xref table. + +Browser export is deterministic only to the extent the browser is. `gl="software"` +(the default) pins SwiftShader so rasterization does not depend on the host GPU; +`gl="hardware"` trades that for speed on large direct-mode payloads. + +## 5. Dimensions and scale + +`_export_dimensions` resolves `width`/`height` from the explicit arguments, else +the figure's own integer size, else `800×500` — a fluid `"100%"` figure has no +concrete size, and a raster needs one. Values must be positive integer pixel +counts. + +`scale` is the device-pixel ratio for raster output (default `2.0`): native +rasters multiply canvas pixels, Chromium sets `deviceScaleFactor`. It is ignored +by SVG and PDF, which are resolution-independent — `render_pdf` pins +`deviceScaleFactor: 1.0` and maps the CSS pixel box at 96 px/in ↔ 72 pt/in. + +## 6. Background, and the theme-parity gap + +`_validated_background` is the shared policy. `None`/`"auto"` keeps each +renderer's default backdrop — opaque white for raster and browser output, +transparent for SVG and PDF. A CSS color (validated against a conservative +`` shape that cannot escape the declaration it is interpolated into) +paints one backdrop consistently across formats. `"transparent"` is valid +wherever alpha exists and is **rejected for JPEG**, which has no alpha channel, +rather than silently flattened. + +An explicit background replaces the *entire* painted backdrop — canvas underlay, +`theme(background=)` figure patch, and the `--chart-bg` plot fill — so the +requested color is what shows regardless of chart theme +(`_svg.apply_export_background`, mirrored for browser capture by +`_background_css`, which needs `!important` to beat the root's inline theme +style). + +**Open gap: screen/export theme parity.** Kernel-side export resolves color +tokens statically from `spec["dom"]["style"]` — the tokens the chart declared in +Python (`_svg._resolve_static_css_vars`). The live client instead reads +*computed* CSS off the chart root (`readTheme`, `js/src/20_theme.js:51`) and +re-reads it on scheme change or a class mutation via `refreshTheme()` +(`js/src/50_chartview.js:4037`). That refresh is **local to the browser**: no +theme snapshot ever travels back. The eight client→kernel request types are +enumerated exhaustively in [wire-protocol.md](../design/wire-protocol.md) §2, and +their fields are view geometry, screen dimensions, trace/vertex indices, +sequence counters, and a `source` label. None of them carries style. + +The consequence is concrete. When a theme comes from the host page (an app +stylesheet, a `.dark` class on an ancestor, a `prefers-color-scheme` flip) the +kernel does not know about it, and `fig.to_image(...)` exports the Python-declared +theme, not what the user is looking at. Themes declared through `xy.theme(...)` +or the chart's own `style` do export faithfully, and the *client-side* modebar +download does match the screen — it snapshots the computed `--chart-*` tokens +(`styling.md` § Standalone HTML). Do not read the client-side match as +end-to-end parity: the two download paths can disagree on the same chart. + +Closing this requires a theme snapshot on the comm channel and a bump of the +message contract; nothing in the current protocol carries it. + +## 7. The `--no-sandbox` auto-fallback + +Chromium launches sandboxed by default. Both browser paths silently downgrade on +failure: + +- `html_to_png` (`export.py:509-526`): if the sandboxed run produces no + screenshot, it rebuilds the argv with `--no-sandbox` inserted and re-runs + before raising. The final error reports both attempts. +- `_browser_session` (`export.py:926-931`): retries + `ChromiumSession(..., sandbox=False)` on `ChromiumError`. + +So `--no-sandbox` can appear without the caller requesting it, on input that +`html_to_png` accepts as arbitrary HTML. This is a known, accepted residual risk +taken to keep CI and container rasterization working where the sandbox cannot +initialize — see [XY-SEC-2026-03 and its 2026-07-20 status +note](../process/security-audit-2026-07-06.md#status-as-of-2026-07-20-xy-sec-2026-03). The +pending follow-up is to make the fallback opt-in, or at minimum warn, so a +sandbox loss is observable. `sandbox=False` remains the explicit escape hatch +for trusted HTML. + +## 8. Batch export + +`write_images(figs, paths, ...)` exports many figures — mixed formats included — +through one amortized pipeline. Per-file format comes from the path extension, +or from `formats=` (one string for all, or one per path). `figures=`/`files=` +are keyword aliases for the positional pair, and composed charts (anything with +a `.figure()`) are accepted directly, with their `export_config` defaults +filling that chart's omitted options. + +Two properties are the point of the API: + +- **Plan-then-write.** The whole plan — engine resolution, quality, background, + per-chart defaults — is resolved before any I/O, so a bad argument fails the + batch up front instead of after a partial export. +- **One browser.** Every browser-resolved file in the batch shares a single + persistent `ChromiumSession` over CDP (`_chromium.py`) instead of paying + browser startup per figure. The session is opened lazily on the first + browser-resolved file and closed in a `finally`. Native files never launch a + browser at all. + +Writes are atomic per file (same-directory temp file, fsync, `os.replace`), so a +reader never observes a partial image. Failure mid-batch is not transactional: +files already written stay on disk. The return value is the list of written +byte strings, in input order. diff --git a/spec/api/interaction.md b/spec/api/interaction.md new file mode 100644 index 00000000..0bd904a3 --- /dev/null +++ b/spec/api/interaction.md @@ -0,0 +1,194 @@ +# Interaction and events + +This document is the authority on which browser interactions exist, which are +configurable, and what every event payload contains. Shipped in PRs #103/#105. +Where another spec describes an interaction as "free" or "always on" (e.g. +[chart-kind-contract.md](chart-kind-contract.md) §"What you get for free"), +this document governs. + +Implementation: `python/xy/components.py` (`interaction_config`, `Chart`, +`FacetChart`), `python/xy/_figure.py` (`set_interaction`, +`_interaction_spec`), `python/xy/channel.py` (kernel-side dispatch), +`js/src/53_interaction.js` (gestures, modebar, view state machine), +`js/src/50_chartview.js` (flag resolution, link channel), +`python/reflex-xy/reflex_xy/` (Reflex event props). + +## 1. How a switch resolves + +`Figure.interaction` starts empty. `_interaction_spec` serializes only keys +that were explicitly set, so an unset switch is *absent* from the wire spec, +not `False`. The client resolves each read through +`_interactionFlag(name, fallback = false)`: an absent key takes the call +site's fallback; a present key is enabled only when it is exactly `true`. +The fallback therefore *is* the default, and it differs per switch. + +`Chart` writes the interaction dict in three passes: chart-level keyword +arguments, then any `xy.interaction_config(...)` children in order, then a +final pass derived from the attached handlers (`components.py:2736-2740`). +That last pass is not a defaulting step — it overwrites. Passing `on_hover=` +together with `hover=False` yields `hover=True`. + +## 2. `xy.interaction_config(...)` + +`python/xy/components.py:2424`. All switches are `Optional[bool]` and default +to `None` (leave unset). "Default" below is the client-side behavior when the +switch is never set. + +| Switch | Default | Effect when disabled | +| --- | --- | --- | +| `hover` | off | Suppresses the `xy:hover` and `xy:leave` DOM events. The tooltip still renders and the kernel `pick` round-trip still runs; only the event surface is gated. | +| `click` | off | `_click` returns immediately: no `xy:click` event and no `click` message to the kernel. | +| `select` | on | Suppresses the `xy:select` event and the `select_clear` message, removes the modebar Selection menu, and (with `brush`) disables shift-drag. | +| `brush` | on | Same conjunction: `canBrush = brush && select` (`53_interaction.js:115`) gates every selection drag and the modebar Selection menu. | +| `crosshair` | off | The two guide elements are created only when the flag is true, at init (`53_interaction.js:80`). Not togglable after mount. | +| `navigation` | on | Master gate on pointer-drag pan, wheel zoom, box-zoom drags, and dblclick reset. | +| `pan` | on | Plain-drag pan is ignored and the modebar Pan button is not built. Requires `navigation` as well. | +| `zoom` | on | Wheel zoom, box zoom, dblclick reset, and the modebar zoom menu are all ignored. Requires `navigation` as well. | +| `view_change` | off | Suppresses the `xy:view_change` event and the `view_change` kernel message. Linked-figure broadcast is independent (§4). | +| `link_group` | unset | See §4. | +| `link_axes` | `("x", "y")` | See §4. | + +`link_select` is the one linking switch `interaction_config` does not expose: +the table above carries `link_group` and `link_axes`, and +`Figure.set_interaction` adds `link_select` (`_figure.py:257`). It is +reachable there and via `facet_chart(link_select=True)` (§6). Its default is +off. + +Selection additionally requires a point-pickable trace: `canBrush` and the +modebar Selection menu both test `this._pickable`, which is true only when +some GPU trace has `pointPick` and is not an undrilled density tier. + +## 3. Event surface + +Every event is a `CustomEvent` on the chart root (`bubbles`, `composed`), +named `xy:`. `view` is `_eventView(source)` — +`{x0, x1, y0, y1, source}` in data coordinates. + +| Event | Detail | +| --- | --- | +| `xy:hover` | `{row, trace, index, view}`; the kernel's exact-value reply re-dispatches with `exact: true`. | +| `xy:leave` | `{view}` with `source: "leave"`. | +| `xy:click` | `{x, y, view, row, trace, index}`; `row`/`trace`/`index` are `null` when the click hit no mark. | +| `xy:brush` | `{range: {x0, x1, y0, y1}, view}` for box/axis-range drags, or `{polygon: [[x, y], …], view}` for lasso. | +| `xy:select` | `{total, view}` — the resolved count after the kernel replies, or `total: 0` on clear. | +| `xy:view_change` | `{x0, x1, y0, y1, source}`, coalesced to one dispatch per animation frame. | + +`source` on a view event is `"pan"` for drag-pan (`53_interaction.js:158`), +`"linked"` for a view applied from a link-group peer, and `"view"` for +everything else (wheel, box zoom, zoom in/out, reset, programmatic). The +kernel coerces it with `str(content.get("source", "view"))`. + +Three further events report GL context lifecycle rather than user intent. +They carry no `view`, and no interaction switch gates them: + +| Event | Detail | +| --- | --- | +| `xy:context_lost` | `{loss_count}` — the canvas lost its WebGL2 context (`50_chartview.js:693`). | +| `xy:context_restored` | `{loss_count, restore_count}` — a live frame is back (`:775`). | +| `xy:context_restore_failed` | `{loss_count, message}` — recovery gave up; the root is replaced with an error string (`:761`). | + +Those nine are the whole `xy:` surface — every one goes through +`_dispatchChartEvent` (`50_chartview.js:451`), and there is no other +`CustomEvent` dispatch in `js/src/`. + +Kernel-side callbacks (`python/xy/channel.py`), wired through +`Chart(on_hover=…, on_click=…, on_brush=…, on_select=…, on_view_change=…)`: + +- `on_hover(row)` / `on_click(row)` — the picked row dict resolved by + `fig.pick(trace, index, drill_seq)`. Not called when the pick misses. +- `on_view_change(view)` — `{"x0", "x1", "y0", "y1", "source"}`, after + `normalize_window(..., require_area=False)` (`channel.py:216-233`). +- `on_brush(brush)` — `{"x0", "x1", "y0", "y1"}` for a range select, or + `{"polygon": [[x, y], …]}` for a lasso. +- `on_select(Selection)` — canonical row indices per trace, not JSON. + `on_brush` always fires before `on_select` for the same gesture; that + ordering is an invariant and is tested. + +Reflex props (`python/reflex-xy/reflex_xy/component.py:82-85`) are the live-mode +mirror: `on_point_hover(row)`, `on_point_click(row)`, +`on_select_end({total, x0, x1, y0, y1, polygon, cleared})`, and +`on_view_change(msg)`. `on_view_change` is resolved in the browser and never +reaches the kernel — `XYChart.jsx` intercepts the outgoing `view_change` +message and invokes the handler directly, because the socket namespace +registers no Python-side view callback. + +## 4. Linking + +`link_group` names a `BroadcastChannel` opened as `` `xy:${group}` `` +(`50_chartview.js:487`). Every view instance mints a random `_linkedSource` +id and drops messages carrying its own id, so a figure never re-applies its +own broadcast. Because the transport is `BroadcastChannel`, linking spans +tabs and windows of the same origin, and is a no-op where the constructor is +unavailable. + +View messages carry the emitting figure's `_eventView` detail. A receiver +copies only the axes listed in `link_axes` (filtered to `x`/`y`; anything +else is dropped) onto its current view, rejects non-finite results, and +applies the result with `animate: false`, `source: "linked"`, and +`broadcast: false` so the update does not echo. A receiver with both `pan` +and `zoom` disabled ignores view messages entirely. + +Broadcast is not gated on `view_change`: `_emitViewChange` runs whenever +either `view_change` is on *or* a link channel exists, and then gates the DOM +event and the kernel message separately. Linked figures therefore stay in +sync without emitting events. + +Selection messages are gated on `link_select` at both ends. The payload is +one of `{clear: true}`, `{range: {x0, x1, y0, y1}}`, or `{polygon: [...]}`; +the receiver applies it locally with `dispatch: false`, so a linked selection +updates the peer's rendering without re-emitting its own events. + +## 5. Gesture map + +From `js/src/53_interaction.js`. `shift` is the only modifier key the +renderer reads anywhere in `js/src/`. + +| Gesture | Action | Requires | +| --- | --- | --- | +| Plain drag | Pan (`dragMode === "pan"`) | `navigation` and `pan` | +| **Shift**-drag | Box select, overriding the current drag mode (`53_interaction.js:117`) | `brush`, `select`, `_pickable` | +| Drag in `select` / `select-lasso` / `select-x` / `select-y` mode | That selection shape | `brush`, `select`, `_pickable` | +| Drag in `zoom` mode | Box zoom | `navigation` and `zoom` | +| Wheel | Cursor-anchored zoom, factor `1.0015 ** deltaY`; `preventDefault` | `navigation` and `zoom` | +| Double click | Clear selection, animate to the home view | `navigation` and `zoom` | +| Click without drag | Pick; a drag past threshold sets `_ignoreNextClick` and swallows the click | `click` | +| Pointer down on a lasso vertex handle | Drag that vertex; re-runs the selection on release | an existing lasso | +| `ArrowRight`/`ArrowDown`, `ArrowLeft`/`ArrowUp`, `Home`, `End` | Keyboard traversal of pickable points in data order | pickable points | +| `Escape` | Dismiss the readout and invalidate the in-flight pick | — | + +A box gesture counts as moved past 3 px in either axis; a lasso needs at +least 3 sampled vertices, sampled at 3 px spacing and capped at 2048. In +`select-x` the y bounds are replaced with the full current view and in +`select-y` the x bounds are, so those modes brush one axis. + +The modebar exposes the same actions: Pan; a zoom menu with Zoom In (×0.5), +Zoom Out (×2), Box Zoom, and Reset View; and a selection menu with Box +Select, Lasso Select, X Range, and Y Range. Each menu is built only when its +flags allow it, and the whole bar is hidden when it does not fit inside the +plot box — hiding is a fit state, not a capability change, and wheel/drag +gestures keep working without it. + +## 6. Facet-level linking + +`facet_chart(..., link=…, link_select=…)` (`python/xy/components.py:4480`; normalization in `FacetChart.__init__`, +`components.py:3514`). +`link` accepts `True` (normalized to `"both"`), `False`/`None`, `"x"`, `"y"`, +or `"both"`; anything else raises. `link_select` is a strict bool. + +`link` expands to `linked_dims`, and each linked dimension is forced into the +shared-domain set alongside `share_x`/`share_y` — a linked axis must start +from the same domain or the first interaction would make panels jump between +incomparable views. When `linked_dims` is non-empty *or* `link_select` is +set, every panel figure receives a common `link_group`: an existing panel +group id if one is present, otherwise `xy-facet-<8 hex>`. `link_axes` is set +to exactly `linked_dims`, so `link_select=True` with `link=None` produces an +empty `link_axes` — panels share selections and nothing else. `link_select` +is written straight into each panel's interaction dict. + +## 7. Unconditional behavior + +Not configurable through any switch: tooltip rendering and the kernel `pick` +round-trip on hover; keyboard point traversal and the live-region readout; +lasso vertex editing on an existing lasso; selection overlay rendering; +modebar presence (subject only to the fit check); and view clamping to axis +bounds. Everything in the §2 table is configurable; nothing else is. diff --git a/docs/engineering/styling.md b/spec/api/styling.md similarity index 76% rename from docs/engineering/styling.md rename to spec/api/styling.md index 9d7810c1..88ac9b4b 100644 --- a/docs/engineering/styling.md +++ b/spec/api/styling.md @@ -5,11 +5,11 @@ restyle the whole chart with plain CSS, attribute selectors, Tailwind, or per-slot inline styles — and your styles always win, without `!important`. This engineering guide explains the implementation contract. The public, -task-oriented references are [Styling](../styling/index.md), -[Component Variations](../styling/component-variations.md), and -[Mark Styles](../styling/mark-styles.md). For the API shapes see -[reflex-shaped-api.md](design/reflex-shaped-api.md); for the render internals -see [renderer-architecture.md](design/renderer-architecture.md). +task-oriented references are [Styling](../../docs/styling/index.md), +[Component Variations](../../docs/styling/component-variations.md), and +[Mark Styles](../../docs/styling/mark-styles.md). For the API shapes see +[reflex-shaped-api.md](../design/reflex-shaped-api.md); for the render internals +see [renderer-architecture.md](../design/renderer-architecture.md). ## The five ways to style @@ -47,7 +47,7 @@ through untouched. In Reflex, Tailwind utilities require `rx.plugins.TailwindV4Plugin()`. Complete literal classes emitted into Reflex's generated JSX work with the plugin's normal scan paths; the original Python or Markdown path does not need to be -added. See the public [Chrome Slots](../styling/chrome-slots.md) guide for the +added. See the public [Chrome Slots](../../docs/styling/chrome-slots.md) guide for the standalone-export and dynamic-class boundaries. ## Rendered marks: standard CSS vocabulary @@ -148,6 +148,43 @@ xy.x_axis( ) ``` +### Axis ticks and label formatting + +Tick placement is computed in f64 on the CPU (§16), never through f32, and is +selected per scale kind with a target of 6 ticks. Every generator caps its +output at 200 ticks. + +| Scale kind | Rule | +| --- | --- | +| linear | Step is the nice step for `(hi - lo) / target` — the smallest of `1, 2, 2.5, 5, 10` times the decade magnitude that covers the rough step. Ticks start at the first multiple of the step at or above `lo`. | +| log | Decade ticks from `floor(log10(lo))` to `ceil(log10(hi))`; multipliers `1, 2, 5` when the decade span is small, `1` alone otherwise. Only powers of ten are labeled, thinned so roughly `target` labels remain. Non-positive bounds yield no ticks. | +| category | Every `ceil(visible / target)`-th category index in view. | +| time | The smallest step in a fixed ladder from 1 ms through 14 days that covers the rough step. Above 14 days per tick, calendar ticks land on UTC month boundaries with a month step from `1, 2, 3, 6, 12, 24, 60, 120`. | + +`xy.x_axis(format=...)` and `xy.y_axis(format=...)` take a format string whose +grammar depends on the axis kind. Both are deliberately small subsets, not full +d3-format or strftime, and neither raises on a spec it does not understand — +but they fail differently, and only the numeric grammar falls back. + +- **Numeric axes** accept `.Nf` (fixed decimals), `,.Nf` (fixed decimals with + locale group separators, via the runtime's default locale), and either form + with a trailing `%`, which multiplies the value by 100 and appends the sign — + for example `.2f`, `,.0f`, `.1%`. The trailing `f` is optional. Any other + string **falls back**: `fmtNumberSpec` returns `null` + (`js/src/30_ticks.js:168`) and `fmtAxis` takes its `|| fmtLinear(...)` branch + (`:209`), so the axis silently reverts to the automatic formatter. On a log + axis, a value in `(0, 1)` that the spec would render as `"0"` falls back the + same way. +- **Time axes** accept a strftime subset of exactly `%Y %m %d %H %M %S %b %B`. + All fields are **UTC**; `%b`/`%B` are English month names. A time spec + **never** falls back: `fmtTimeSpec` (`js/src/30_ticks.js:180-200`) + substitutes the tokens it knows and copies every other character through + verbatim, so it always returns a string and the `|| fmtTime(...)` branch at + `:204` is unreachable. An unrecognized `%` token such as `%y` therefore + renders literally as `%y`. The automatic time formatter is reached only when + `format` is absent or not a string. +- **Category axes** ignore `format=` and render the category label. + ## Slot reference Every element below is rendered with `data-xy-slot=""`, so @@ -233,18 +270,26 @@ Set them on `.xy` or any ancestor: | Token | Themes | Default | | --- | --- | --- | | `--chart-bg` | Plot-rect background only (`theme(plot_background=)`, mpl `axes.facecolor`) | transparent | -| `--chart-text` | Title, tick/axis titles, legend, annotation labels | inherited text (canvas labels: `currentColor` @ 85%) | +| `--chart-text` | Title, tick/axis titles, legend, annotation labels, modebar glyphs | inherited text (canvas labels: `currentColor` @ 85%) | | `--chart-grid` | Grid lines (canvas) | `currentColor` @ 14% | -| `--chart-axis` | Axis lines (canvas), modebar glyphs | `currentColor` @ 55% | +| `--chart-axis` | Axis lines (canvas) | `currentColor` @ 55% | | `--chart-tooltip-bg` / `--chart-tooltip-text` | Tooltip | `rgba(20,24,33,.92)` / `#fff` | | `--chart-legend-bg` | Legend background | `rgba(128,128,128,.08)` | | `--chart-badge-bg` / `--chart-badge-text` | Reduction badges | `rgba(255,255,255,.82)` / `#0f172a` | -| `--chart-modebar-bg` / `--chart-modebar-active` | Modebar / active button | `rgba(255,255,255,.78)` / `rgba(128,128,128,.22)` | +| `--chart-modebar-bg` / `--chart-modebar-active` | Modebar / active button | `rgba(255,255,255,.78)` / `rgba(128,128,128,.2)` (light; see below) | | `--chart-selection` / `--chart-selection-fill` | Box-select rectangle | `rgba(90,140,240,.9)` / `…,.15)` | | `--chart-zoom-selection` / `--chart-zoom-selection-fill` | Box-zoom drag rectangle | `rgba(120,120,120,.9)` / `…,.12)` | | `--chart-crosshair` | Crosshair lines | `rgba(15,23,42,.42)` | | `--chart-annotation-text` | Annotation label color | falls back to `--chart-text` | | `--chart-cursor` / `--chart-cursor-pan` | Plot cursor (box-zoom / pan) | `crosshair` / `grab` | +| `--chart-focus` | Keyboard focus ring on the plot canvas and modebar buttons | `#2563eb` | + +The modebar defaults are **scheme-aware**: a `.dark` class on the chart root or +any ancestor flips the internal fallbacks to `rgba(37,42,52,.9)` / +`rgba(255,255,255,.16)`. The public `--chart-modebar-*` tokens override both +schemes; the modebar's border and shadow have no public token and are internal +`--xy-modebar-*` defaults only. `--chart-focus` is likewise not carried into +client-side PNG/SVG export, which snapshots the other `--chart-*` tokens. The **figure background** (matplotlib's `figure.facecolor` — the whole card including margins, title, and tick labels) is not a token: `theme(background=)` @@ -257,7 +302,11 @@ The compact toolbar appears while the chart is hovered or one of its controls has keyboard focus. Drag its grip to move it within the chart. Zoom and selection modes are grouped into menus; completed lasso selections expose up to 16 adaptively simplified handles that can be dragged to refine the selected -range. The grip's menu exports PNG, SVG, or the chart's resident data as CSV. +range. The grip's menu defaults to PNG, SVG, and the chart's resident data as +CSV. `xy.export_config(formats=[...])` governs which of `png`, `jpeg`, `webp`, +`svg`, and `csv` appear and in what order; `pdf` and `html` are Python-side +formats and are skipped in the client menu. An explicit empty list hides the +download items, leaving the grip a drag handle only. Client PNG and SVG export snapshot the chart's computed `--chart-*` tokens, text color, and font styles so themes inherited from a host application are preserved in the downloaded image. @@ -356,7 +405,7 @@ fill with an opaque outline, keep whole-mark opacity at `1` and set ### Scatter markers — `symbol`, `stroke`, `stroke_width` `scatter` markers take any of the 17 renderer-backed symbols listed in the -public [Mark styles](../styling/mark-styles.md#mark-specific-appearance) guide, +public [Mark styles](../../docs/styling/mark-styles.md#mark-specific-appearance) guide, plus a `stroke` color and `stroke_width` (px) for a border, e.g. `scatter(x, y, symbol="triangle", stroke="#fff", stroke_width=2)`. Each is an antialiased SDF in the point shader, so shapes stay crisp at any size and the @@ -379,7 +428,7 @@ dense and smoothing is invisible by construction. ### Common typed appearance combinations This table compares the most feature-rich typed appearance props. The public -[Mark Styles](../styling/mark-styles.md) matrix is exhaustive across every +[Mark Styles](../../docs/styling/mark-styles.md) matrix is exhaustive across every rendered mark family and its accepted `style=` properties. | Mark | Color/opacity | Gradient fill | Corner radius | Stroke | Curve | Dash | Size/width | @@ -435,6 +484,31 @@ and thus fully CSS-styleable. ## Static export +`fig.to_image(format="png", *, width=, height=, scale=2.0, background=, +engine=xy.Engine.auto, quality=, optimize=, custom_css=)` returns image bytes, +and `fig.write_image(path, *, format=None, ...)` writes them (format inferred +from the path suffix when omitted). Both are mirrored on `Chart` and +`FacetChart`. The five formats are `png`, `jpeg` (alias `jpg`), `webp`, `svg`, +and `pdf`; `to_svg` and `to_png` remain as the two-format shorthands described +below. + +| Format | Nature | Notes | +| --- | --- | --- | +| `png` | Raster | `optimize=True` trades latency for indexed-palette compression | +| `jpeg` | Raster, lossy | `quality` 1–100 (default 90); rejects `background="transparent"` | +| `webp` | Raster | Native encoder is lossless; `quality` applies to Chromium's lossy WebP | +| `svg` | Vector | Native-only — `engine=chromium` is not available | +| `pdf` | Vector | Text, axes, and marks stay vector; density and heatmap layers embed as bounded rasters (hybrid-vector policy) | + +`scale` is the device-pixel ratio for raster output and is ignored by the vector +formats. `background` accepts `"auto"` (per-format default), a CSS color, or +`"transparent"`. + +`engine=xy.Engine.auto` — the default for `to_image`/`write_image` — resolves +deterministically: the browser-free native path for every format, and Chromium +only when `custom_css` requires a real CSS engine. `xy.Engine.default` pins the +native path and `xy.Engine.chromium` pins the browser. + `fig.to_svg(path?, width=, height=)` renders the same decimated payload the browser client consumes into a standalone, resolution-independent SVG — pure Python, no browser, no extra dependencies. Because decimation runs first, the @@ -447,6 +521,24 @@ built-in **Rust rasterizer** paints that same decimated payload — no browser a millisecond export. Pass `optimize=True` to trade latency for indexed-palette PNG compression and smaller files. Text uses a baked bitmap font (the core has no FreeType), so small labels are slightly less refined than a browser's. + +**Native text coverage.** The atlas is a generated grayscale DejaVu Sans sheet +(`src/font.rs`, regenerated by `scripts/gen_font.py`) baked at a 16 px base +cell and blitted and scaled at runtime, so glyph coverage is a fixed set: ASCII +32–126 plus 110 enumerated extras (Greek, common math operators, arrows, +super/subscripts, typographic quotes and dashes). A codepoint outside that set +— CJK, Cyrillic, Arabic, emoji, most accented Latin — has no glyph and is +**silently skipped**, with no tofu box and no advance reserved, so a title, +tick label, legend entry, or annotation containing one renders shortened rather +than raising. Use `engine=xy.Engine.chromium` for full Unicode text. + +The atlas bounds the native **raster** formats (PNG, JPEG, WebP) only. The +other two native formats carry their own text contracts: SVG emits real +`` elements in a `system-ui` stack and so resolves against the viewer's +fonts, while PDF sets text with the base-14 Helvetica family in WinAnsiEncoding +and replaces any character outside WinAnsi with `?` — a deterministic, +locale-independent substitution. + For browser CSS, font, and WebGL fidelity, `engine=xy.Engine.chromium` screenshots the standalone HTML with an installed Chrome, Chromium, Edge, or `chrome-headless-shell`. Set `XY_BROWSER` to an executable path to override diff --git a/docs/engineering/assets/benchmark-snapshot.svg b/spec/assets/benchmark-snapshot.svg similarity index 100% rename from docs/engineering/assets/benchmark-snapshot.svg rename to spec/assets/benchmark-snapshot.svg diff --git a/docs/engineering/assets/launch-benchmark-comparison.svg b/spec/assets/launch-benchmark-comparison.svg similarity index 100% rename from docs/engineering/assets/launch-benchmark-comparison.svg rename to spec/assets/launch-benchmark-comparison.svg diff --git a/spec/benchmarks/methodology.md b/spec/benchmarks/methodology.md new file mode 100644 index 00000000..3f173baf --- /dev/null +++ b/spec/benchmarks/methodology.md @@ -0,0 +1,339 @@ +# Benchmark Methodology — Defensible Numbers + +**Status:** methodology spec. Extends the shipped harness (`benchmarks/ +bench_vs.py` adapters, `categories.py` registry, `_browser.py` chart-ready +TTFR probe, `bench_scatter_native.py`, `bench_transport.py`, the CodSpeed simulation +modules under `benchmarks/test_codspeed_*.py`, CI `benchmark` job feeding +`docs/benchmark_ci.md`). Written to survive a hostile Hacker News thread: every +number is mode-scoped, reproducible, oracle-checked, and the cases we *lose* +are published. + +## 0. The three rules + +1. **Mode-scoped claims only** (§2 of the dossier, already policy): "10M + density in X ms" is a different claim from "10M individually-styled + markers." Output labels every row `direct | decimated | density | sampled` + — a reader can never mistake an aggregate result for a raw-marker result. +2. **Same-work comparisons.** Each competitor renders the *same visual + contract*, not the same API call: if XY aggregates at 10M, the + fair Plotly comparison is Plotly failing/succeeding at raw markers AND + Plotly+Datashader doing aggregation — both are reported. We never + benchmark our fast path against a competitor's wrong tool. +3. **Truthfulness is a benchmark axis, not a footnote** (§6). Fast-but-wrong + fails the suite. + +## 1. Metric definitions (each with a precise probe) + +| Metric | Definition | Probe | +|---|---|---| +| payload prep | wall time: canonical arrays → wire payload (spec+blob) | `perf_counter` around `build_payload`; competitors: their figure→HTML/JSON serialize | +| wire size | bytes crossing to the client | `len(blob)+len(spec_json)`; competitors: HTML/JSON size | +| browser TTFR | figure build + interactive HTML serialization + navigation → visible chart surface | shipped `_browser.py` chart-ready probe; page FCP is diagnostic only | +| loopback transport | `channel.handle_message()` dispatch → HTTP envelope → client decode; raw/gzip bytes, Python allocation, p50/p95, browser heap, next frame | `benchmarks/bench_transport.py`; binary arm uses the production versioned frame and shipped JS decoder | +| interaction-ready | navigation → first successful synthetic wheel-zoom applied (view actually changes) | extend page probe: dispatch wheel, RAF-poll view/transform change | +| pan/zoom latency | p50/p95/p99 of input→pixels for repeated gestures | dispatched DOM events + draw + WebGL readback in `bench_interaction.py`; standalone-client scope is explicit | +| memory (kernel) | peak RSS delta + tracemalloc peak during prep | shipped psutil/tracemalloc pattern | +| memory (browser) | precise JS heap at chart-ready/dashboard settle; GPU memory remains unavailable | Chromium `performance.memory` with precise-memory flag; GPU numbers are not claimed | +| install size | `pip install` into a fresh venv: total site-packages bytes + wheel bytes + transitive dep count | scripted venv; competitors measured identically (Plotly+kaleido vs plotly alone reported separately) | +| cold import | `python -c "import lib"` best-of-5 fresh interpreters | subprocess timing (already the §33 import-budget concern) | +| small-data | full pipeline at N=1k/10k: prep+TTFR+interaction-ready | the "performance library must not lose the everyday case" check | +| large-data | N=1M/10M/100M per mode ladder | scatter+line+heatmap scenarios | + +## 2. Truthfulness/exactness checks (the novel part) + +Every performance scenario carries an **oracle assertion**; a run that fails +its oracle produces no number (it produces a bug): + +- **Extrema preservation (lines):** decimated polyline's per-pixel-column + min/max equals NumPy oracle min/max (M4's contract). Global max/min pixels + are lit within ±1px column. +- **Count conservation (density/histogram):** sum(grid) == in-window row + count (already the pattern in kernel tests; promoted to the bench harness + at 10M/100M). +- **Channel honesty (after LOD phase 1):** per-cell mean grid equals NumPy + groupby-mean oracle within f32 eps; majority cells match; purity ∈ [0,1]. +- **Drill exactness:** zoom to ≤ budget ⇒ rendered point count == oracle + window count; hover row values equal source f64 exactly. +- **Determinism:** two identical runs produce byte-identical payloads and + (SwiftShader) identical pixel hashes — the anti-shimmer guarantee, and + it's what makes all other numbers reproducible. +- **Reduction disclosure:** spec records tier/mode/visible for every + reduced trace (assert present) — "no silent quality changes" as a test. + +Competitors get the same oracles where the claim applies (e.g. Datashader +count conservation — it passes; Plotly scattergl marker dropout at high N — +documented finding, with repro). + +## 3. Competitor matrix + +Shipped adapters: xy, Plotly, matplotlib, seaborn, Bokeh, Altair, +Datashader, hvPlot/HoloViews, and **plotly-resampler** ✅ (`bench_line.py` — +the honest line competitor, same decimation thesis, so comparing against +vanilla Plotly alone on lines would be a strawman; both run, with the M4 +extrema oracle on the xy row). Still to add: **PyGWalker** (adapter: +programmatic `walk()` export path; if headless render proves unstable, report +prep+payload only and say so). Every adapter: `unavailable` rows rather than +silent omission (harness behavior). Two adjacent metric harnesses now ship +beside the scatter comparison: `bench_install.py` (cold import + install +footprint, §1) and the PNG-export path (`Figure.to_png`) that makes the +static-export size row a real, non-raster-only comparison. + +Per-competitor fairness notes ship in the report: Plotly measured both via +kaleido-PNG (their static path) and browser-HTML (their interactive path); +matplotlib is Agg (it's not interactive — its interaction rows are `n/a`, +not zero); Datashader is prep-only unless embedded in HoloViews (both +reported). + +## 4. Environment & disclosure protocol + +- **Two tiers of numbers:** (a) CI numbers — GitHub Actions ubuntu runner, + SwiftShader software GL; reproducible by anyone from the repo, labeled + "CI (software GL)"; (b) reference-hardware numbers — one pinned desktop + spec (documented CPU/GPU/driver/browser build), labeled as such. Never mix + tiers in one table. HN's first attack is "benchmarked on a potato/cherry + machine" — pre-empt by publishing both. `benchmarks/README.md` is the shipped + runbook for both tiers: exact dependency pins against + `benchmarks/launch_baselines/xy-0.1.0/macos-arm64-m5-pro` (`pyproject.toml`, + `uv.lock`, `environment.json`), copy-paste repro commands, and the prescribed + `reference hardware` vs `CI (software GL)` table labels. +- Every table header carries: timestamp, python version/implementation, + platform system/machine, and harness commit + dirty state + (`bench_vs.to_markdown`). Library versions, executables, CPU count, xy + backend, and browser renderer are recorded in the JSON `environment` block + (`benchmarks/environment.py`) but are not yet rendered into the markdown + header. `benchmark.json` (emitted by the CI `benchmark` job) is the canonical + artifact; `docs/benchmark_ci.md` is rendered from it and + `spec/benchmarks/metrics.md` is emitted by `scripts/check_regressions.py + --emit-md`. `spec/benchmarks/results.md` is hand-maintained and must only quote rows + present in those artifacts — numbers in prose that don't exist in the + artifact are banned (existing policy, kept). +- **Warm/cold discipline:** every timing reports which it is; first-run + (cold cache) and steady-state are separate rows for TTFR and import. +- **Losses ship.** The report has a standing "where XY loses" table + (e.g., tiny-data static PNG export vs matplotlib; ecosystem breadth). + Nothing buys credibility like published losses. + +## 5. Scenario set (the public story) + +1. `small_startup`: 1k line + 10k scatter — TTFR/interaction-ready vs all. +2. `line_10M`: decimated line vs plotly-resampler vs Bokeh — extrema oracle. +3. `scatter_10M_plain` and `scatter_10M_channels`: density path vs + Datashader/HoloViews; channel-honesty oracle; Plotly raw as the + documented-failure row. +4. `scatter_100M_pan`: pyramid pan/zoom percentiles (post LOD phase 3) — + the headline; includes never-blank check (frame sampling: no frame + without chart pixels). +5. `drilldown_truth`: CodSpeed native cycle covers density overview → exact + points → density zoom-out; the 10M+ exact-hover oracle remains the larger + browser/widget follow-up. +6. `core_2d_payloads`: CodSpeed tracks native histogram, area, bar, heatmap, + and composed/layered `xy.chart(...)` payload prep; + `benchmarks/bench_2d_charts.py` stays the Plotly/Seaborn chart-to-pixels + comparison. +7. `dashboard_scale`: `benchmarks/bench_dashboard.py` attempts 10/20/50 mixed + charts, checks every canvas initially and while scrolling, and records payload + prep, navigation readiness, JS heap, redraw-submission p95, per-chart context + loss/restoration events, and the stable loss-free chart-count ceiling. Partial + dashboards remain successful measurement rows rather than losing their metrics. + CI hard-gates the 10-chart row as loss-free/nonblank and applies deliberately + loose catastrophic budgets to its render, scroll, and redraw timings. +8. `install_import`: lower-bound distribution size plus opt-in fresh-venv total + site-packages, transitive distribution count, install time, and cold import. +9. `public_workflows`: `benchmarks/bench_workflows.py` tracks ingestion shapes, + streaming refresh/incremental pyramid update, and HTML/SVG/native-PNG/Chromium-PNG export independently. + +## 6. Remaining implementation plan + +1. Extend the shipped count-conservation and per-pixel line extrema oracles with + channel-aggregation and cross-library pixel-dropout oracles. +2. Add the final widget/comm and Reflex request-to-pixels probes. The shipped + loopback transport diagnostic now covers `channel.handle_message()` through + real HTTP plus browser decode/next-frame — including the production + versioned binary frame on both sides of the wire — but deliberately excludes + real widget comm transport (the widget arm stubs `_send`), GPU upload, and + visible-pixel readback. +3. PyGWalker adapter and Plotly/Bokeh equivalents for the + `dashboard_20` scenario. +4. Record the dataset generator and its seed in the emitted reports (seeds are + currently hardcoded per harness and never captured), and render the JSON + `environment` library versions into the markdown table headers, so §4's + disclosure protocol holds end to end. + +Timing regression policy is two-level: movement beyond 2x is advisory on shared +runners, while movement beyond 8x is a hard failure. Interaction and visual +budgets are capped in the report verifier so a benchmark change cannot silently +make its own gate easier. + +## 7. Regression-gate inputs + +The regression-gate step lives in the `test` job of `.github/workflows/ci.yml`, +which carries no `continue-on-error`. It runs three harnesses and feeds all +three to `scripts/check_regressions.py`: + +| Input | Harness | Artifact | +|---|---|---| +| scatter | `benchmarks/bench_scatter_native.py --sizes 1e5,1e6,1e7 --production` | `scatter.json` | +| kernel | `scripts/bench_native.py --sizes 1e6,1e7` | `kernel.json` | +| transport | `benchmarks/bench_transport.py --n 1e6 --reps 15 --browser-reps 12 --chromium --require-browser` | `transport.json` | + +Each artifact is schema-checked by `scripts/verify_benchmark_report.py` +(`--kind scatter-native`, `kernel-native`, `transport-loopback`) before the gate +reads it, and `check_regressions.py` independently rejects a transport input +whose `measurement_scope` is not `loopback-channel-transport-diagnostic` — a +different harness cannot be substituted for the third input. The same run +emits `spec/benchmarks/metrics.md` via `--emit-md`. + +`bench_transport.py` is the third input and the only one whose CI invocation +hard-gates a browser: + +- **Fixture.** A scatter `Figure` over `n` points (`numpy.random.default_rng(42)`, + uniform x, normal y) plus one `density_view` message. The CLI rejects + `--n` below 250000, so the gated fixture always exercises the density + transport path rather than a direct-tier payload. +- **Same-work arms.** Both arms dispatch the identical message through + `xy.channel.handle_message`; only the response encoding differs. `binary-frame-v1` + is the production versioned frame (`encode_frame`); `base64-json-prototype` is + the current Reflex prototype shape, base64 buffers inside JSON. This is rule 2 + of §0 applied to a wire format instead of a competitor library. +- **Three layers.** In-process envelope encode (wire bytes, gzip bytes at level 6 + with `mtime=0`, encode p50/p95, `tracemalloc` peak, and an explicit count of + payload re-encodes — a count of format transformations, not a claim about + hidden interpreter or socket copies); Python loopback through a real + `ThreadingHTTPServer` (request → decode p50/p95, first two iterations + discarded); and an optional Chromium probe that fetches both endpoints, + decodes with the shipped JS `decodeFrame`, and awaits the next animation + frame. `--require-browser` makes a skipped or failed probe a nonzero exit, so + the browser arm is not silently optional in CI. +- **Append diagnostics.** Widget binary transmission count and bytes (measured + with `FigureWidget._send` stubbed) plus the wire cost an unaffected second + trace adds to a single-trace append. + +Only deterministic byte counts reach the gate. `flatten()` lifts `wire_bytes`, +`gzip_bytes`, and `wire_to_payload_ratio` per envelope mode, plus the five +`append_diagnostics` counters: 11 rows in `spec/benchmarks/metrics.md`, all +prefixed `transport.`. Every one of the 11 gates **hard** under `policy()` — +byte counts and the ratio at 2% tolerance, `widget_binary_transmissions` at zero +tolerance. The harness's measured timings (encode p50/p95, HTTP p50/p95, browser +request-to-next-frame, JS heap delta) are reported and uploaded but deliberately +ungated: they are wall-clock numbers from a shared runner, and §4's warm/cold +and tier discipline applies to them rather than a threshold. + +## 8. CodSpeed simulation modules + +`.github/workflows/codspeed.yml` runs `pytest benchmarks/test_codspeed_*.py +--codspeed` under `CodSpeedHQ/action` in `simulation` mode, after asserting +`xy.kernels.BACKEND == "native"`. Simulation counts instructions rather than +wall time, so browser, install, and cross-library process benchmarks stay out of +it — those live in `benchmark-refresh.yml`, and the workflow says so inline. + +The glob collects three modules — `test_codspeed_kernels.py`, +`test_codspeed_pyplot.py`, and `test_codspeed_transport.py` — for 94 rows +total. These are trend-tracked in CodSpeed, not gated: none of them feed +`scripts/check_regressions.py`, whose three inputs are §7's. + +**`benchmarks/test_codspeed_pyplot.py` — 14 rows, seven paired arms.** Each pair +expresses one chart twice over the same input arrays: once through the +declarative API (`xy.chart` + marks) and once through the identical +matplotlib-style calls in `xy.pyplot`. Both arms end at the same terminal work — +`build_payload_split(2048)`, or PNG bytes for the export pair — so the gap +between a `*_pyplot` row and its `*_raw` twin is exactly what the shim adds: +matplotlib-call translation, fmt-string parsing, and figure-lifecycle +bookkeeping. The pyplot arm includes `plt.close("all")` because figure-registry +bookkeeping is part of the shim's per-figure cost. The pairs are a 10k line, a +1M line, a 100k scatter, a 100k-value/200-bin histogram, a 1k-category bar, a +5k-point three-series styled panel (title, axis labels, legend — the shim's +worst honest case, where translation is largest relative to data), and a +100k-point PNG export. + +Fairness pins, all asserted in the module rather than assumed: + +- Both arms build the same 640x480 canvas (`plt.subplots()`'s 6.4x4.8in at + dpi=100), and the raw arm declares the explicit axes the shim adds implicitly, + so chrome layout work is identical on both sides. +- The export pair uses `scale=2.0` to match `savefig`'s fixed 2x supersampling — + both emit a 1280x960 canvas. A smaller raw canvas would overstate the gap. +- Every pair except the histogram asserts byte-identical payload sizes across + arms, so a row cannot get faster by shipping a different chart. This is §2's + oracle discipline applied to a shim-overhead measurement. +- The histogram pair is exempt and documents why: `ax.hist` pre-bins with NumPy + because it must return matplotlib's `(n, bins, patches)` tuple and then ships + bar geometry, while `xy.histogram` bins natively and ships rect columns. Its + assertion is instead that the payload stays bounded by bin count, never by + observation count. +- A session fixture warms both arms' lazily imported submodules (marks, + `_payload`, the export stack, the shim's translation tables) before any + measured region. Without it the first benchmark of each arm tracks package + source size instead of its own workload. + +**`benchmarks/test_codspeed_transport.py` — 7 rows.** These isolate the Python +envelope codec and deliberately exclude what `bench_transport.py` covers: +sockets, compression, JS heap, and animation frames are wall-clock/browser +measurements and are not meaningful under simulation. Fixtures are a 128 KiB +density buffer and an 800 KiB two-buffer direct payload. The rows are +`encode_frame` on both fixtures; `encode_frame_parts` on the direct fixture, +asserting the parts sum to the single-body length and that the buffer parts stay +zero-copy `memoryview`s; `decode_frame` on both fixtures, asserting the decoded +buffers alias the original body rather than copying it; and base64-JSON +encode/decode comparators on the direct fixture, so the codec's advantage stays +a continuously tracked ratio rather than a remembered one. + +**`benchmarks/test_codspeed_kernels.py` — 73 rows** (70 functions; two are +parametrized, over 2 ingest flavors and 3 `bin_2d` thread-cap regimes). This is +the bulk of the suite and covers the native compute core the rest of the engine +sits on: decimation tiers, f32 encoding, and zone maps (`spec/design-dossier.md` +§5, §4/§16, and §22 respectively), plus the end-to-end figure → wire-payload +path. Sizes are pinned at three scales — 10k, 100k, and 1M — so a regression is +attributable to a regime (normal dashboard chart, exact WebGL workload, +screen-bounded large-data path) rather than to one arbitrary N. Seventeen rows +are `first_payload_*`, one per chart kind, which makes this the per-kind guard +that §7's scatter-only gate does not provide. Its module docstring carries the same backend rule the workflow +asserts: fallback timings are correctness smoke data, not production +performance data. + +## 9. Shim gates and CI coverage + +Two `make` targets bound the `xy.pyplot` shim from opposite sides. Both appear +in the focused-gate table of `spec/process/production-readiness.md`. + +- **`make check-pyplot`** → `pytest tests/pyplot -q`. The correctness side: shim + behavior, matplotlib interoperability, and the reference corpus. It also + carries the *relative* speed gate, `tests/pyplot/test_perf_guardrail.py`, + which asserts the pyplot build stays within 1.6x the declarative build at 10k + points and 1.5x at 100k (best-of-N, plus a 100us absolute allowance for CI + timer jitter) and that theme/axis components come from the component cache + instead of being rebuilt. Its own docstring scopes it: a structural-regression + gate for an O(n) copy or per-build revalidation sneaking into the shim, not a + re-measurement of the margin. +- **`make check-pyplot-speed`** → `benchmarks/bench_pyplot_vs_matplotlib.py + --profile standard --reps 21 --warmups 3 --target-speedup 10 --require-target` + (needs the `.[bench]` extra). The *absolute* side: figure construction through + a completed, compressed PNG at a shared 1800x840 target against + matplotlib/Agg, reported per family — line, scatter, histogram, bar, + pcolormesh, contour. `--require-target` exits nonzero unless every family + reaches the 10x total-time target. The run alternates library order to reduce + drift bias, excludes data generation, imports, and the first warm-up render, + retains raw timing samples in its JSON artifact, and fails a case whose PNG + comes back blank for either library. + +The CodSpeed pyplot pairs of §8 track the same promise continuously, so a +structural regression surfaces attributed to the `*_pyplot` arm instead of as an +unexplained engine slowdown. + +**CI coverage, verified against `.github/workflows/ci.yml`.** The jobs carrying +no `continue-on-error` are `matplotlib_reference`, `test`, `browser_conformance`, +`python_floor`, `sdist`, `wheels`, and `install_without_rust`; only +`benchmark_vs`, `benchmark_methodology`, and `benchmark` are non-blocking. +`matplotlib_reference` is **not** absent from the Release-Blocking Gates table in +`spec/process/production-readiness.md`: it is the "Matplotlib reference" row, whose +evidence column names exactly the job's two halves, +`python scripts/sync_matplotlib_compat.py --check` and `pytest tests/pyplot`. +What the job adds beyond that row's wording is the pin and the environment — it +installs `matplotlib==3.11.0` into a fresh uv venv and asserts the resolved +version before checking the reviewed snapshot, then runs +`test_launch_compat.py`, `test_reference_corpus.py`, and +`test_reference_semantics.py` under `MPLBACKEND=Agg`. + +The real coverage gap is narrower and is on the speed side: no workflow invokes +`bench_pyplot_vs_matplotlib.py`. The per-family 10x PNG target is therefore +enforced by `make check-pyplot-speed` locally and watched by the CodSpeed pyplot +rows' trend, but no blocking CI job asserts it. diff --git a/spec/benchmarks/metrics.md b/spec/benchmarks/metrics.md new file mode 100644 index 00000000..d36e47a2 --- /dev/null +++ b/spec/benchmarks/metrics.md @@ -0,0 +1,56 @@ +### Auto-generated benchmark metrics + +Regenerated from the CI benchmark run; do not hand-edit. + +Source CI reports: commit `914a32787de211d87f2c065e1786e98058a8194a`; latest measurement `2026-07-20T23:32:08.817015Z`. + +| metric | value | +|---|---:| +| kernel.bin_2d_mpts_s.1000000 | 373.42 | +| kernel.bin_2d_mpts_s.10000000 | 598.06 | +| kernel.bin_2d_ms.1000000 | 2.68 | +| kernel.bin_2d_ms.10000000 | 16.72 | +| kernel.encode_mpts_s.1000000 | 2,623.53 | +| kernel.encode_mpts_s.10000000 | 2,523.51 | +| kernel.histogram_mpts_s.1000000 | 692.25 | +| kernel.histogram_mpts_s.10000000 | 769.43 | +| kernel.histogram_ms.1000000 | 1.44 | +| kernel.histogram_ms.10000000 | 13.00 | +| kernel.local_density_mpts_s.1000000 | 103.63 | +| kernel.local_density_mpts_s.10000000 | 103.58 | +| kernel.local_density_ms.1000000 | 1.93 | +| kernel.local_density_ms.10000000 | 1.93 | +| kernel.local_density_n.1000000 | 200,000 | +| kernel.local_density_n.10000000 | 200,000 | +| kernel.m4_full_mpts_s.1000000 | 547.13 | +| kernel.m4_full_mpts_s.10000000 | 621.15 | +| kernel.normalize_mpts_s.1000000 | 1,167.41 | +| kernel.normalize_mpts_s.10000000 | 1,346.31 | +| kernel.range_mpts_s.1000000 | 1,772.16 | +| kernel.range_mpts_s.10000000 | 1,517.41 | +| kernel.range_ms.1000000 | 0.56 | +| kernel.range_ms.10000000 | 6.59 | +| kernel.zone_maps_mpts_s.1000000 | 599.01 | +| kernel.zone_maps_mpts_s.10000000 | 660.25 | +| kernel.zoom_redecimate_ms.1000000 | 0.08 | +| kernel.zoom_redecimate_ms.10000000 | 0.42 | +| scatter.tier.100000 | direct | +| scatter.tier.1000000 | density | +| scatter.tier.10000000 | density | +| scatter.wire_bytes.100000 | 800,983 | +| scatter.wire_bytes.1000000 | 266,000 | +| scatter.wire_bytes.10000000 | 264,072 | +| scatter.wire_bytes_per_point.100000 | 8.01 | +| scatter.wire_bytes_per_point.1000000 | 0.27 | +| scatter.wire_bytes_per_point.10000000 | 0.03 | +| transport.append.extra_unaffected_trace_wire_bytes | 80,430 | +| transport.append.single_trace_append_wire_bytes | 80,962 | +| transport.append.two_trace_append_wire_bytes | 161,392 | +| transport.append.widget_binary_bytes | 160,016 | +| transport.append.widget_binary_transmissions | 2 | +| transport.base64-json-prototype.gzip_bytes | 107,059 | +| transport.base64-json-prototype.wire_bytes | 174,139 | +| transport.base64-json-prototype.wire_to_payload_ratio | 1.34 | +| transport.binary-frame-v1.gzip_bytes | 96,274 | +| transport.binary-frame-v1.wire_bytes | 130,792 | +| transport.binary-frame-v1.wire_to_payload_ratio | 1.01 | diff --git a/docs/engineering/benchmark.md b/spec/benchmarks/results.md similarity index 90% rename from docs/engineering/benchmark.md rename to spec/benchmarks/results.md index 23e745d9..1feaa224 100644 --- a/docs/engineering/benchmark.md +++ b/spec/benchmarks/results.md @@ -33,8 +33,9 @@ large line, and a 30-chart dashboard stress different parts of the system. These are the categories we track or plan to add to CI. The stable category IDs live in `benchmarks/categories.py`. CI's benchmark JSON -artifacts (`benchmark.json`, `line.json`, `install.json`, `interaction.json`, -`dashboard.json`, `scatter.json`, and `kernel.json`) use schema version 2: +artifacts (`benchmark.json`, `line.json`, `install.json`, `install-fresh.json`, +`interaction.json`, `dashboard.json`, `workflows.json`, `scatter.json`, +`kernel.json`, and `transport.json`) use schema version 2: they include the full registry, `tracked_categories`, and a machine-readable `environment` block with Python, platform, package, executable, git commit, and dirty-worktree metadata. The @@ -56,15 +57,22 @@ commit so CI artifacts are quick to inspect from logs. | `adaptive_scatter_drilldown` | Adaptive scatter drilldown | tracked | The large-data claim needs a credible path from overview to exact visible points. | visible-query latency, tier-switch latency, exact-point recovery, badge accuracy | `benchmarks/test_codspeed_kernels.py::test_adaptive_drilldown_cycle` | Exact points when visible count is under budget; sampled/density with explicit counts otherwise. | | `huge_line_time_series` | Huge line / time series | tracked | Common observability and finance workload; Plotly-resampler sets the bar here. | decimation time, zoom re-decimation latency, TTFR, extrema preservation | `benchmarks/bench.py`, `bench_native.py`, `bench_interaction.py`, `test_decimate_view` | Screen-bounded line payloads with extrema-preserving decimation and fast zoom refresh. | | `many_chart_dashboards` | Many-chart dashboards | tracked | Plotly-class apps often fail from total page weight and many live canvases, not one chart. | payload prep, navigation readiness, JS heap, redraw submission, scroll visibility, context loss/restore, stable chart-count ceiling | `benchmarks/bench_dashboard.py` | Measure the 10-50 chart scaling curve and expose LRU context eviction without discarding partial-row metrics. | -| `interaction_smoothness` | Interaction smoothness | tracked | Users judge performance by pan/zoom/hover, not just export time. | pan/zoom FPS, wheel latency, hover latency, tooltip stability, selection latency, frame color delta | `benchmarks/bench_interaction.py` | Stay responsive during interaction, avoid blank/flickering frames, then refine view after interaction settles. | -| `payload_export_size` | Payload/export size | tracked | Notebooks, static HTML, docs, and dashboards pay for every byte shipped. | standalone HTML bytes, binary payload bytes, bundle bytes | `bench_vs.py`, `bench_scatter_native.py`, `test_first_payload_density_large`, `test_memory_report_density_medium`, example app asset sizes | Keep data payloads binary and screen-bounded where possible; warn when exact export would be huge. | +| `interaction_smoothness` | Interaction smoothness | tracked | Users judge performance by pan/zoom/hover, not just export time. | pan/zoom FPS, wheel latency, hover latency, tooltip stability, selection latency, frame color delta | `benchmarks/bench_interaction.py`; `benchmarks/bench_transport.py` | Stay responsive during interaction, avoid blank/flickering frames, then refine view after interaction settles. | +| `payload_export_size` | Payload/export size | tracked | Notebooks, static HTML, docs, and dashboards pay for every byte shipped. | standalone HTML bytes, binary payload bytes, bundle bytes | `bench_vs.py`, `bench_scatter_native.py`, `bench_transport.py`, `test_codspeed_transport.py`, `test_first_payload_density_large`, `test_memory_report_density_medium`, example app asset sizes | Keep data payloads binary and screen-bounded where possible; warn when exact export would be huge. | | `core_2d_chart_breadth` | Core 2D chart breadth | tracked | The library needs to stay fast beyond the scatter wedge: bars, histograms, areas, and heatmaps are everyday chart workloads. | payload-prep time, payload bytes, standalone HTML bytes, TTFR | `benchmarks/bench_2d_charts.py` vs Plotly/Seaborn; `benchmarks/bench_pyplot_vs_matplotlib.py`; `bench_interaction.py`; CodSpeed core-2D rows | Beat Plotly on user-visible first paint for common 2D charts while tracking Matplotlib/Seaborn raster baselines where applicable. | -| `launch_scatter` | Core launch scatter baseline | tracked | Launch claims need an immutable, apples-to-apples record of default product behavior from small charts through the 1B-point capacity case. | static PNG time/RSS; interactive TTFR and Python/browser RSS; hardware and SwiftShader kept separate | `benchmarks/bench_launch_scatter.py` vs Plotly and Matplotlib at 10k, 100k, 1M, 10M, and 1B | Preserve the fixed launch contracts and add versioned environment baselines rather than overwriting prior results. | +| — (not in `benchmarks/categories.py`) | Core launch scatter baseline | tracked outside the registry | Launch claims need an immutable, apples-to-apples record of default product behavior from small charts through the 1B-point capacity case. | static PNG time/RSS; interactive TTFR and Python/browser RSS; hardware and SwiftShader kept separate | `benchmarks/bench_launch_scatter.py` vs Plotly and Matplotlib at 10k, 100k, 1M, 10M, and 1B | Preserve the fixed launch contracts and add versioned environment baselines rather than overwriting prior results. | | `input_ingestion` | Input ingestion | tracked | Real applications provide converted, strided, datetime, list, pandas, and Arrow inputs rather than only contiguous f64 arrays. | ingest latency, copies, peak Python memory | `benchmarks/bench_workflows.py` ingestion rows | Keep zero-copy inputs cheap and make unavoidable conversions visible. | -| `streaming_updates` | Streaming updates | tracked | Monitoring and notebook workflows append repeatedly; stable-domain batches should update indexes incrementally while domain growth may rebuild. | append latency, refresh bytes, incremental pyramid update, domain-growth rebuild | `benchmarks/bench_workflows.py` streaming rows | Keep stable-domain appends proportional to the batch and expose unavoidable rebuild stalls. | +| `streaming_updates` | Streaming updates | tracked | Monitoring and notebook workflows append repeatedly; stable-domain batches should update indexes incrementally while domain growth may rebuild. | append latency, refresh bytes, incremental pyramid update, domain-growth rebuild | `benchmarks/bench_workflows.py` streaming rows; `benchmarks/bench_transport.py` append diagnostics | Keep stable-domain appends proportional to the batch and expose unavoidable rebuild stalls. | | `log_autorange` | Log autorange | tracked | Large positive/negative and non-finite series are common in monitoring and scientific charts, and log axes must avoid full-data rescans. | range latency, positive-domain correctness, peak Python memory | `benchmarks/bench_workflows.py` log autorange row; `tests/test_figure.py` | Compute correct positive log domains from zone statistics with cost proportional to chunks, not points. | | `static_export` | Static export | tracked | HTML, SVG, and PNG have distinct serialization and browser costs. | export latency, output bytes, peak Python memory | `benchmarks/bench_workflows.py` export rows; `benchmarks/bench_pyplot_vs_matplotlib.py` matched PNG rows | Track each target independently without mixing browser and payload work. | +The launch scatter baseline has no entry in `BENCHMARK_CATEGORIES`, so it +carries no category ID: `scripts/verify_benchmark_report.py` rejects any row +whose category ID is absent from the registry, and +`benchmarks/bench_launch_scatter.py` reports its fixed launch contracts through +the versioned baselines under `benchmarks/launch_baselines/` instead. Giving it +a registry ID is pending. + Mode labels in benchmark output should stay explicit: `direct`, `decimated`, `density`, `sampled`, or `adaptive`. A 10M density result is a real large-data visualization result, but it is not the same claim as 10M individually styled @@ -73,7 +81,10 @@ markers. The benchmark reports should make that distinction impossible to miss. ## Interaction, drilldown, and dashboard probes CodSpeed is native-only and intentionally focused on hot paths that should not -regress between commits. The suite asserts `xy.kernels.BACKEND == +regress between commits. The CodSpeed job runs +`pytest benchmarks/test_codspeed_*.py --codspeed` +(`.github/workflows/codspeed.yml`), so every `test_codspeed_*` module is in +scope, not the kernels module alone. The suite asserts `xy.kernels.BACKEND == "native"` before timing anything. It tracks: - Rust kernels for f32 encoding, min/max, zone maps, M4 decimation, density @@ -102,6 +113,15 @@ regress between commits. The suite asserts `xy.kernels.BACKEND == - The native adaptive drilldown cycle with `test_adaptive_drilldown_cycle`: a warmed large scatter viewport moves from density overview, to exact visible points, and back out to density. +- The `xy.pyplot` shim suite (`benchmarks/test_codspeed_pyplot.py`): paired + raw-versus-shim rows for line, scatter, histogram, categorical bar, and + styled-panel builds, plus matched PNG export + (`test_png_export_line_raw` / `test_png_export_line_pyplot`), so shim overhead + over the native path stays visible. +- The transport frame suite (`benchmarks/test_codspeed_transport.py`): + `encode_frame` / `decode_frame` rows for density and direct payloads, + a parts-encode row, and base64 encode/decode comparators standing in for the + JSON-embedded prototype shape. That keeps CodSpeed focused on native range queries, pyramid composition, tier-switch payload generation, payload prep, zoom latency, and memory-report @@ -432,7 +452,8 @@ This closes the biggest fairness gap: previously xy' number excluded the WebGL draw while the HTML libraries' numbers excluded the browser entirely — both stopped before pixels existed. The CI `benchmark` job now runs the TTFR pass (capped at 100k points, since each row launches a browser) and the numbers land -in `docs/benchmark_ci.md` / the `benchmark-report` artifact. Locally, xy' +in `docs/benchmark_ci.md`, which is not committed — CI regenerates it each run +and uploads it as the `benchmark-report` artifact. Locally, xy' standalone-export TTFR under software GL (SwiftShader) is ~180 ms for a density page; on real GPU hardware it is lower. @@ -441,8 +462,12 @@ page; on real GPU hardware it is lower. The xy, matplotlib, seaborn, and Plotly rows are refreshed from the 2026-07-09 `benchmark-refresh` CI run (Ubuntu, Python 3.12, native Rust core; Plotly via kaleido→PNG). The Bokeh, Altair, Datashader, and hvPlot rows (¹) are -from an earlier local expanded run and are not re-measured here — the refresh -workflow does not install those libraries. Reproduce the refreshed rows with: +carried over from an earlier local expanded run and have not been republished +from a refresh artifact. The `benchmark-refresh` workflow does install those +libraries and passes no `--libraries` filter, so `bench_vs.py` defaults to every +adapter; re-running it and republishing from its `scatter-vs.json` would +supersede these rows. That refresh is pending. Reproduce the refreshed rows +with: ```bash PYTHONPATH=python .venv/bin/python benchmarks/bench_vs.py \ @@ -514,9 +539,13 @@ PYTHONPATH=python .venv/bin/python benchmarks/bench_vs.py \ | Datashader ¹ | PNG raster § | not measured | — | — | — | | hvPlot / HoloViews ¹ | Bokeh HTML ‡ | not measured | — | — | — | -¹ The expanded local adapters were not installed by the 10M CI refresh; no -10M result is claimed for those rows. The 100k results above remain the latest -measurements for that group. +¹ No 10M result has been republished for those adapters; the cells are left +explicit rather than filled from an older run. The 100k results above remain the +latest measurements for that group. The refresh workflow does install these +libraries, so a 10M refresh is possible and pending — `bench_vs.py` marks any +adapter exceeding the per-config budget `skipped(over budget)` and stops +climbing to larger N, so some of these rows may resolve to a ceiling rather than +a timing. At 10M points, xy prepares a fixed-size screen-bounded payload while the direct-rendering libraries retain work proportional to the input size. This @@ -602,8 +631,9 @@ local-density lookup. | 1M | 3.05 ms | 3,187 Mpt/s | 0.48 ms | 3 ms | 1.11 ms / 200k | | 10M | 29.93 ms | 3,184 Mpt/s | 4.76 ms | 30 ms | 1.14 ms / 200k | -The kernel-shape mode of `benchmarks/bench_scatter_native.py` reported the grid -alone; it is not the production transport payload: +The `--production` mode of `benchmarks/bench_scatter_native.py` +(`measurement_scope: production-figure-payload`) reports the real Figure +transport payload. The rows below are that mode: | points | tier | data prep | wire bytes | browser render | |---:|---|---:|---:|---:| @@ -612,6 +642,11 @@ alone; it is not the production transport payload: | 10M | density | 10.2 ms | 258 KB | not measured in this native-only refresh | | 100M | density | 65.7 ms | 258 KB | not measured in this native-only refresh | +The harness's default kernel-shape mode (`measurement_scope: +native-kernel-shape`) is a different scope and does not appear in that table: it +reports the raw f32 grid alone, a constant `GRID_W * GRID_H * 4` = 768 KiB for +every density row (`benchmarks/bench_scatter_native.py:151`). + CI regression gating uses `--production`, which calls the real Figure payload compiler and includes compact spec metadata plus the sampled point overlay. The density grid is log-encoded directly into the same one-byte R8 precision @@ -748,8 +783,11 @@ bench_install.py`; best-of-5 fresh-interpreter import; distribution files only | seaborn | 1.50 s | 1.0 MB | | hvplot | 4.65 s | 685 KB | -xy imports 6–730× faster and is 40–75× smaller on disk than the -mainstream libraries (the §33 import-budget goal, made comparative). +Sizes are binary units (`bench_install.py` divides by 1024 and labels the result +KB/MB). On that convention, across the eight comparators above xy imports +6–730× faster and is 1.2–75× smaller on disk; against the heavy stacks (plotly, +matplotlib, bokeh) the disk gap is 43–75× (the §33 import-budget goal, made +comparative). The harness also has `--fresh-venv`, which installs each requested target into an isolated `uv` environment and reports total site-packages bytes, file count, @@ -772,12 +810,29 @@ runners: noise never breaks the build. The current metric table is regenerated (never hand-typed) into -[`docs/engineering/benchmark_metrics.md`](benchmark_metrics.md). CI uploads that table plus -the raw `scatter.json` and `kernel.json` inputs as the +[`spec/benchmarks/metrics.md`](metrics.md). CI uploads that table plus +the raw `scatter.json`, `kernel.json`, and `transport.json` inputs as the `regression-benchmark-report` artifact, even when the hard regression gate fails. Re-bless the baseline from a CI run with `check_regressions.py --update-baseline`. +### Transport loopback + +`benchmarks/bench_transport.py` (report kind `transport-loopback`) is the third +gated input. It measures the transport-neutral channel dispatcher: two HTTP +endpoints both call `xy.channel.handle_message` and differ only in response +encoding — the Reflex prototype shape (base64 buffers inside JSON) versus xy's +production versioned binary frame. It also records append diagnostics +(single-trace, two-trace, and unaffected-trace wire bytes) and an optional +Chromium probe that fetches and decodes both formats from the same loopback +server, waiting for the next animation frame after each decode. That probe does +not claim request-to-pixels or GPU-upload latency. + +CI runs it at 1e6 points with `--require-browser`, verifies it, and passes +`--transport transport.json` to `check_regressions.py`. Its `transport.*` +metrics gate **hard**: `wire_bytes`, `gzip_bytes`, and `wire_to_payload_ratio` +are deterministic, and `transmissions` is gated at zero tolerance. + ### Static image export `Figure.to_png()` defaults to `Engine.default`, the browser-free native Rust @@ -808,7 +863,7 @@ time and label each row's mode and target. | Plotly `Scatter` (SVG) | — over budget above 1 M | | | | | At 10 M points xy is **~19× faster than matplotlib**, **~47× faster than -Seaborn**, and **~321× faster than Plotly's WebGL path**, at **4–13× lower peak +Seaborn**, and **~320× faster than Plotly's WebGL path**, at **4–13× lower peak memory**. Plotly's SVG path never reached 10 M (over budget above 1 M). > Scope note (important): every column above comes from the same harness pass @@ -845,8 +900,13 @@ not draw — the design targets 100 M–1 B (dossier §2). ### xy (native core; "render" = build GPU payload) -This is the retained 2026-07-08 cross-library run and predates log-u8 density -transport; the current production refresh is reported above. +These per-N tables are the retained **2026-07-03** cross-library run. They +predate log-u8 density transport *and* the 2026-07-08 refresh that produced the +Headline table above, so their rows are not interchangeable with it — the +Plotly `Scattergl` 10M row here (33,907 ms) is the 07-03 measurement, while the +Headline's 54,064 ms is the later 07-08 re-measurement of the same +configuration. The current production refresh is reported above; these tables +are kept for the per-N scaling shape, not for headline figures. | N | build | render | total | peak mem | payload | points/sec | |---|---|---|---|---|---|---| diff --git a/docs/engineering/design-dossier.md b/spec/design-dossier.md similarity index 86% rename from docs/engineering/design-dossier.md rename to spec/design-dossier.md index 87f46a1c..16074281 100644 --- a/docs/engineering/design-dossier.md +++ b/spec/design-dossier.md @@ -41,9 +41,9 @@ Every claim in this dossier is **mode-scoped and testable** — no universal num |---|---|---| | Round 2 (self) | 15 findings | Resolved in-place → §15–§26 | | Round 3 (external) | 10 findings | Resolved in-place → §27–§31 | -| Round 3 (deep, Appendix A) | F1–F3 (Critical/Major) | **Resolved** → Part IV (§32–§35): distribution, filtering, GPU ceilings | +| Round 3 (deep, Appendix A) | F1–F3 (Critical/Major) | **F1–F2 resolved** → Part IV (§32–§35): distribution, filtering. **F3 specified, not implemented** — the tier decision is still count-only and vertex buffers are unchunked (§5) | | Round 3 (deep, Appendix A) | **F4–F12** | **Outstanding** — not yet folded into the spec (see below) | -| Round 4 (styling) | F-S1–F-S3 | **Resolved** → §36 (export parity, probe-element color resolution, indexed series tokens) | +| Round 4 (styling) | F-S1–F-S3 | **Partly resolved** → §36: probe-element color resolution shipped; client-side export parity shipped, kernel-side theme snapshot and indexed series tokens still pending | **Outstanding work (F4–F12), the honest to-do list:** - **F4** — per-trace f32 offsets (a single viewport origin can't serve traces at wildly @@ -144,13 +144,13 @@ If a milestone doesn't move one of these numbers, it's not in scope for that mil │ spec (small JSON) + typed column buffers (binary) ▼ ┌───────────────────────────────────────────────────────────────┐ -│ CORE (Rust → compiled to WASM for browser, native for export) │ +│ CORE (Rust cdylib, C ABI — runs inside the Python process) │ │ ┌─────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ │ │ Ingest & │ │ LOD / │ │ Scene graph (retained) │ │ │ │ column store│→ │ decimation / │→ │ + diff engine │ │ │ │ (1 copy) │ │ aggregation │ │ │ │ │ └─────────────┘ └──────────────┘ └────────────────────────┘ │ -│ runs in a Web Worker; SharedArrayBuffer to main thread │ +│ loaded via ctypes; NumPy buffers passed by pointer │ └───────────────┬───────────────────────────────────────────────┘ │ GPU buffers (uploaded once) + draw commands ▼ @@ -161,6 +161,10 @@ If a milestone doesn't move one of these numbers, it's not in scope for that mil └───────────────────────────────────────────────────────────────┘ ``` +An in-browser WASM core running in a Web Worker with SharedArrayBuffer transport was +the original design and was dropped in favor of the native-in-kernel core (§32); a WASM +client remains a possible future pure-browser path. + The two requirements live primarily in the **data pipeline (§4–§6)**. The renderer (§7) matters, but memory and scale are won or lost in how we store and reduce data. @@ -239,8 +243,8 @@ The two requirements live primarily in the **data pipeline (§4–§6)**. The re The key insight: **never push the GPU more primitives than the screen has pixels.** The engine picks a tier per trace and re-picks on zoom (operating only on the visible -window). **The tier heuristic is NOT point count alone** — real GPUs impose two other -ceilings (both documented by deck.gl in production): +window). Beyond vertex count, real GPUs impose two other ceilings (both documented by +deck.gl in production): - **Fill-rate:** fragment work = `count × mark_pixel_area × overdraw`. 10M radius-5 points ≈ 1B fragment invocations/frame; a 500k-point scatter with large or @@ -248,15 +252,28 @@ ceilings (both documented by deck.gl in production): - **Allocation:** Chrome caps a single allocation at ~1 GB; deck.gl documents crashes between 10M–100M items during buffer creation for exactly this reason. -So: `tier = f(visible_count, mark_pixel_area × estimated_overdraw)` — a dense -large-marker scatter trips Tier 2 aggregation even at "sub-ceiling" counts — and the -render core **chunks vertex buffers** (multi-buffer draws, ~128 MB segments) as a -structural rule, not an edge case. "One instanced draw call for N points" is the -mental model; the implementation is a handful of chunked draws, which costs nothing -measurable and never hits the allocation cliff. - -**Tier 0 — Direct (up to ~1–2M *drawn, small-marker* points):** upload raw columns, -draw with instancing (chunked per the above). Simple, exact. +**What ships today is count-only:** `tier = f(visible_count)`, hysteresis-guarded. +`drill_decision(visible, budget, in_drill, exit_factor)` in `python/xy/lod.py` returns +`visible <= budget * (exit_factor if in_drill else 1.0)`, with +`DRILL_EXIT_FACTOR = 1.15` (`python/xy/config.py`) so a trace that has drilled down to +real points stays drilled until the count clearly exceeds the budget again. The client +mirrors the same rule (`LOD_DIRECT_POINT_BUDGET`, `LOD_DRILL_EXIT_FACTOR` in +`js/src/45_lod.js`). Mark pixel area and overdraw do **not** enter the decision. + +*Pending (F3, not implemented):* folding `mark_pixel_area × estimated_overdraw` into the +tier decision, so a dense large-marker scatter trips Tier 2 aggregation at sub-ceiling +counts; and **chunked vertex buffers** (multi-buffer draws, ~128 MB segments) so the +allocation cliff is structurally unreachable. Both remain the intended design; neither +exists in `js/src/` today. + +**Tier 0 — Direct.** Upload raw columns, draw with instancing. Simple, exact. The +budget is channel-dependent: `Trace.use_density()` (`python/xy/_trace.py`) picks +`DIRECT_SOFT_CEILING = 2_000_000` when the trace carries a per-point color or size +channel, and `SCATTER_DENSITY_THRESHOLD = 200_000` otherwise (both in +`python/xy/config.py`). A plain scatter therefore aggregates at 200k — its whole win is +not drawing 10M dots — while a scatter whose per-point color/size aggregation would +destroy stays direct up to 2M. `js/src/45_lod.js` carries the matching 200k client +budget. **Tier 1 — Decimated lines (LTTB / min-max per pixel column):** for line/area traces with more points than horizontal pixels, reduce to ~2–4 points per pixel column @@ -269,7 +286,8 @@ scatter and heatmaps, don't draw points — draw a colormapped **density texture The naive version ("bin all points into a screen-sized texture") is *not* "O(points) once": the bin grid depends on the viewport, so every pan/zoom would re-bin the whole -dataset. The actual design is a **data-space tile pyramid**: +dataset. The intended design is a **data-space tile pyramid** — the four bullets below +describe that target, not current behavior; see the shipped subset after them: - At ingest (or lazily on first Tier-2 entry), build aggregation tiles in *data* coordinates at power-of-two zoom levels — count/sum per cell, ~256² cells per tile. @@ -285,7 +303,20 @@ dataset. The actual design is a **data-space tile pyramid**: - Colormapping (including perceptual/log scaling and dynamic-range normalization) happens at *composite* time on the aggregate values, so restyling never re-bins. -So the honest cost model: **O(points) once at build, O(visible tiles) per frame, +*Shipped today (`src/tiles.rs`, `python/xy/interaction.py`):* a **single square count +pyramid**, not tiles. One trace-wide grid over the full data bounds whose finest level +is `PYRAMID_BASE_DIM`² (2048², `python/xy/config.py`), each coarser level an exact 4→1 +u64 sum saturating to u32 down to 1². Built lazily on the first density view at +≥ `PYRAMID_MIN_POINTS` (2,000,000). There is no per-tile fetch entry point and no tile +addressing — the C ABI is `xy_pyramid_build` / `_append` / `_count` / `_compose` / +`_free`, and composition happens kernel-side over the whole window, refusing past +`MAX_UPSAMPLE` (2×, `src/tiles.rs`) so below-floor windows fall back to the exact +`range_indices` + `bin_2d` path. Only the `count` plane exists; the per-channel `sum` / +`argmax+purity` planes above are not built. *Pending:* tiling proper (per-tile build, +fetch, and pan-time reuse), so "pan is pure tile reuse" is design, not behavior. See +`spec/design/lod-architecture.md` §4 for the full design and its shipped-status ledger. + +So the honest cost model of the tiled design: **O(points) once at build, O(visible tiles) per frame, O(visible points) on deep zoom past the pyramid floor.** This is also exactly the structure Tier 3 needs (§28) — Tier 2 and Tier 3 share the tile machinery; Tier 3 just adds not-all-tiles-resident. @@ -322,21 +353,53 @@ per-trace-kind rules live in the LOD/Tiling Contract (§28). Tier transitions are automatic and hysteresis-guarded (to avoid thrashing at the boundary), and every downsampling decision is logged so we never *silently* hide data. +### 5.1 Tuning constants — `python/xy/config.py` + +Every tier/decimation threshold lives in one module (§28: no silent decisions). The +table is the complete contents of `python/xy/config.py`; values are the ones shipped +today. "Read by" lists the modules that *consume* the constant — `python/xy/_figure.py` +re-exports several of them as a historic import path and is not listed for those. + +| Constant | Value | What it gates | Read by | +| --- | --- | --- | --- | +| `PROTOCOL_VERSION` | `3` | Wire-spec version stamped on every payload; the client refuses a mismatch loudly (§33). | `_payload.py` | +| `DECIMATION_THRESHOLD` | `10_000` | Line/area traces with more points than this ship M4-decimated (Tier 1); at or below, raw columns go over the wire. Also gates re-decimation on the interaction path. | `_payload.py`, `interaction.py` | +| `SCATTER_DENSITY_THRESHOLD` | `200_000` | Tier-0 → Tier-2 count budget for a scatter with **no** per-point channel (`Trace.use_density()`), and the visible-count budget for view-LOD planning and drill decisions. | `_trace.py`, `interaction.py`; mirrored client-side as `LOD_DIRECT_POINT_BUDGET` in `js/src/45_lod.js` | +| `DIRECT_SOFT_CEILING` | `2_000_000` | Tier-0 → Tier-2 count budget for a scatter that **does** carry a per-point color or size channel; above it density is forced and the channels are warned about, never silently dropped (§5 F5). | `_trace.py`, `marks.py` | +| `DENSITY_GRID` | `(512, 384)` | Default density-grid cell dimensions for the initial spec, before the client requests a viewport-matched size via `density_view`. | `_payload.py` | +| `MAX_SCREEN_DIM` | `4096` | Upper clamp on any browser-supplied pixel dimension, so untrusted widget/comm input cannot inflate decimation buckets or density grids. | `lod.py`, `_native.py` | +| `MAX_CONTOUR_WORK` | `4_000_000` | Ceiling on contour `cells × levels`; a request over it raises instead of allocating an unbounded segment buffer. | `marks.py`, `_native.py` | +| `DRILL_EXIT_FACTOR` | `1.15` | Hysteresis multiplier on the drill boundary: a trace already drilled to real points stays drilled until the visible count exceeds `budget × 1.15`. | `lod.py` (`drill_decision`, `plan_view_lod`), `interaction.py`; mirrored as `LOD_DRILL_EXIT_FACTOR` in `js/src/45_lod.js` | +| `DENSITY_TARGET_POINTS_PER_CELL` | `16.0` | Target points per cell when sizing an aggregation grid, so a barely-over-budget view does not get a one-point-per-pixel grid that looks like static and re-ships large. | `lod.py` | +| `DENSITY_SAMPLE_TARGET` | `8_192` | Size of the deterministic real-point sample shipped over an aggregated scatter's density texture (hybrid overlay). | `_payload.py`, `interaction.py` | +| `DENSITY_SAMPLE_SEED` | `0` | Seed for that sample; a fixed seed makes the overlay identical across re-ships of the same view. | `_payload.py`, `interaction.py` | +| `DEFAULT_PALETTE` | 10 CVD-safe hex entries | Per-trace default color cycle and the fallback categorical palette when a channel supplies none (§20/§36). | `marks.py`, `_payload.py`, `_svg.py`, `_raster.py` | +| `PYRAMID_MIN_POINTS` | `2_000_000` | Trace size at/above which a Tier-3 tile pyramid is built lazily; smaller traces never pay for one. | `interaction.py` | +| `PYRAMID_BASE_DIM` | `2048` | Edge of the pyramid's base level in cells (`dim²` u32 counts, ~1/3 overhead for the coarser levels); sets resident pyramid bytes. | `interaction.py` | + +Nothing in this table folds mark pixel area or overdraw into a tier decision — that is +F3, still pending (above). + --- ## 6. Memory-reduction techniques (checklist) - ✅ Binary Arrow transport — no JSON, no parse bloat, no string numbers. - ✅ Single physical copy — references through the pipeline, not clones. -- ✅ f32 coordinates by default; f64 opt-in. +- ✅ f64 canonical CPU-side, uploaded as window-centered offset-encoded f32 (§4/§16) — + *not* "f32 by default", which §4 rejects outright. - ✅ Struct-of-Arrays typed columns → direct GPU upload. - ✅ Dictionary encoding for categoricals. -- ✅ Drop/free CPU buffers after GPU upload; `mmap` on native. +- ✅ GPU buffers are droppable caches, rebuilt from the retained canonical CPU store + (§27 rule 1); `mmap` on native. Dropping the *canonical* copy after upload is a + narrow, explicit opt-in for static Tier-0 traces, never the default (§4). - ✅ Aggregation tiers so hot memory is screen-bounded, not data-bounded. - ✅ **Ring buffers for streaming** — fixed-capacity circular GPU buffer; appends overwrite oldest, constant memory, no re-allocation, no re-serialize. -- ✅ SharedArrayBuffer between worker and main thread where available; **transferable - ArrayBuffers (zero-copy move) as the universal fallback** — never postMessage-copy. +- ✅ No worker/main-thread data duplication: the core computes in the Python process and + the client receives screen-bounded buffers over the comm channel (§8/§37). The + SharedArrayBuffer/transferable-ArrayBuffer scheme belonged to the dropped in-browser + WASM core (§8). - ✅ Retained scene graph + buffer diffs — updating a color is a uniform write, not a data re-upload. - ✅ Large columns live *outside* WASM linear memory (JS ArrayBuffers / GPU); linear @@ -368,21 +431,27 @@ boundary), and every downsampling decision is logged so we never *silently* hide ## 8. Compute & threading -- The Rust core runs in a **Web Worker** (WASM); the main thread only mounts, forwards - input, and draws chrome. Big data never blocks the UI. -- Heavy stages (decimation, binning, autorange, KDE, stacking) run in the worker on the - columnar buffers. -- **SharedArrayBuffer where available — transferable ArrayBuffers as the mandatory - fallback.** SAB requires cross-origin isolation (COOP/COEP response headers), which - Jupyter, embedded iframes, and most third-party contexts **cannot set**. Those are - precisely the environments the "runs everywhere Plotly does" requirement exists for, - so SAB cannot be a load-bearing assumption. The fallback is `postMessage` with - **transferable** ArrayBuffers — a zero-copy *move* (ownership transfer) rather than - a share. It forbids concurrent access but costs no copies; the pipeline is designed - around ownership handoff (ingest → worker → upload) so both modes use the same code - path. SAB is a latency optimization, not a correctness dependency. (Same story for - `OffscreenCanvas`: use when present, render on main thread from worker-computed - buffers when not.) +- The Rust core runs **natively inside the Python process**, loaded as a C-ABI cdylib + through `ctypes` (`python/xy/_native.py`; `Cargo.toml` `crate-type = ["cdylib", + "rlib"]`). It is not compiled to WASM and does not run in a browser thread. Heavy + work happens off the browser entirely, so the UI thread is never the bottleneck for + big data. +- Heavy stages (decimation, binning, autorange, KDE, stacking) run in the kernel on the + columnar buffers; NumPy arrays are passed to the core by pointer, without copying. +- The browser side is a **thin JS/WebGL2 client** on the main thread: it mounts, + forwards input, uploads the screen-bounded buffers the kernel computes, and draws + chrome. Results arrive over the comm channel (§37), not over `postMessage`, so + neither SharedArrayBuffer nor cross-origin isolation (COOP/COEP) is required — + which is what lets the client run in Jupyter, embedded iframes, and third-party + contexts that cannot set those headers. +- One Web Worker exists client-side, and it is not the core: `js/src/46_worker.js` + re-bins the retained density sample for kernel-less standalone exports (`to_html`), + off the main thread, booted from a Blob URL. Environments without workers fall back + to the stretched overview texture. +- *Dropped path, kept for history:* the original design put the Rust core in a Web + Worker as WASM, with SharedArrayBuffer transport and transferable ArrayBuffers as the + universal fallback. §32 supersedes it. A WASM client remains a future pure-browser + option. - The *same Rust* compiled **native** does headless static export (PNG/SVG/PDF) with no browser — faster than Kaleido. **Consistency claim, stated honestly:** *logically* identical (same scales, same layout, same LOD decisions — guaranteed by sharing the @@ -474,7 +543,7 @@ struct LodCache { | SAB unavailable (no COOP/COEP in notebooks/iframes) | Transferable-ArrayBuffer ownership-handoff path is the default design; SAB is an optimization (§8) | | Timestamps/large magnitudes break f32 | Offset+scale encoding is the default upload path; f64 view transform on CPU (§4, §16) | | WebGL2 can't do compute-shader binning | Additive-blend float-target binning; worker-side SIMD binning as last resort (§5) | -| Browser caps live GPU contexts (~8–16); dashboards want 30+ charts | One shared context + per-chart viewport/scissor compositing (§18) | +| Browser caps live GPU contexts (~16 in Chrome); dashboards want 30+ charts | Per-chart context under an LRU governor, budget 12 (§18) | | VRAM exhaustion / device loss | Byte-budgeted caches with eviction; full GPU state rebuildable from scene graph (§6, §18) | | Canvas is invisible to screen readers | Structured a11y layer: ARIA summary, keyboard nav, data-table export (§20) | | WASM bundle bloat vs plotly.js partial bundles | Feature-gated trace modules, size budget in CI (§23) | @@ -515,7 +584,7 @@ missing entirely.* | 4 | Tier-2 GPU binning assumed compute shaders + atomics; WebGL2 has neither — flagship feature silently absent on fallback | **Critical — feature gap** | Three-implementation ladder: compute / additive-blend float target / worker SIMD (§5) | | 5 | "Free CPU buffer after upload" is a no-op for browser memory — wasm32 linear memory never shrinks, caps at 4 GB | **Major — memory claim** | Large columns never enter linear memory; screen-sized scratch arenas (§4, §6) | | 6 | Hover/pick undefined for aggregated tiers; naive pick readback + LOD recompute breaks interaction latency | Major | Interaction latency model, stale-while-revalidate, progressive refinement (§17) | -| 7 | No multi-chart story; browsers cap live GPU contexts (~8–16), dashboards want 30+ | Major | Shared-context renderer with viewport compositing (§18) | +| 7 | No multi-chart story; browsers cap live GPU contexts (~16 in Chrome), dashboards want 30+ | Major | Per-chart context under an LRU governor, budget 12 (§18) | | 8 | No null/NaN semantics; NaN in f32 vertex data corrupts primitives | Major | Validity bitmaps end-to-end, gap semantics (§19) | | 9 | Canvas rendering is an accessibility regression vs Plotly's SVG | Major | Structured a11y layer (§20) | | 10 | Autorange is an O(n) full scan per update | Moderate | Chunk zone maps make it O(chunks) (§22) | @@ -567,22 +636,36 @@ Part I said "60fps" without defining what has to happen inside a frame. The budg ## 18. Many charts per page (the dashboard problem) Plotly's real-world habitat is dashboards with 10–50 figures. Browsers cap live -WebGL/WebGPU contexts per page (typically 8–16; oldest silently dies). One context per -chart is therefore an architecture bug, not a scaling detail. - -- **One renderer, one GPU context per page** (per worker). Each chart is a *client* - of the shared renderer: it owns a scene subgraph and a target rectangle. +WebGL contexts per page (~16 in Chrome) and LRU-evict the oldest on overflow, which +permanently blanks the earliest charts of a big dashboard. + +**Shipped: one context per chart, governed.** `XY_CONTEXT_GOVERNOR` +(`js/src/50_chartview.js`) keeps the page inside a budget — default **12**, overridable +via `window.XY_CONTEXT_BUDGET` — leaving headroom under Chrome's cap for host-page GL. +When a view is about to acquire a context at budget, the least-recently-visible +*off-screen* view releases its own via `WEBGL_lose_context` and re-acquires when +scrolled back into view; an over-budget panel keeps showing its last frame as a static +image. Under the budget nothing is ever released. Every decision is observable: +`data-xy-ctx` on the canvas reads `live` | `released` | `lost`. See +`spec/process/production-readiness.md` §"WebGL context cap" for the claim limits. + +**Device/context loss is a first-class event:** all GPU state is derived state, rebuilt +from the scene graph + column store on a new context. The visible cost is one reupload +flicker, never lost data. The governor depends on this — a governed release is a +deliberate context loss put through the same restore path. + +*Unimplemented design option — shared-context compositing:* + +- **One renderer, one GPU context per page.** Each chart becomes a *client* of the + shared renderer: it owns a scene subgraph and a target rectangle. - Two compositing modes, chosen per environment: (a) a single full-page canvas behind the DOM, charts drawn into scissored viewports — cheapest, works everywhere; or (b) per-chart canvases fed by `transferControlToOffscreen` / `drawImage`-from-shared-framebuffer where layout demands real DOM interleaving. -- Shared context also means **shared caches**: two charts of the same DataFrame +- Shared context would also mean **shared caches**: two charts of the same DataFrame reference the same columns and the same GPU buffers — a dashboard of 20 views of one 10M-row table holds the data **once**, which is a memory win Plotly cannot express at all. -- **Device/context loss is a first-class event** (mandatory in WebGPU's model): all - GPU state is derived state, rebuilt from the scene graph + column store on a new - device. The visible cost is one reupload flicker, never lost data. ## 19. Nulls, NaN, and gaps @@ -661,10 +744,15 @@ Where it must run, and what each environment denies us: | Old browsers / no WebGL2 | GPU entirely | same pure-JS + 2D-canvas fallback, capped; loudly reported via the §5 no-silent-caps rule | | Server / CI (native) | no display | headless native path (§8) | -**Bundle budget is a CI-enforced number, not an aspiration:** core (spec + layout + -Tier 0/1 + cartesian traces) ≤ **1.5 MB** wasm+js gzipped; every additional trace -family is a lazily-loaded feature module (Plotly's partial-bundle pain, solved by -default). A size regression fails the build exactly like a perf regression (§12). +**Bundle size.** The shipped client is a single JS bundle — `python/xy/static/index.js`, +309 KB uncompressed / ~74 KB gzipped — with no WASM payload and no lazily-loaded trace +modules. The one size gate CI enforces is on the **wheel**: `.github/workflows/ci.yml` +asserts the built wheel is ≤ 15 MB (§33). CI also verifies the committed JS bundles are +*fresh* (`node js/build.mjs` reproduces them), but it does not measure their bytes. + +*Pending:* a gzipped-size budget on the client bundle, failing the build exactly like a +perf regression (§12), plus per-trace-family lazy feature modules (Plotly's +partial-bundle pain). Neither exists today. ## 24. Extensibility & the compatibility contract @@ -881,9 +969,11 @@ hits a source build requiring a Rust toolchain — an instant adoption cliff. **Contracts that keep it honest:** - **Comm-protocol versioning.** The native core and JS client ship together but can - drift (cached notebook outputs, pinned server assets). Every message carries a - protocol version; mismatch fails **loudly with an upgrade hint**, never silently - renders wrong. + drift (cached notebook outputs, pinned server assets). The first-paint spec carries + a protocol version; mismatch fails **loudly with an upgrade hint**, never silently + renders wrong. Requests and replies carry no version of their own — the handshake + happens once, at first paint, before any request is possible + (`spec/design/wire-protocol.md` §7). - **No-wheel behavior is defined:** the native Rust core is required and there is no pure-Python fallback. A source install compiles the core (Rust toolchain required); if the core cannot be loaded — an unsupported platform with no wheel and no local @@ -952,8 +1042,9 @@ receive Arrow slices, not JSON. - **Phase 1** adds Filter Tier A (zone-map range filtering) — near-free once zone maps (§22) exist. - **Phase 2** adds Filter Tier B (visible-window re-bin) alongside the Tier-2 - pyramid — they share the SIMD binning kernel — and implements the fill-rate-aware - tier heuristic + buffer chunking (F3) in the render client. + pyramid — they share the SIMD binning kernel. The fill-rate-aware tier heuristic and + buffer chunking (F3) are **still pending**: the shipped tier decision is count-only + and the render client does not chunk vertex buffers (§5). - **Phase 4** adds Filter Tier C (summed-area index + linked views) with the shared-context dashboard work (§18) — same milestone because cross-filtering is a dashboard feature. @@ -988,37 +1079,43 @@ the chart match my site" actually means — typography and chrome. ```css .my-dashboard { - --chart-bg: #0f1520; - --chart-grid: #24313f; - --chart-font: "Inter", sans-serif; /* also drives DOM chrome */ - --chart-series-1: #6e9bff; /* indexed — override one without restating all, */ - --chart-series-2: #3fc3b1; /* and each index cascades independently */ - --chart-series-3: #e29350; - --chart-colormap: viridis; /* sequential ramp for Tier-2 density */ + --chart-bg: #0f1520; + --chart-grid: #24313f; + --chart-axis: rgb(226 232 240 / 55%); + --chart-text: #e5e7eb; /* chrome text: ticks, titles, legend, annotations */ } ``` -Mechanism: at mount the client resolves the `--chart-*` tokens and writes them to -renderer state — clear color, grid uniform, per-series palette buffer, colormap LUT -texture. Two implementation realities (audit round 4): **(1)** `getComputedStyle` -returns a custom property's *raw token stream*, not a resolved color — so color tokens -are resolved via a hidden **probe element** (assign `color: var(--chart-series-1)`, -read back the browser-computed rgb), which handles every CSS color format (`oklch()`, -`color-mix()`, named colors) without shipping a CSS color parser. **(2)** Series -tokens are **indexed** (`--chart-series-N`), not a space-separated list — a list can't -be overridden per-entry or cascaded per-index. Series beyond the highest defined index -cycle the defined set with a documented lightness rotation. Crucially, **because +Mechanism: `readTheme()` (`js/src/20_theme.js`) resolves the canvas tokens at mount and +writes them to renderer state — clear color, grid and axis uniforms, label color. Chrome +tokens are consumed directly by the stylesheet (`XY_CHROME_CSS`, zero-specificity +`:where()` rules) rather than through the renderer. One implementation reality (audit +round 4): `getComputedStyle` returns a custom property's *raw token stream*, not a +resolved color — so color tokens are resolved via a hidden **probe element** (assign +`color: var(--chart-bg)`, read back the browser-computed rgb), which handles every CSS +color format (`oklch()`, `color-mix()`, named colors) without shipping a CSS color +parser. Crucially, **because tokens flow through CSS variables, the cascade, inheritance, and media queries all work** — the *variables* cascade even though the pixels don't, so per-container theming, brand overrides, and `@media (prefers-color-scheme)` behave exactly as a CSS author expects. **The theme contract:** -- **A documented token vocabulary** — `--chart-bg`, `--chart-grid`, `--chart-axis`, - `--chart-font`, `--chart-series` (categorical list), `--chart-colormap` (named ramp - or stops), `--chart-selection`, `--chart-muted`. Unset tokens fall back to a built-in - theme; the categorical/sequential defaults are the accessible, CVD-safe palettes from - §20, not arbitrary. +- **A documented token vocabulary**, split by consumer. Canvas tokens read by + `readTheme()`: `--chart-bg`, `--chart-grid`, `--chart-axis`, `--chart-text`. Chrome + tokens read by the stylesheet: `--chart-tooltip-bg` / `--chart-tooltip-text`, + `--chart-legend-bg`, `--chart-badge-bg` / `--chart-badge-text`, `--chart-modebar-bg` / + `--chart-modebar-active`, `--chart-selection` / `--chart-selection-fill`, + `--chart-zoom-selection` / `--chart-zoom-selection-fill`, `--chart-crosshair`, + `--chart-annotation-text`, `--chart-cursor` / `--chart-cursor-pan`, `--chart-focus`. + Unset tokens fall back to a built-in theme (`currentColor` at documented opacities for + grid/axis/label). The public reference is `docs/styling/themes-and-tokens.md`. +- *Pending:* a **series-palette token** (`--chart-series-N`, indexed rather than a + space-separated list so entries cascade and override individually, cycling with a + lightness rotation past the highest defined index) and a **colormap token** + (`--chart-colormap`, named ramp or stops → LUT texture). Neither is wired: series + colors and colormaps come from the spec / `theme()` only. The categorical and + sequential defaults are the accessible, CVD-safe palettes from §20, not arbitrary. - **Live re-resolution.** The client watches `matchMedia('(prefers-color-scheme: dark)')` and a `MutationObserver` on the container's `class`/`data-theme`/`style`, re-resolving tokens on any change. Because of the retained scene graph (§7), **a theme change is @@ -1028,12 +1125,18 @@ author expects. - **Programmatic parity.** Everything a token sets is also settable from Python (`fig.theme(...)`) so notebook users who never touch CSS get the same control; the CSS bridge is the web-author's path to the same renderer state, not a separate system. -- **Export parity (audit round 4 — this was broken as first written).** Static export - renders kernel-side (§8, §32), where CSS does not exist — so a CSS-themed chart would - export with the wrong theme. Fix: the client sends a **theme snapshot** (its resolved - token values) back over the comm channel on every theme change; the kernel holds the - effective theme and export always matches the screen. If no client is attached - (headless script), the Python-set theme is authoritative. +- **Export parity — partially closed; the kernel path is still an open gap.** Two export + routes exist. *Client-side* (modebar download) is themed correctly: `_exportSvgMarkup()` + (`js/src/53_interaction.js`) inlines the resolved `--chart-*` tokens and inherited text + styles onto the detached clone before serializing, so the downloaded SVG/PNG matches + the screen. *Kernel-side* export (`to_svg` / `to_png` / `write_image`, rendered by + `python/xy/_svg.py` and `python/xy/_raster.py`) has no CSS: it uses only the + Python-set theme (`xy.theme(...)`), so a chart themed purely through CSS custom + properties exports with the Python theme, not the on-screen one. + *Pending:* a **theme snapshot** — the client sending its resolved token values back + over the comm channel on every theme change, so the kernel holds the effective theme. + No producer or handler exists today. With no client attached (headless script) the + Python-set theme is authoritative and correct either way. - **Escape hatches.** `MutationObserver` misses stylesheet swaps and non-color-scheme media flips → a public `refreshTheme()` exists for apps that restyle dynamically. `forced-colors: active` (Windows High Contrast) restyles DOM chrome automatically @@ -1047,7 +1150,7 @@ reachable by CSS and never will be — there's no node. This goes through the sp selected/hover/highlight styling, resolved on the GPU. This is a real limitation, not an oversight: it's the same reason the engine scales. -**Summary:** chrome is plain CSS; background/grid/palette/colormap are CSS via the +**Summary:** chrome is plain CSS; background/grid/axis/text are CSS via the `--chart-*` token bridge (cascade + dark-mode included); per-mark data-driven styling is spec-level. The one thing we explicitly *don't* promise is arbitrary per-element CSS selectors on marks — the price of the pixel-based scale everything else buys. @@ -1089,7 +1192,7 @@ everything, so the manifest mechanism *is* the recovery mechanism. | Filter toggle **back** to a previous state | **0 bytes** (typ.) | old `filter_hash` generation still cached — flipping US on/off re-sends nothing | | New filter state | recomputed *visible* tiles only | §34 Tier A/B; tagged with new `filter_hash` | | Streaming append | dirty tile cells + ring delta | O(appended), per §28 | -| Theme / style change | **0 bytes** | client-side uniforms/LUT (§36); tiny theme-snapshot upstream | +| Theme / style change | **0 bytes** | client-side uniforms/LUT (§36); nothing goes upstream | | Hover / pick | ~1 row on drill | pick resolves client-side; drill fetches top-k rows | | New trace on same DataFrame | new columns only | column IDs shared across figures — a dashboard of 20 views of one table transfers the data **once** (§18) | diff --git a/spec/design/chart-grammar.md b/spec/design/chart-grammar.md new file mode 100644 index 00000000..d867d2de --- /dev/null +++ b/spec/design/chart-grammar.md @@ -0,0 +1,335 @@ +# Core Declarative Chart Grammar + +**Status:** API design proposal. Goal: fix the composition model **before** +the catalog grows to 40 kinds, so nothing here forces a public-API rewrite +later. Grounded in what ships today (`python/xy/components.py`: `Chart` + +`Mark` + `Axis` + `Legend`, `_MARK_APPLIERS` registry; `python/xy/marks.py`: +the mark implementations, bound as `Figure`'s fluent methods at +`python/xy/_figure.py:340-362` so `Figure.scatter is marks.scatter`; +`python/xy/_payload.py:278`: the `_emit_` dispatch +(`getattr(self, f"_emit_{t.kind}", None)`), reachable on `Figure` through +`PayloadMixin`) — the proposal is an *extension* of that shape, not a +replacement. + +Related: `reflex-shaped-api.md` covers the public API/styling proposal for a +Reflex-like component surface without making Reflex a core dependency. + +## 1. The model in one paragraph + +A **Figure** is a grid of **Panels** (1×1 today; subplots later). +A Panel owns **Scales** (any number of named x/y axis ids) and a z-ordered +list of **Marks**. A Mark = kind + data bindings + style props + optional channel +encodings. Axes, legend, and title are Panel/Figure chrome that *read* scales +and marks — they never own data. Everything is a plain declarative node +(dataclass), composable as children, and compiles to the existing internal +engine figure + wire spec. One public front door over that engine: + +- **Compositional (Reflex-flavored):** `xy.chart(xy.scatter(...), xy.line(...), + xy.x_axis(...))` — declarative, component-tree shaped, what a Reflex wrapper + serializes naturally. (The internal `_figure.Figure` fluent methods share the + same mark implementations, so the vocabulary cannot fork.) + +Rule G0: **the public composition API and the internal mark core are the same +vocabulary** (same mark names, same prop names, same defaults — one +implementation in `marks.py`). This is already true and is the thing we must +not break. + +## 2. The layering/overlay rules (what makes composition sound) + +- **G1 — Marks layer by order.** Children render in listed order (painter's + model). No z-index prop until a real need appears. +- **G2 — One shared coordinate space per panel.** All marks in a panel share + x and y scales; autorange is the union of every mark's contributing columns. + The per-kind hook is `Figure._range_columns(trace, axis_id)` + (`python/xy/_figure.py:1024`), which switches on `trace.kind` — area and + error_band contribute `[y, base]`, triangle_mesh `[x0, x1, x]`, rect-like + kinds `[x0, x1]` — and the union, padding, and log clamping happen in + `Figure._range` (`python/xy/_figure.py:840`). The hook lives on `Figure`, + not on `Trace`. A mark never gets a private scale — the escape hatch is a + second *panel*, or an explicit named scale ★: `xy.y_axis(id="y2", + side="right")` plus `mark(..., y_axis="y2")`. Silent dual axes are how + charts lie, so y2 is loud by construction: every mark factory takes + `x_axis`/`y_axis` id strings, and referencing an id with no matching axis + node is a build error (`… has no matching xy.y_axis(id='y2')`, + `python/xy/components.py:3828`). +- **G3 — Scale type is a panel decision** (`linear | time | log | category`), + auto-inferred from marks (time columns → time; bar categories → category) + but overridable on the axis node. Mixing marks whose natural scales + conflict (bar-category + scatter-linear x) is a build-time error with a + fix-it message, not a coercion. +- **G4 — Chrome reads, never owns.** Legend derives entries from mark + channel modes (already true); axes derive from scales; tooltips derive + from the hovered mark's readout row. Adding a mark kind never edits chrome + code (contract already enforces this via capabilities). +- **G5 — Declarative all the way down.** Every node is data (kind + props). + No callbacks in the tree except the event props (`on_hover`, `on_click`, + `on_brush`, `on_select`, `on_view_change`), which is exactly what a Reflex + component can serialize + wire to server events without escape hatches. + +## 3. The 10 common charts (all expressible today or with planned nodes) + +```python +import xy + +# 1. line +xy.chart(xy.line(x="date", y="close", data=df), title="Price") +# 2. multi-series line (wide → long handled by repeated marks) +xy.chart(xy.line(x="date", y="aapl", data=df, name="AAPL"), + xy.line(x="date", y="msft", data=df, name="MSFT"), xy.legend()) +# 3. scatter with channels +xy.chart(xy.scatter(x="gdp", y="life", color="continent", size="pop", data=df)) +# 4. big scatter (auto density tier — same call, no special API) +xy.chart(xy.scatter(x="x", y="y", data=ten_million_rows)) +# 5. area +xy.chart(xy.area(x="date", y="active_users", data=df)) +# 6. histogram +xy.chart(xy.histogram(values="latency_ms", bins=256, data=df)) +# 7. bar +xy.chart(xy.bar(x="region", y="revenue", data=df)) +# 8. horizontal bar (x is always the category arg; orientation moves it to y) +xy.chart(xy.bar(x="region", y="revenue", data=df, orientation="horizontal")) +# 9. heatmap +xy.chart(xy.heatmap(z=matrix, colormap="viridis")) +# 10. time series with unit-scale y +xy.chart(xy.line(x="ts", y="value", data=df), xy.y_axis(label="watts")) +``` + +(`xy.chart` is the kind-neutral container; the existing `scatter_chart`/ +`line_chart`/… wrappers remain as readable aliases — they already just tag +`Chart(kind_str, children)`.) + +## 4. The 5 complex overlays (the composition stress tests) + +```python +# A. line-on-scatter (regression overlay) — shared scales, order = layering +xy.chart( + xy.scatter(x="x", y="y", data=df, opacity=0.4), + xy.line(x=xs_fit, y=ys_fit, color="var(--accent)", width=2), +) + +# B. area-under-line (band + emphasis line) +xy.chart( + xy.area(x="date", y="p95", base="p5", data=df, opacity=0.25, name="p5-p95"), + xy.line(x="date", y="median", data=df, name="median"), +) + +# C. histogram + KDE-style line +xy.chart( + xy.histogram(values="dur", bins=200, density=True, data=df), + xy.line(x=kde_x, y=kde_y, width=2), +) + +# D. volume-under-candles (finance pane pair) — PANELS, not one panel: +xy.figure( + xy.panel(xy.candlestick(x="t", open="o", high="h", low="l", close="c", data=df), + height=3), + xy.panel(xy.bar(x="t", y="volume", data=df), height=1), + link_x=True, # shared x scale + synced pan/zoom across panels +) + +# E. threshold rule + annotated scatter +xy.chart( + xy.scatter(x="x", y="y", color="cluster", data=df), + xy.hline(0.8, color="#ef5350"), # rule annotation + xy.label(3.2, 0.85, "SLA"), # text annotation +) +``` + +D is the deliberate boundary case: volume-under-candles is **not** an overlay +(different y meaning) — the grammar answers it with linked panels, matching +the "chart owns rendering, app owns chrome" decision already taken. `link_x` +is the chart-level primitive (one shared x scale + view sync); arranging the +panes is layout the Figure grid owns. + +## 5. Node vocabulary (target; ★ = exists today) + +- Containers: `figure` (grid of panels; today implicit 1×1), `panel` + (planned), `chart` ★ (sugar: figure with one panel). +- Marks ★: `scatter, line, area, histogram, bar/column, heatmap, errorbar, + error_band, box, violin, ecdf, hexbin, contour, step, stairs, stem` (+ every + future kind — one `Mark` node each, registry-dispatched). +- Annotations ★ (tiny data): rules `hline`/`vline`/`threshold`, bands + `x_band`/`y_band`/`threshold_zone`, and `label`/`text`/`marker`/`arrow`/ + `callout`. These are not literally Mark nodes: they compile to an + `Annotation` node dispatched through `_ANNOTATION_APPLIERS` + (`python/xy/components.py:4370`), a sibling registry to `_MARK_APPLIERS` + (`components.py:4347`) with the same kind→applier shape. +- Chrome: `x_axis`/`y_axis` ★ (take `id=`, so named secondary scales are + already expressible; grows `type_="log"|"category"` ★partial), `legend` ★, + `title` (prop today, node later if styling demands). +- Events ★: `on_hover`, `on_click`, `on_brush`, `on_select`, + `on_view_change`. All five are `Chart` constructor params + (`python/xy/components.py:2559-2563`), fields of `ChannelCallbacks` + (`python/xy/channel.py:90-94`), and dispatched by the channel layer + (`handle_message` for hover/click/view_change, `_selection_reply` for + brush/select); + `on_view_change` carries the viewport for server-driven cross-filtering + and is wired in the Reflex wrapper + (`python/reflex-xy/reflex_xy/component.py:85`). +- Interaction ★: `interaction_config` — see §5.1. +- Faceting ★: `facet_chart` — see §5.2. + +### 5.1 `interaction_config` — the gesture and event switchboard + +`xy.interaction_config(...)` (`python/xy/components.py:2424`) builds an +`Interaction` node (`components.py:291`) carrying nine behavioral switches +plus two cross-chart linking props. Every switch defaults to `None`, meaning +"unset"; the renderer resolves an unset switch through +`ChartView._interactionFlag(name, fallback)` +(`js/src/50_chartview.js:436`), which treats only the literal `true` as on: + +| Switch | Default when unset | Effect | +| --- | --- | --- | +| `hover` | off | Pointer motion emits hover events and drives the tooltip. | +| `click` | off | Picked marks emit click events. | +| `crosshair` | off | Plot-aligned hover guides are created. | +| `view_change` | off | Pan/zoom/reset emit range events. | +| `select` | on | Shift-drag box selection. | +| `brush` | on | Brush selection (also requires `select`). | +| `pan` | on | Plain-drag pan. | +| `zoom` | on | Wheel, box zoom, double-click reset, modebar zoom. | +| `navigation` | on | Master gate: when off, neither pan nor zoom gestures run. | + +`navigation` is checked before `pan` and `zoom` at each gesture site +(`js/src/53_interaction.js:112`, `:240`, `:251`), so it disables pointer +navigation wholesale while leaving `pan`/`zoom` as the finer-grained +switches. + +The Python side sets these implicitly from the callbacks a `Chart` was given: +`hover`, `click`, `brush`, `select`, and `view_change` are emitted as `True` +exactly when the matching handler is present (`components.py:2736-2740`). +That pass runs *last* — after the chart-level keywords (`components.py:2708`) +and after any `interaction_config` nodes (`:2722`) — and it overwrites rather +than defaults: `on_hover=` together with `hover=False` yields `hover=True`. +[interaction.md](../api/interaction.md) §1 is the authority on resolution order. + +Channel traffic follows `view_change` specifically. `_emitViewChange` +(`js/src/50_chartview.js:460`) coalesces to one `requestAnimationFrame`, and +sends `comm.send({type: "view_change", …})` only when the flag is on — so +disabling it removes the per-viewport server round-trip while leaving local +pan/zoom fully interactive. + +**Cross-chart linking.** `link_group` is an opaque identifier; charts sharing +one join the `BroadcastChannel` named `` `xy:${group}` `` +(`js/src/50_chartview.js:487`). `link_axes` defaults to `("x", "y")` and is +filtered to those two names at runtime; only the listed dimensions of the +broadcast view are copied onto the receiving chart. Semantics that matter: + +- What propagates is the **view window** (`x0/x1/y0/y1`), not data, selection, + or scale type. A receiver ignores messages tagged with its own source id, so + linking does not echo. +- A receiver applies a linked view only if `pan` or `zoom` is enabled + (`js/src/50_chartview.js:514`); a fully navigation-locked chart broadcasts + but does not follow. +- Being in a link group makes a chart compute view events even with + `view_change` off — it broadcasts locally without sending to the server. +- Selections propagate only under the separate `link_select` wire flag + (range, polygon, or clear — `js/src/50_chartview.js:491`), applied without + re-dispatching events so linked charts do not feed back. `link_select` is + public through `Figure.set_interaction(link_select=…)` (`_figure.py:257`, + applied at `:270`) and through `facet_chart(link_select=True)` + (`components.py:3679`). It is the one linking switch with no + `interaction_config` parameter. + +### 5.2 `facet_chart` — small multiples + +`xy.facet_chart(*children, by, cols=3, share_x=True, share_y=True, link=None, +link_select=False, gap=12)` (`python/xy/components.py:4480`, class at +`:3511`) repeats the child composition once per distinct value of `by`. +`by` is required (`TypeError` when omitted), `cols` must be a positive +integer, and `gap` a non-negative one. + +- **Panel derivation and order.** `by` resolves to a column of the + chart-level data or to a per-row array (`python/xy/facets.py:79`). Rows + group by their `category_label` display string — matching categorical + channels — and panels appear in **first-seen row order**, not sorted order; + the `np.unique` fast path explicitly restores first-seen order + (`facets.py:108-113`). Each panel is built over its row subset + (`_subset_data`, `facets.py:24`). +- **Layout.** `cols` is the maximum column count; rows are + `ceil(n_panels / cols)` and one panel is + `max(120, (width - (cols - 1) * gap) // cols)` pixels wide + (`FacetGrid.rows`, `FacetGrid.panel_width`, `facets.py:146-154`). Each + panel's chart title is its facet label. +- **`share_x` / `share_y` are global, not per-panel.** For each shared axis + id the grid takes every panel's `_range(axis_id)` and applies the merged + `(min, max)` to all panels (`components.py:3657-3666`). Categorical axes + are unioned in first-seen order and the panels are **rebuilt** with that + order pre-seeded, because category positions commit at ingest + (`components.py:3646-3656`). An axis that is categorical in some panels and + numeric in others cannot be shared: it warns and skips sharing for that + axis only. +- **`link` is runtime sync, `share_*` is initial domain.** `link=True` or + `"both"` links x and y; `"x"`/`"y"` links one; `False`/`None` disables. + A linked dimension is force-shared even when its `share_*` is `False` + (`components.py:3610-3615`), since panels starting from incomparable + domains would jump on first interaction. `link` and `link_select` assign + all panels one generated `link_group` (`xy-facet-`), so propagation is + exactly the §5.1 BroadcastChannel path; `link_select` additionally echoes + data-space selections across panels. +- **Decimation budget is per panel.** `FacetGrid` composes N *independent* + figures rather than one multi-panel spec, and each is built with + `px_width=panel_width` (`facets.py:185`). LOD therefore targets each + panel's own narrower width; total decode and draw work scales with panel + count rather than being budgeted across the grid. A grid-wide budget is + pending. + +## 6. Reflex integration (no escape hatches) + +The tree above is precisely a Reflex component tree's shape: snake_case +props, children composition, `data=` + column-name resolution (`data_key` +idiom), event props. It remains a XY-owned tree, not a Reflex object, +so the core package keeps zero Reflex dependencies. A future Reflex wrapper is +therefore a *thin* codegen layer: + +1. Each `xy.*` factory maps 1:1 to a Reflex component; props serialize as-is + (they're plain scalars/strings/arrays). +2. Data flows as the existing binary payload through a Reflex asset/endpoint + (the 100M live-drilldown demo already proves the comm shape: a `comm` + object with `send`/`onMessage` over any transport). +3. Events (`on_hover/on_select/on_view_change`) map to Reflex event handlers; + payloads are the same dicts the widget path already emits. +4. State-driven restyle = re-render with new props; data identity is kept by + the column-store handles so unchanged columns don't re-ship (buffer-diff + updates, dossier §6 — planned, and this is the API that needs it). + +Anti-goals, stated so they stay out: no `raw_plotly_json`-style passthrough +node, no per-mark imperative `draw(ctx)` callback, no CSS-in-props for mark +geometry (theming stays on the `--chart-*` token path). + +## 7. Migration & compatibility + +- Everything shipped keeps working: `chart(...)` is additive sugar; + `panel`/`figure` introduce multi-pane without touching single-pane specs + (spec gains `panels: [...]` only when >1 — protocol bump at that point, + bundled with the view-message unification the contract already schedules). +- The wire spec keeps its current per-trace shape; panels reference traces by + id. Axes are already explicit spec objects keyed by axis id + (`"axes": {axis_id: …}`, `python/xy/_payload.py:238`); hoisting scale type + and domain into a separate `scales` object rides the same bump. +- The `Chart(kind_str, ...)` wrappers never encoded behavior (kind string is + cosmetic) — safe to keep forever. + +## 8. Implementation order + +1. ★ Done. `xy.chart` (`python/xy/components.py:4380`) plus the annotation + set — rules, bands, and text nodes shipped under the names in §5, through + `_ANNOTATION_APPLIERS` rather than as `Mark` nodes. +2. Partial. Per-axis spec objects ship: `build_payload` emits + `"axes": {axis_id: …}` over `self.axis_options` + (`python/xy/_payload.py:238`). Still pending: hoisting scale type and + domain into a distinct `scales` object, and completing `category`/`log`. +3. Pending. `panel`/`figure` grid + `link_x` view sync (enables the finance + pair and subplot grids; carries the protocol bump). No `xy.panel` or + `xy.figure` factory exists yet — §4 example D remains aspirational. + Cross-chart view sync itself already ships via `link_group` (§5.1), so + this item is now the *layout* half only. +4. ★ Done. Named secondary scales ship as `xy.y_axis(id="y2", …)` + + `mark(..., y_axis="y2")`, with the loud opt-in enforced by the + no-matching-axis build error (§5.1 G2). The `secondary=True` spelling was + dropped in favor of axis ids. +5. ★ Done, with a different shape than planned: `xy.facet_chart(by=…)` + (§5.2), not `facet(col=…)`, and it composes N independent figures in a + `FacetGrid` rather than compiling to a panel grid. Folding it onto the + item-3 panel grid once that lands is pending, as is a grid-wide + decimation budget. diff --git a/docs/engineering/design/lod-architecture.md b/spec/design/lod-architecture.md similarity index 88% rename from docs/engineering/design/lod-architecture.md rename to spec/design/lod-architecture.md index 41c5efed..d0d2a73c 100644 --- a/docs/engineering/design/lod-architecture.md +++ b/spec/design/lod-architecture.md @@ -16,9 +16,13 @@ Interactive means: pan/zoom stays inside the §17 frame budget at any N. ## 1. The tier ladder (per-kind, not global) A **tier is a property of a (trace, viewport) pair**, never of a dataset: -`tier = f(visible_count, mark_pixel_area × overdraw)` (§5). Implemented today -for scatter (drill-in/out with hysteresis); this doc extends the same rule to -every kind. +what ships is count-only, `tier = f(visible_count)`, hysteresis-guarded (§5). +`drill_decision(visible, budget, in_drill, exit_factor)` in `python/xy/lod.py` +returns `visible <= budget * (exit_factor if in_drill else 1.0)`; `js/src/45_lod.js` +mirrors it. Implemented today for scatter (drill-in/out with hysteresis); this +doc extends the same rule to every kind. Folding `mark_pixel_area × overdraw` +into the decision is dossier F3 — *specified, pending, not implemented*; no +pixel-area or overdraw term exists in `python/xy/` or `js/src/` today. | Tier | Name | Representation | Cost model | Status | |---|---|---|---|---| @@ -258,14 +262,26 @@ contract entry before it lands. Pyramid-served density views still need tile-aware sample overlays so the same anti-shimmer contract holds without rescanning raw rows. -**Phase 3 — pyramid (~2-3 wks)** -6. `src/tiles.rs`: pyramid build + tile fetch (C ABI: `xy_pyramid_build`, - `xy_pyramid_tile`); Python `Pyramid` cache keyed per trace; build lazily - on first Tier-2 entry above a size threshold (recorded). -7. `density_view` serves from pyramid when present (level select + compose); - below-floor re-bin via `range_indices`. +**Phase 3 — pyramid (build + serve shipped; client cache and bench gate open)** +6. **Done (count plane only):** `src/tiles.rs` builds a square count pyramid + over the trace's full data bounds — finest level is `PYRAMID_BASE_DIM`² + (2048², `python/xy/config.py`), each coarser level an exact 4→1 u64 sum + saturating to u32, so every level conserves total count. C ABI is + `xy_pyramid_build` / `xy_pyramid_append` / `xy_pyramid_count` / + `xy_pyramid_compose` / `xy_pyramid_free` (no per-tile fetch entry point; + composition happens kernel-side). Handles are slab indices behind a mutex, + cached per trace and built lazily by `interaction._ensure_pyramid` on the + first density view at ≥ `PYRAMID_MIN_POINTS` (2,000,000), released by a + weakref finalizer. Channel planes (§2, §4.1) are not built yet. +7. **Done:** `density_view` estimates the window with `pyramid_count` and + serves it with `pyramid_compose` when that estimate sits safely above the + drill threshold; `compose` picks the coarsest level that still meets the + render resolution and refuses beyond `MAX_UPSAMPLE` (2×), so below-floor + and near-drill windows fall through to the exact `range_indices` + + `bin_2d` path. Level is recorded per update as `binning: "pyramid-L"`. 8. Client: tile-keyed cache replaces window-keyed `densityCache` (same - eviction, same crossfades); `pyramid_level` in updates. + eviction, same crossfades). Still pending — `js/src/45_lod.js` keys the + cache by density window, and no client code reads the served level. 9. Bench gate: 100M pan p95 < 16ms kernel time, zoom step < 50ms, memory within 1.5× finest level. diff --git a/docs/engineering/design/reflex-integration.md b/spec/design/reflex-integration.md similarity index 83% rename from docs/engineering/design/reflex-integration.md rename to spec/design/reflex-integration.md index 57721ffb..6f659d15 100644 --- a/docs/engineering/design/reflex-integration.md +++ b/spec/design/reflex-integration.md @@ -51,11 +51,12 @@ per-request HTTP overhead, no SSE keep-alive tuning. proved it — but each piece costs something the socket gets free: reverse proxies must be taught each route; every `/msg` pays request setup + headers; SSE is a second long-lived connection per chart with its own reconnect -logic; and none of it inherits app-plane auth. The §3.2 binary frame format -(XYBF) exists because HTTP bodies need framing — socket.io attachments +logic; and none of it inherits app-plane auth. The XYBF binary frame format +(`python/xy/_framing.py`; versioning in [wire-protocol.md](wire-protocol.md) +§7) exists because HTTP bodies need framing — socket.io attachments already carry length-delimited binary, so on this transport the framing -layer disappears too. XYBF remains in `xy.channel` for HTTP/export -hosts; the namespace does not use it. +layer disappears too. XYBF remains in `python/xy/_framing.py` (re-exported +from `xy.channel`) for HTTP/export hosts; the namespace does not use it. ### The cost we accept (recorded, §28 spirit) @@ -114,6 +115,9 @@ aligned, zero-copy into `Float32Array`s. No JSON numbers for data, no base64, no custom framing (§29 preserved; the socket.io protocol already length-prefixes attachments). +The envelope is below; the `m` payload it carries is specified field by field +in `spec/design/wire-protocol.md`. + ``` client -> server (namespace /_xy) sub {fig, px?, mid} subscribe; join figure room; reply `payload` @@ -136,6 +140,30 @@ Inbound handlers are total: malformed input drops or answers `err`, never raises — `channel.py`'s "hostile client must not crash the kernel" contract extended to the transport. +### 2.1 Message catalog (specified in wire-protocol.md) + +The envelope above is transport; the `m` object it carries is the kernel +protocol, dispatched by `xy.channel.handle_message`. Every request type, every +reply shape, and the `seq` / `_pickSeq` / `drill_seq` staleness rules are +specified in [wire-protocol.md](wire-protocol.md), which is the sole authority +for all of it. Unknown types and malformed fields return no reply at all (§2's +totality contract). This section records only what is specific to this host. + +**`view_change` does not reach the kernel here.** The wrapper intercepts the +outgoing message and invokes the Reflex `on_view_change` prop directly +(`python/reflex-xy/reflex_xy/assets/XYChart.jsx:164-169`), because the +namespace registers no Python-side view callback (§5). Every other request +type crosses the socket unchanged and is dispatched by the shared +`handle_message`. + +**Client-supplied dimensions are untrusted.** `px`, `w`, and `h` pass through +`lod.screen_shape`, which rejects non-finite values and clamps the rest to +`[16, MAX_SCREEN_DIM]` (`MAX_SCREEN_DIM = 4096`, `xy/config.py`) — a hostile +client cannot make the kernel allocate an arbitrary density texture. The bound +matters more here than in the notebook: the namespace is reachable by anyone +who can reach the app, so the clamp is an access-control boundary rather than +a sanity check (§3.3). + ## 3. Figures: registry as cache, state as truth ### 3.1 The figure var (the pattern that sidesteps the distributed problem) @@ -170,6 +198,13 @@ subclass points dependency analysis at it), so: - Reconnect (same node or another): the cached token comes back with the state; the component re-`sub`s; hit → serve, miss → §3.2. +Two values are not tokens. Before session hydration there is no client +token to mint from, so the var evaluates to `""`; and a builder may return +`None` for "no chart right now", which **releases** any existing registry +entry and likewise yields `""`. The wrapper treats `""` as "not ready / no +chart" and mounts nothing, so both cases are a blank mount rather than an +error. + Async builders are first-class, mirroring reflex's own `ComputedVar`/`AsyncComputedVar` split with the same `iscoroutinefunction` dispatch `rx.var` uses: an `async def` builder becomes @@ -177,6 +212,14 @@ an `AsyncFigureVar` (an `AsyncComputedVar`), evaluated and cached by reflex's normal async-var machinery, and the rebuild path awaits the same builder when a fresh worker recovers the figure. +Figure vars must be **public**: a leading-underscore builder name is refused +at decoration time with `ValueError`, because backend (underscore) vars never +sync to the client and so their tokens could never reach the wrapper — +failing at import beats compiling a chart nobody can subscribe to. +`@reflex_xy.figure(...)` forwards arbitrary computed-var keywords (`deps=`, +`auto_deps=`, `interval=`, …) straight through to the underlying var, with +`cache=True` set as the default. + Builders must be pure functions of their state instance — the discipline cached computed vars already impose — because purity is exactly what makes the figure a *rebuildable cache* instead of precious process state (for @@ -223,8 +266,9 @@ whether the kernel still matters: **Static payload tier — pass the Chart straight to the component.** `reflex_xy.chart(xy.scatter_chart(...))` compiles the figure to its first-paint payload at page build, writes it into the app's `assets/xy/` as -one content-addressed XYBF frame (`.xyf` — the §3.2 framing's -natural home), and hands the wrapper a `src` URL instead of a token. The +one content-addressed XYBF frame (`.xyf` — the `_framing.py` +envelope's natural home), and hands the wrapper a `src` URL instead of a +token. The wrapper fetches the static file and runs the render client **kernel-less**: the exact `renderStandalone` semantics of `Figure.to_html()` exports — client-side hover from retained columns, pan/zoom, worker-based density @@ -302,6 +346,14 @@ which the wrapper fetches and renders kernel-less. Semantic-event props apply to live sources; a static chart resolves hover tooltips client-side but dispatches no backend events. +Sizing is the mount's, not the payload's. `chart()` defaults the outer +element to `width: 100%` / `height: 420px` (override with any style prop), +and the wrapper rewrites the payload spec's own `width`/`height` to `100%` +on both the static and live paths, so a chart always follows the box Reflex +reserved instead of the dimensions baked into its payload. Charts built with +`width="100%"` therefore track the element responsively, and a fixed-size +payload cannot paint outside the page flow. + `chart()` is a plain `rx.Component` whose `library` is a **local JSX shared asset** (`$/public/external/reflex_xy/assets/XYChart.jsx`, the same mechanism reflex's own radix color-mode provider uses) — no npm package, no @@ -358,7 +410,7 @@ python/reflex-xy/ reflex_xy/assets/ XYChart.jsx; links xy's installed render client examples/demo_app/ 1M-point drilldown + hover + cross-filter + stream + a direct-Chart static payload -tests/reflex_adapter/ 65 tests: token/registry/var/bridge/payload-asset +tests/reflex_adapter/ 69 tests: token/registry/var/bridge/payload-asset units, component compile, and a real-websocket integration suite (uvicorn + socketio client) covering payload/pick/select/affinity/rebuild/ @@ -377,9 +429,11 @@ supported surface exists. The previous revision of this document specified `GET /_xy/{token}/payload`, `POST /_xy/{token}/msg`, an SSE `/events` invalidation stream, and the XYBF -binary frame (§3.2). What survives: `handle_message` extraction (shipped as -`xy.channel`), the XYBF frame helpers (still in `xy.channel` for -HTTP/export hosts), the registry API shape, and the two-planes analysis. What +binary frame (`python/xy/_framing.py`). What survives: `handle_message` +extraction (shipped as +`xy.channel`), the XYBF frame helpers (still in `python/xy/_framing.py`, +re-exported from `xy.channel`, for HTTP/export hosts), the registry API +shape, and the two-planes analysis. What changed: transport (§1–§2) and the multi-worker story — the old draft called the registry's process-locality "the honest hard problem" and sketched a figure-server; the figure-var + rebuild-from-state design (§3) dissolves it @@ -391,9 +445,12 @@ message protocol is transport-agnostic either way. - **Payload push sizing**: room-wide refreshes use the figure's default `px_width`; per-sid re-fit to each viewport is a straightforward follow-up. -- **Static tier px baseline**: payload assets build at the fluid default - (2048 px) like `to_html()` exports; decimated line tiers cannot re-refine - without a kernel, so extreme upscaling shows the export tier's limits. +- **Static tier px baseline**: payload assets build at the figure's own + resolved px width, same as `to_html()` exports — a chart declared with a + numeric `width=` builds at that width, and a responsive chart + (`width="100%"`) falls back to the 2048 px fluid default until a browser + reports a real width. With no kernel, decimated line tiers cannot re-refine, + so extreme upscaling shows the export tier's limits. Orphaned `assets/xy/*.xyf` digests accumulate under changing data until manually cleared; a compile-time sweep of unreferenced digests is a possible follow-up. diff --git a/docs/engineering/design/reflex-shaped-api.md b/spec/design/reflex-shaped-api.md similarity index 95% rename from docs/engineering/design/reflex-shaped-api.md rename to spec/design/reflex-shaped-api.md index 5ddafd9f..d3bd2db4 100644 --- a/docs/engineering/design/reflex-shaped-api.md +++ b/spec/design/reflex-shaped-api.md @@ -151,6 +151,7 @@ Target node types: | `legend(...)` | Legend chrome | Exists | | `tooltip(...)` | Tooltip chrome/customization | Exists | | `modebar(...)` | Zoom/pan/reset controls | Exists | +| `colorbar(...)` | Continuous-scale chrome | Exists | | `theme(...)` | CSS variable defaults | Exists | The implementation can stay dataclass-like. It does not need React-style @@ -244,7 +245,12 @@ Recommended token surface: | `--chart-tooltip-text` | Tooltip text | | `--chart-legend-bg` | Legend panel | | `--chart-selection` | Box select fill/stroke | -| `--chart-accent` | Default accent color for marks/modebar | +| `--chart-modebar-bg` | Modebar and modebar-menu background | +| `--chart-modebar-active` | Modebar active/hovered button background | + +`--chart-accent` is not an XY token; the renderer never reads it. It is a +caller-defined convention variable that XY only passes through when a mark +color is written as `var(--chart-accent)`. Example: @@ -357,8 +363,9 @@ xy.scatter(x="x", y="y", data=df, color="var(--chart-accent)") This keeps Tailwind useful without pretending CSS can reach GPU buffers. -The Reflex adapter mirrors class strings from a fixed `xy.Chart`/`xy.Figure` -into generated JSX so Tailwind can discover them at compile time. A live token +The Reflex adapter mirrors class strings from a fixed `xy.Chart` (or an +internal `Figure`) into generated JSX so Tailwind can discover them at compile +time; the inventory comes from `Figure.dom_class_strings()`. A live token or Var figure does not exist until runtime; its complete utility names must also appear literally in ordinary application source or in the host safelist. @@ -426,7 +433,7 @@ Events should be semantic, small, and transport-independent. ```python def hover(row: dict): ... -def selected(selection: dict): ... +def selected(selection: xy.Selection): ... def view_changed(view: dict): ... xy.chart( @@ -442,8 +449,8 @@ Event payloads: | Event | Payload | |---|---| | `on_hover` | One row/readout dict | -| `on_select` | Trace ids, row indices/counts, optional aggregate summary | -| `on_view_change` | x/y ranges, pixel shape, sequence id | +| `on_select` | An `xy.Selection`: `.per_trace` (trace id to index array), `.index` (concatenated indices), `len()`, and `.xy(trace_id)` for the selected `(x, y)` arrays | +| `on_view_change` | `{x0, x1, y0, y1, source}` — the view edges in data space plus the gesture label (`channel.py:216-233`). No pixel shape, no sequence id. | Render target behavior: @@ -695,10 +702,11 @@ Now part of the core alpha contract: - `xy.colorbar(...)` with inferred built-in continuous-scale chrome, the same CSS slots, and an opaque adapter-render replacement contract. - `class_name`, `class_names`, and `style` props. +- DOM `CustomEvent`s for standalone host integration (see Phase 4 for the + dispatched set). Can add: -- DOM `CustomEvent`s for standalone host integration. - A separate adapter package with optional/minimal Reflex integration. Should avoid: @@ -737,8 +745,10 @@ Should avoid: ### Phase 4: Events And Standalone Host Hooks - Add `on_view_change`. -- For standalone HTML, dispatch optional DOM `CustomEvent`s: - `xy:hover`, `xy:select`, `xy:view`. +- For standalone HTML, dispatch DOM `CustomEvent`s. Shipped: `xy:hover`, + `xy:select`, `xy:view_change`, `xy:click`, `xy:brush`, `xy:leave`, + `xy:context_lost`, `xy:context_restored`, `xy:context_restore_failed`. + All are emitted through one template as `xy:${name}`, bubbling and composed. - Keep Python callbacks notebook-only. ### Phase 5: External Reflex Adapter diff --git a/spec/design/renderer-architecture.md b/spec/design/renderer-architecture.md new file mode 100644 index 00000000..a11145ed --- /dev/null +++ b/spec/design/renderer-architecture.md @@ -0,0 +1,405 @@ +# Renderer Architecture — Audit & Target Design + +**Status:** audit of the shipped WebGL2 render path + the architecture it +should converge to as chart kinds multiply. Findings below are verified +against the source, not hypothetical. + +**Scope.** The client builds from 14 parts, concatenated in filename order into +one bundle by `js/build.mjs`. The audit findings in §2 cover the render path — +`40_gl.js`, `45_lod.js`, `50_chartview.js`, `55_marks.js`, `60_entries.js`. +The module inventory below covers all 14. Ticks and axis label formatting +(`30_ticks.js`) are specified in §6; the gesture contract in `53_interaction.js` +is specified in [interaction.md](../api/interaction.md) §5, and the client half of +the kernel channel (`54_kernel.js`) in [wire-protocol.md](wire-protocol.md). +What remains normatively unspecified is the ARIA/DOM accessibility contract — +see R11. + +### Module inventory + +Line counts are as of this revision and will drift; they are recorded to show +relative mass, not as a budget (see §3 on why a line count failed as a metric). + +| Module | Lines | Responsibility | +| --- | ---: | --- | +| `00_header.js` | 172 | Bundle preamble: the client-wide design comment, the `PROTOCOL` version constant, and the binary frame codec (`XY_FRAME_MAGIC` `"XYBF"`, `decodeFrame`, 8-byte alignment). Every inbound frame is validated here — magic/version, u64 fields, zero-padding, and per-field size limits — before any other module sees it. | +| `10_colormaps.js` | 51 | The `COLORMAP_STOPS` table (§36 CVD-safe defaults) as compact RGB stop lists, and `buildLutData`, which linearly interpolates a stop list into the 256-texel RGBA LUT uploaded once per colormap as a texture. | +| `20_theme.js` | 163 | Resolves chrome and mark colors: arbitrary CSS color expressions and `--chart-*` custom properties are resolved against a live probe element into f32 RGBA for GL, with a fallback on unparseable input. Also owns `XY_CHROME_CSS` and its one-time stylesheet injection. | +| `30_ticks.js` | 224 | CPU-side tick generation in f64 for linear, log, category and time axes, plus every axis/colorbar label formatter (automatic and `format=`-driven). Specified in §6. | +| `40_gl.js` | 829 | WebGL2 primitives: shader compile/link, `makeProgram` with its per-program uniform-location memo (R1), the fixed `ATTR_SLOTS` attribute-slot table bound at link time, and the shader inventory itself. The only module that is GPU-API-specific by design (§4). | +| `45_lod.js` | 567 | View-dependent level-of-detail orchestration, deliberately chart-agnostic: tier selection, drill enter/exit hysteresis (`LOD_DRILL_EXIT_FACTOR`), cross-tier fades, and the retained tier caches. Calls back into `view._draw*` rather than drawing itself, which is the seam tests intercept. | +| `46_worker.js` | 59 | The standalone density re-bin worker: a worker source string carried inside the bundle and booted from a Blob URL. Re-bins the retained sample off the main thread so kernel-less (`to_html`) density charts refine on zoom instead of stretching the overview texture; absence of workers falls back to stretching. | +| `50_chartview.js` | 4167 | The `ChartView` class: the four drawing surfaces, scale/view state, chrome (background, grid, axes, legend, colorbar), GL buffer and VAO management (R2), and pick orchestration. Modules 51–54 extend this same class. | +| `51_annotations.js` | 591 | The 2D overlay canvas above the marks canvas: annotation markers, arrows, shape fills, and collision-nudged labels. Separates canvas shape style keys from label CSS so annotation styling never leaks into the DOM label. | +| `52_tooltip.js` | 260 | Hit → source row → tooltip DOM. Renders the local f32-decoded row immediately, then replaces it with the kernel's exact f64 row when that reply arrives (sequence- and `drill_seq`-guarded); composes text nodes, never HTML. | +| `53_interaction.js` | 1803 | The entire user-facing interaction surface: pointer/drag/wheel wiring, crosshair, box select, box zoom, lasso, the modebar and its export menu, and the animated pan/zoom view state machine. The gesture→action mapping, the modebar tool inventory and the `interaction_config` switches are specified in [interaction.md](../api/interaction.md) §2 and §5. | +| `54_kernel.js` | 408 | The client half of the kernel channel: debounced density/decimated view-requests, streaming `append` handling, the inbound message dispatcher, and the deep-zoom drill lifecycle (§16). Degrades to `46_worker.js` when there is no comm. The message catalog it consumes is specified in [wire-protocol.md](wire-protocol.md). | +| `55_marks.js` | 220 | The `MARK_KINDS` registry — `build`/`draw` per kind plus the `pointPick`/`retainCpu`/`refreshColor` capability flags — mirroring the kernel's `_emit_` dispatch so adding a 2D chart is an entry here, not a branch in `ChartView`. | +| `60_entries.js` | 76 | Mount/unmount entry points for both hosts (`render` for anywidget, `renderStandalone` for exported HTML) and `payloadBuffers`, which materializes first-paint columns in whichever layout the spec declares. Keeps aligned views zero-copy; a spec/transport disagreement throws. | + +## 1. What's structurally right (keep) + +- **Mark registry** (`MARK_KINDS` + capabilities `pointPick/retainCpu/ + refreshColor` + `markOf` fallback): the render loop is kind-blind; new + kinds are entries. This is the load-bearing decision — preserve it. +- **LOD module** (`45_lod.js`): tier orchestration/fades/caches are outside + ChartView and call back through `view._draw*`, so tests intercept the + renderer and future kinds can swap mark drawing. Keep the callback seam. +- **Four-surface layering**, bottom to top: 2D chrome canvas (background, + grid, axes) → GL canvas (marks) → 2D overlay canvas (annotation shapes: + markers, arrows, shape fills, nudged labels) → DOM (labels/tooltip/ + legend/modebar/crosshair). The overlay sits *above* the marks canvas by + design: the exporters emit annotation marks after every data trace, and a + dense opaque mark (heatmap) would otherwise bury them. Crisp text, + selectable tooltips, zero GL cost for chrome — correct division (§7). +- **Uniform-only pan/zoom**: geometry is static offset-encoded f32; view + changes touch two vec2 uniforms per mark (`_map`). This is why interaction + is cheap; nothing below may regress it. +- **Bounded inputs in the aggregated tiers**: decimated ships M4 output + bounded by the plot's pixel width; density ships a fixed grid. Direct-tier + traces still ship O(N) columns — the bound there is the tier threshold, not + the screen: `DECIMATION_THRESHOLD` = 10_000 for lines/areas, + `SCATTER_DENSITY_THRESHOLD` = 200_000 for scatter, raised to + `DIRECT_SOFT_CEILING` = 2_000_000 when a per-point color or size channel is + present (`python/xy/config.py`). Dossier §31 states the same claim + mode-scoped: 12–24 bytes/point in direct modes, screen-bounded *resident* + memory in the aggregated ones. +- **Two first-paint wire layouts**, and the renderer branches on the spec, not + on the transport. *Packed*: one blob, every column carrying a global + `byte_offset`. *Split*: `buffer_layout: "split"` in the spec and one wire + buffer per column, each column carrying an integer `buf` index into the + buffer list — this skips the join copy, the largest allocation of a + direct-tier build. Split is what both shipping hosts use + (`build_payload_split` in `python/xy/widget.py` and + `python/reflex-xy/reflex_xy/namespace.py`); packed remains for standalone + export and for streaming `append`, which always re-ships packed + (`python/xy/interaction.py`). `payloadBuffers` (`60_entries.js`) and + `_columnView` (`50_chartview.js`) treat any spec/transport disagreement as a + thrown error — fail loud, never a silent fallback. +- **DPR correctness**: backing stores sized `css × devicePixelRatio` + everywhere including the pick FBO and its lazy resize-realloc; line widths + and point sizes multiply by dpr. Audited clean. (Gap: DPR *changes* — see + R7.) + +## 2. Audit findings — must-fix-before-20-charts list + +Ordered by how much each compounds as kinds multiply. + +- **R1 — Per-draw uniform lookups.** ✅ **Done.** `makeProgram` attaches a + per-program memo and all draw sites go through `uniformOf()`; each name + hits the driver once at first use. Verified by the pixel-determinism + smoke probe (bitwise-identical frames through the cached path). +- **R2 — No VAOs.** ✅ **Done.** `_bindVao(g, key, parts, setup)` keeps one + VAO per (trace × draw-config). The config signature is `parts.join("|")` — + buffer identity tags (`buf._fcId`, bumped on every `_upload`) plus the + on/off state of the optional channels — so a replaced buffer (data update, + drill swap) rebuilds its VAO rather than aliasing a stale one. Attribute + slots are fixed at link time by `ATTR_SLOTS` + `bindAttribLocation`, so one + VAO is valid for every program that draws the trace; draw and pick share + it. This removed the per-frame `getAttribLocation`/enable/pointer/divisor + churn *and* the leftover-attrib disable loops, including their per-frame + `getParameter(MAX_VERTEX_ATTRIBS)` driver round-trip. `_deleteVaos` frees + them on teardown; the grid quad holds its own VAO. Landed ahead of the R9 + rect-pick trigger it was originally sequenced behind. Note the VAO + utilities live in `50_chartview.js`, next to the GL context they mutate, + not in `40_gl.js` as §3 originally projected. +- **R3 — Draw-state discipline.** Re-audit: the discipline is currently + sound — BLEND is enabled once at init and toggled in exactly one place + (the pick pass, correctly paired), and every texture bind sets its unit + explicitly. A `withState` helper today would be ~40 lines guarding one + pair — over-engineering by this codebase's own standard. **Deferred:** + adopt when a second state-toggling pass lands (e.g. scissored panels or + an additive-blend mark). +- **R4 — No GL context-loss handling.** ✅ **Done.** + `_initContextLossRecovery` listens for loss, prevents default eviction + handling, quiesces draw/animation/re-bin work, and increments the request + sequence so pre-loss kernel/worker replies cannot mutate the rebuilt state. + Streaming appends still replace the retained canonical payload while the + context is down. Restore drops dead handles, re-runs `_initGl` from that + payload, and re-fires the view request to re-sync live tiers; a failed + restore remains explicitly failed instead of throwing from the event + handler. The dependency-free smoke forces three `WEBGL_lose_context` + cycles, checks pixel-identical frames after each, and zooms after recovery. +- **R5 — Shader source conventions are informal.** ✅ **Done.** `build.mjs` + lints every shader at build time: `#version 300 es` first line, every FS + declares `precision highp float;`, every VS references a `u_*map` uniform + (quad shaders exempted by name), uniforms `u_`-prefixed, attributes + `a`-prefixed. Violations fail the build (negative-tested). +- **R6 — Instancing is per-mark bespoke.** Line uses 4-corner strip + + divisor-1 endpoints; rects likewise; points use POINTS. `MARK_KINDS` now + registers 18 kind names over 9 mark objects (`RECT_MARK`, `BAR_MARK`, + `SEGMENT_MARK`, `MESH_MARK`, `AREA_MARK`, plus inline hexbin, heatmap, + scatter and line), so kind growth has so far been absorbed by *reusing* + mark objects rather than by adding shaders: the corner-expansion preamble + is duplicated across `RECT_VS` and `BAR_VS`, not once per kind. The + duplication the finding predicted is therefore real but small, and + `GLSL_COMMON` does not exist yet. *Fix, still pending:* extract shared VS + chunks (corner tables, px↔clip helpers) via string includes in `40_gl.js` + (`GLSL_COMMON`), not a shader framework. Deliberately stop there — a + "shader graph" is over-engineering at this scale. The original "fine at 5 + kinds" deferral rationale has expired; the trigger is now a third + corner-expanding VS. +- **R7 — DPR/zoom *changes* aren't observed.** ✅ **Done.** `_armDprWatch` + re-arms a one-shot `matchMedia('(resolution: Ndppx)')` per dpr value; + `_resize` re-reads devicePixelRatio so a pure-DPR change re-derives + backing stores with no container resize. Smoke `dprw` flag covers it. +- **R8 — Lifecycle cleanup.** ✅ **Already complete** (re-audit): `destroy()` + → `_destroyGlResources` frees per-trace static + drill buffers, density/ + heatmap/LUT textures (dedup via `texSeen`), pick FBO/texture, the quad, and + all programs. The original finding is stale. Remaining nicety only: move + per-kind buffer-name lists into a registry `dispose` hook when a kind + arrives whose resources don't fit the shared name list. +- **R9 — Picking model won't stretch as-is.** GPU ID pass is point-only + (`pointPick`); rect-family picking (bars/candles) is planned as a + registry `pick` step (contract). The ID encoding is a single *global* + 32-bit integer (`u_pick_base + gl_VertexID`) packed across all four RGBA8 + channels — alpha carries id bits 24–31, it is not a per-trace slot. Trace + ranges are `[pickBase, pickBase + n)`, bases start at 1 so the all-zero + clear stays the background sentinel, and the hit is resolved by range + containment. The limit is therefore cumulative *pickable vertices*, not + trace count; the overflow guard already exists (`base + pg.n > 0x7fffffff` + marks the trace unpickable rather than aliasing a stale range) and must be + preserved as rect-family traces start consuming id space. One addition + when rect picking lands: shared pick-VS chunks per R6, so each mark's ID + pass reuses its draw geometry. +- **R10 — Tooltip data path**: solid (local approx row → kernel-exact + replacement, seq-guarded, drill_seq-versioned, XSS-safe text nodes). Only + gap: the row schema is implicit per kind. Write it down in the chart-kind + contract (`{x, y?, ohlc?, color_*?}` + `_dist` for non-point hover) so new + kinds emit compatible rows — doc task, no code. +- **R11 — Client parts are unspecified.** *Mostly closed.* Every module now + has a one-paragraph contract in the inventory above; `30_ticks.js` has a + full specification in §6; and the two largest remaining gaps have since been + written down. `53_interaction.js` is the second-largest module in the bundle + and defines the entire user-facing gesture and chrome-interaction contract — + drag pan, wheel zoom, box select and box zoom, lasso, double-click reset, + the modebar and its export menu, tooltip show/hide, and keyboard point + navigation — and [interaction.md](../api/interaction.md) now specifies it: §5 is + the normative gesture→action table (`shift` is the only modifier the + renderer reads anywhere in `js/src/`) with the per-gesture required + switches, §5 also carries the modebar tool inventory, §2 is the + `interaction_config` switch table with defaults, §3 the event payloads, and + §7 the unconditional behaviors. `54_kernel.js` (the client half of the + channel protocol) is specified in [wire-protocol.md](wire-protocol.md), as + are `00_header.js`'s `PROTOCOL` constant and XYBF envelope (§7 there); + `20_theme.js`'s token resolution is specified in [export.md](../api/export.md) + §6, and `46_worker.js`'s standalone re-bin role in design-dossier.md §8. + Only `51_annotations.js`, `52_tooltip.js` and `10_colormaps.js` still have + no spec reference beyond the inventory above. **Remaining doc task:** the + ARIA/DOM accessibility contract. interaction.md states *that* keyboard + traversal and a live-region readout exist and are unconditional. What no + document states *normatively* is the DOM that implements them: the + `role="region"` root with its `aria-label`/`aria-describedby` summary, the + `role="status"` `aria-live="polite"` readout, the `role="img"` focusable + canvas (`50_chartview.js:1110-1147`, `:1212`), and the modebar's + toolbar/menu roles and roving `tabIndex` (`53_interaction.js:718-1000`). + Those citations are accurate but descriptive — dossier §20 sketches the + intent, production-readiness.md tracks the status, and this paragraph + inventories the attributes. None of the three binds an implementer, so a + refactor could change any of them without contradicting a spec. That is the + open item. No code change — the accessibility contract is still only + readable as source. + +## 3. Target architecture (converged form) + +``` +60_entries mount/unmount, comm plumbing (unchanged) +54_kernel view-requests, streaming appends, + inbound messages, drill lifecycle (extracted from 50) +53_interaction pointer/drag/wheel, crosshair, box + + lasso select, modebar, view state (extracted from 50) +52_tooltip hit → source row → tooltip DOM (extracted from 50) +51_annotations overlay canvas: markers, arrows, + shape fills, nudged labels (extracted from 50) +50_chartview ChartView = surfaces, scales/view + state, chrome, pick orchestration, + GL buffer + VAO management (still growing — see below) +55_marks MARK_KINDS: build/draw/pick/hover/ + dispose/refreshColor per kind (grows, stays declarative) +45_lod tier orchestration (kind-blind) (unchanged) +40_gl programs, shader inventory, + ATTR_SLOTS, uniform location cache (gained R1/R5; R3/R6 pending) +``` + +51–54 all extend the same class via `Object.assign(ChartView.prototype, …)`, +so `this.*` is unchanged across the split — the file boundary is an editing +seam, not a module boundary. Two projections in the original table did not +hold: interaction is no longer a `50_chartview` responsibility, and the VAO +utilities landed in `50_chartview.js` (with the GL context) rather than in +`40_gl.js`. `GLSL_COMMON` (R6) is still unwritten. + +ChartView's steady-state responsibilities: own the GL context + view rect + +event wiring + chrome; delegate everything mark-shaped to the registry and +everything tier-shaped to lod. It should *lose* lines as kinds are added, not +gain them. + +**The line-count form of that metric has failed and is retired.** +`50_chartview.js` was 1707 lines when this document was written and is 4167 +today — 2.4× the original figure — and that is *after* the annotations, +tooltip, interaction and kernel extractions moved a further 3062 lines into +51–54. The whole file is one `class ChartView`, so there is no small-class +escape hatch: the ~1500-line ceiling was breached and the number was never +refreshed, which is exactly how a raw line count fails as a metric. + +Replacement metric, which does not go stale on every edit: **no per-kind +branches in `ChartView`.** Any `if (kind === …)` or kind-name switch outside +`55_marks.js` is the regression to catch; growth in shared machinery +(buffers, VAOs, pick, chrome) is expected and fine. Track the count of +kind-name literals in `50_chartview.js`, not its length. + +## 4. WebGPU migration path + +Position: **WebGL2 remains the shipping target** (universal, and our loads +are instancing-friendly); WebGPU is an *additive backend*, attractive for +Tier-2 compute (atomic-add binning in a compute shader, dossier §5) and +storage-buffer picking. The architecture above is deliberately the portable +subset: + +1. Everything GPU-API-specific already lives in `40_gl.js` + the `_draw*`/ + `_upload*` methods; marks talk to buffers/uniform-maps, not raw GL, once + R1-R3 land. That interface (`createProgram/uploadBuffer/draw(markState)`) + is implementable on WebGPU nearly 1:1 (uniform maps → bind groups, + VAOs → vertex-buffer layouts). +2. GLSL→WGSL is mechanical for our shader inventory (no derivatives beyond + fwidth — supported; no geometry tricks). Keep shaders tiny and + convention-bound (R5) so a one-time port stays a week, not a rewrite. +3. Migration trigger, stated so we don't drift: adopt WebGPU **only when** + (a) a measured Tier-2/3 client-side re-bin path needs compute, or + (b) WebGL2 fill-rate demonstrably caps a real workload the pyramid can't + absorb. Not before — two backends is a tax. + +## 5. Sequencing + +1. ✅ R1 (`makeProgram` location cache) and R2 (VAO helper) landed as pure + refactors, byte-identical smoke as expected. R3's state helper is still + deferred on its own trigger (a second state-toggling pass). +2. R4 context-loss + R8 dispose (one PR: lifecycle) + smoke probes + (lose/restore renders again; destroy leaves zero live GL objects via + `WEBGL_debug`-style accounting where available). +3. R5 build-time shader lint + R6 GLSL_COMMON extraction (with the next new + rect-family kind, not before). +4. R7 dpr watch (fold into `_resize`). +5. R9 pick generalization lands with the first pickable rect mark, per + contract trigger. +6. R11 interaction/module contracts — done (interaction.md, wire-protocol.md); + the ARIA/DOM accessibility contract is the residue, doc-only, no trigger. + +## 6. Axis ticks and label formatting (`30_ticks.js`) + +Ticks are computed on the CPU in f64 and never round-trip through the f32 +render path (§16). `ChartView._ticksFor` dispatches on the axis +(`50_chartview.js:395–398`), checking in this order: `kind === "time"` → +`timeTicks`, `kind === "category"` → `categoryTicks`, `scale === "log"` → +`logTicks`, otherwise `linearTicks`. Every generator takes `(lo, hi, target)` +with `target = 6` by default and returns `{ ticks, step }`; `logTicks` adds +`labels` and `log: true`. + +### 6.1 Generators + +**Linear.** `niceStep` takes the rough step `(hi − lo) / target`, drops to the +decade below it (`10^floor(log10(rough))`), and scales that decade by the first +of `1, 2, 2.5, 5, 10` that reaches the rough step (compared with a `1 + 1e-12` +slack so a step landing exactly on a candidate is not pushed to the next one). +`linearTicks` then emits multiples of that step from `ceil(lo / step) * step` +upward, with a `step * 1e-9` tolerance on the upper bound so the last tick is +not lost to float error, and snaps near-zero values to exactly `0` under the +same tolerance. Degenerate inputs return early: non-finite bounds give no +ticks, `lo === hi` gives a single tick. + +**Log.** `logTicks` requires strictly positive bounds — a domain touching zero +or negative values yields no ticks at all, it does not fall back to linear. It +walks integer decades from `floor(log10(lo))` to `ceil(log10(hi))`. When the +decade span is at most `max(2, target)` each decade emits mantissas `1, 2, 5`; +beyond that only `1`. Ticks outside the domain are dropped (again with a +relative `1e-12` slack). Labels are thinned separately from ticks: only +mantissa-1 ticks are labelled, and only every `labelEvery` decade where +`labelEvery = ceil((decades + 1) / target)` — so minor ticks draw unlabelled. +If thinning produces nothing, every tick is labelled. + +**Category.** Positions are integer category indices. `categoryTicks` clamps +the visible index range to `[0, categories.length − 1]`, then strides it by +`max(1, ceil(visible / target))`, so labels thin out as more categories come +into view rather than overlapping. + +**Time.** Time axes carry epoch milliseconds. `timeTicks` picks the first +member of `TIME_STEPS` — a fixed ladder of 1/2/5/10/20/50/100/200/500 ms, then +second, minute, hour, day multiples up to 14 days — that is at least the rough +step, and emits multiples of it from `ceil(lo / step) * step`. This ladder is +uniform-duration only. Once the rough step exceeds 14 days, `timeTicks` hands +off to `calendarTicks`, which switches to calendar arithmetic: it picks a month +stride from `1, 2, 3, 6, 12, 24, 60, 120` months and emits UTC month starts via +`Date.UTC`, so ticks land on real month and year boundaries rather than drifting +by a fixed 30-day approximation. `calendarTicks` reports `step` back as +`stepM * 30 * MS.d`, an approximation used only to choose a label format. + +**Bounds.** Every generator caps output at 200 ticks (`calendarTicks` at 1000) +so a pathological domain cannot produce an unbounded loop or DOM label count. + +### 6.2 Automatic label formats + +With no `format=` on the axis, labels come from the step: + +- `fmtLinear` switches to one-decimal exponential (with `e+` normalized to `e`) + when `|v| ≥ 1e6` or `0 < |v| < 1e-4`. Otherwise it derives the decimal count + from the tick step — `ceil(−log10(step))`, then increments while the step is + not representable at that precision to within a thousandth of itself — and + caps at 8 decimals. Ticks on one axis therefore share a decimal count. +- `fmtTime` picks the unit from the step: year alone for January on + month-or-coarser steps, otherwise `Mon YYYY`; `Mon DD` at day steps; `HH:MM` + at minute steps; `HH:MM:SS` at second steps; `MM:SS.mmm` below. All fields + are read in UTC. +- `fmtCategory` rounds the position to an index and returns that category, or + the empty string when the index is out of range. +- `fmtGeneral` reproduces Python's `:g` (default 6 significant digits) and is + used for *explicit* colorbar ticks, whose precision is authored and must not + be inferred from an unrelated automatic step. The colorbar itself ticks with + `linearTicks(lo, hi, 8)` (`50_chartview.js:1467`). + +### 6.3 The `format=` mini-language + +`fmtAxis` consults the axis's `format` string before falling back to the +automatic formatter. The two accepted grammars are narrow. + +**Numeric axes** (`fmtNumberSpec`). One optional trailing `%` is stripped +first; the remainder must match exactly + +``` +/^(,)?\.([0-9]+)f?$/ +``` + +That is: an optional thousands-separator comma, a literal `.`, one or more +digits of precision, and an optional trailing `f`. The comma selects +`toLocaleString` with fixed fraction digits; without it the value goes through +`toFixed`. A `%` suffix multiplies by 100 and re-appends `%`. So `.2f`, +`,.0f`, `.1%` and `,.2f%` are the entire accepted surface. There is no +currency prefix, no sign flag, no `e`/`g`/`s` type, no explicit width or fill. + +**Time axes** (`fmtTimeSpec`). A strftime *subset* substituted by +`/%[YmdHMSbB]/g`: `%Y`, `%m`, `%d`, `%H`, `%M`, `%S`, `%b` (short month name), +`%B` (long month name). All fields are UTC — there is no timezone support and +no `%j`, `%p`, `%I`, `%Z` or literal-`%%` escape. Any other text in the string +passes through verbatim, which means an unrecognized token such as `%y` renders +literally as `%y` rather than falling back. + +**Log-axis carve-out.** On a log axis, a value in `(0, 1)` that a numeric +format renders as the string `"0"` is re-rendered with `fmtLinear` instead, so +a low decade is not labelled as a row of zeros. + +### 6.4 Sharp edge: silent fallback on unmatched formats + +`fmtNumberSpec` returns `null` on anything the regex rejects, and `fmtAxis` +treats that as "no format" — it silently uses the automatic formatter. No +warning is raised anywhere on the path: the Python side stores `format` as free +text with no validation (`python/xy/_figure.py:204` via `_optional_text`), so +an unsupported spec survives the whole pipeline and simply does nothing. + +The in-tree example is `tests/test_components.py:555`, which sets +`format="$,.0f"` on a y-axis. The leading `$` cannot match the regex, so the +whole format is discarded and the axis renders with `fmtLinear` — the `$` never +appears. The test does not catch this because it asserts on the emitted spec, +not on rendered labels. + +Two consequences to keep in mind when extending this: the failure mode for a +typo'd numeric format is a *plausible-looking wrong label*, not an error; and +the two grammars fail differently, since an unrecognized `%`-token on a time +axis is echoed literally instead of triggering the fallback. Making either loud +is a behavior change, not a doc fix, and is not proposed here. diff --git a/spec/design/rust-engine.md b/spec/design/rust-engine.md new file mode 100644 index 00000000..e74102c5 --- /dev/null +++ b/spec/design/rust-engine.md @@ -0,0 +1,312 @@ +# Rust Data Engine — Module Boundaries & FFI Protocol + +**Status:** design. Decides what lives in Rust vs Python and how the FFI seam +evolves without rewrites. Grounded in the shipped engine: single-dependency +cdylib (`src/lib.rs`, ABI v36; `png` is the one crate, for static export), +ctypes binding (`_native.py`), and +dispatch in `kernels.py`. The native core is required — `kernels.py` raises a +clear ImportError when it can't load, with no pure-Python fallback. + +## 1. The placement rule + +**Rust owns row-scan loops; Python owns decisions.** Precisely: + +- If work is O(N) over data rows and sits on an interaction path (build, + zoom, pan, drill) → Rust kernel. +- If work is O(traces), O(screen), or policy (tier choice, budget math, spec + assembly, validation, warnings) → Python. Moving policy into Rust buys + nothing (it's microseconds) and costs iteration speed — policy is what + changes weekly. +- The client (JS) owns nothing O(N): it receives screen-bounded buffers only + (§29). This boundary is what keeps the browser safe at 1B rows and it never + moves. + +### Current placement audit + +| Concern | Today | Verdict | +|---|---|---| +| zone maps, encode_f32, m4, bin_2d, min_max, histogram_uniform, normalize_f32, range/validity indices, local_log_density | Rust (ABI v36) | correct — new equal-length x/y columns use a paired zone-map call with bit-identical per-column reductions; full-domain density first paint fuses binning with uniform or counted-u8 overlay sampling while retaining exact standalone outputs; mesh/rectangle validity scans consume only columns not already proven finite by zone metadata | +| fixed-width string/bytes/bool factorization | Rust (ABI v36) | correct — compact palettes use a bounded L1-resident codebook with full-record collision checks and emit exact counts; U1 uses a direct Unicode-scalar table with endian support; ≥512k rows probe a prefix then encode disjoint chunks in parallel, merging late labels by canonical first-row order before any retry; Python sees only unique labels and retains display-label ordering policy | +| static display-list raster, row-banded polyline/point/segment paint, batched fill+stroke triangle meshes, affine scatter projection plus typed color/size resolution, density/heatmap colormap and sampling | Rust (ABI v36) | correct — commands borrow f32/u8 payload or canonical spans synchronously; compact stratified sampling reuses factorization counts; batched/banded output is byte-identical | +| signal processing: `xy_rfft`, `xy_welch_spectra`, `xy_spectrogram` | Rust (ABI v36) | correct — O(N) transforms over sample columns; window/segment policy stays in Python | +| geometry/triangulation: `xy_delaunay_triangles`, `xy_polygon_triangles`, `xy_marching_squares`, `xy_marching_triangles`, `xy_streamlines`, `xy_vector_segments`, `xy_quad_mesh_triangles`, `xy_sector_triangles`, `xy_indexed_triangles`, `xy_triangle_edges` | Rust (ABI v36) | correct — output is screen-bounded index/vertex buffers; level choice and styling stay in Python | +| statistics: `xy_correlation`, `xy_weighted_ecdf`, `xy_histogram2d`, `xy_stacked_bounds` | Rust (ABI v36) | correct — row-scan reductions; binning policy and labels stay in Python | +| style/text helpers: `xy_css_check` (`css.rs`), `xy_svg_poly_path` (`svg.rs`) | Rust (ABI v36) | correct by a different rule — not O(rows) but O(points)/per-value on the export and validation paths, where per-item Python object churn dominates; error *messages* still assembled in Python | +| ohlc_decimate (when finance returns) | was NumPy-in-kernels.py | acceptable stopgap **only** because candles decimate to ≤px buckets; promote to Rust with the pyramid work | +| tier decisions, hysteresis, drill_seq, spec/emitters, channel resolution | Python | correct — keep | +| visible-count mask for drill | NumPy expression in `lod.visible_mask` | promote: it's O(N) per zoom step at 100M — fold into `xy_range_indices` (already exists) so count+indices come from one pass | + +### Target Rust ownership (matches the priority list) + +binning (1D/2D/channel-aware) ✅/plan · decimation (M4 ✅, OHLC plan) · +range filtering (`xy_range_indices` ✅) · implicit uniform and compact-u8 +stratified sampling (`xy_sample_range_indices` / `xy_stratified_sample_range_u8` ✅) · grouping/category encoding +(`xy_factorize_fixed` ✅ for contiguous fixed-width values; defensive Python +label canonicalization for mixed objects) · histogram stats ✅ · quantiles (plan: +`xy_quantiles`, needed by box/violin) · box/violin stats (thin composition +over quantiles — stats in Rust, assembly in Python) · multi-resolution tile +generation (`tiles.rs` ✅, including stable-domain incremental updates) · +Rust-owned streaming column buffers (plan: `stream.rs`, §5 below). + +## 2. Module boundaries (crate layout) + +``` +src/ # 15,423 lines shipped, 8 modules + lib.rs (3073) # C ABI shell ONLY: extern fns, pointer/len marshaling, + # ABI_VERSION, the ffi_guard panic shield (§3.2 E4), + # and the mutex-guarded handle registry backing + # tiles.rs. No math. Every fn: null/len checks → + # slice → call kernels::*/raster::* → write out-params. + kernels.rs (6113) # pure safe Rust row-scan kernels: zone maps/encode/M4, + # binning + sampling, factorization, histogram/min-max/ + # normalize/range+validity indices, density, signal + # (rfft/Welch/spectrogram), geometry (Delaunay, marching + # squares/triangles, streamlines, vector segments), + # correlation/weighted ECDF/stacked bounds. + # No unsafe, no I/O. + raster.rs (3423) # the entire native rasterizer — the whole static PNG + # export path below Python's geometry/scale/colormap + # computation. Consumes a tagged display-list command + # stream (optionally borrowing f32/u8 payload or + # canonical spans) and paints into a straight-alpha + # RGBA8 framebuffer the caller owns: + # · scanline polygon fill, flat + linear gradient, + # with a rectangle fast path + # · SDF/distance-based stroke and point-symbol paint + # (round caps/joins and AA fall out of the metric) + # · affine scatter projection, typed per-point color/ + # size resolution, batched triangle meshes, segments + # · image blit incl. density/heatmap colormap sampling + # · text, blitted and bilinearly scaled from font.rs + # · row-banded multithreaded paint (std::thread::scope, + # byte-identical to the serial path) and the fused + # raster→PNG encode via `png`/fdeflate. + # No unsafe; owns the crate's one third-party dep. + font.rs (1039) # generated by scripts/gen_font.py — do not hand-edit. + # Baked DejaVu Sans grayscale coverage atlas for + # raster.rs: 205 glyph records (advance, w, h, left, + # top, coverage offset/len) at BASE_PX=16 plus the + # row-major coverage bytes, and EXTRA_CODEPOINTS, the + # sorted table of the 110 non-ASCII codepoints. Data + # only — no shaping, no FreeType, no unsafe. Coverage + # limits are a product constraint, not a detail: §2.1. + css.rs (788) # tiered CSS value/color validation behind `xy_css_check`. + svg.rs (58) # screen-space coordinate serialization for the SVG path: + # `poly_path` alone, folding parallel f64 x/y arrays + # into one `M`/`L` path-data string with Python-matching + # 2-decimal fixed-point trimming (including the `-0` + # case). Rejects length-mismatched, empty, or non-finite + # input by returning None. Deliberately narrow: it + # exists to kill one Python string per point, and the + # rest of SVG scene construction stays in Python. + simd.rs (448) # AVX2 twins of eligible kernels, runtime-dispatched (§3.4). + # The one module besides lib.rs allowed `unsafe`. + tiles.rs (481) # pyramid build/compose/incremental append. Owns tile memory; + # handles are opaque u64 ids passed over the ABI (§3.3). + stream.rs # (plan) Rust-owned canonical append buffers. + stats.rs # (plan) quantiles/box/violin/factorize. +``` + +Line counts are `wc -l` at this revision and drift with the code; the ordering +(kernels > raster > lib > font > css > tiles > simd > svg) is the stable fact — +rasterization is the second-largest thing in the crate, and it is not a helper. + +Rules: `lib.rs` is the only file with `unsafe` (except `simd.rs`, §3.4 rule +3); kernel modules are pure +functions over slices (fuzzable, testable without FFI); **dependencies are +minimized, not prohibited** (policy 2026-07-05): a crate may be added when it +pays for itself — measured win, small dependency tree, well-maintained — and +the C-ABI/one-cdylib-per-platform property is preserved. Note the dev sandbox +cannot reach crates.io, so required crates must be vendored or the sandbox +loses local build/test; prefer feature-gated optional deps (e.g. SIMD +argminmax, tsdownsample-class speed) with the lean build as default. + +### 2.1 Native text is a bounded subset, and misses are silent + +Declining FreeType bought the single-cdylib property (§3.1) and paid for it in +Unicode coverage. `font.rs` bakes exactly 205 glyphs: ASCII 32–126 (95) plus +the 110 codepoints enumerated in `font::EXTRA_CODEPOINTS` — lowercase Greek +(α–ω) and the eleven uppercase Greek letters that differ from Latin forms +(Γ Δ Θ Λ Ξ Π Σ Υ Φ Ψ Ω), math operators (`∂ ∇ ∈ − ∓ √ ∝ ∞ ∫ ≈ ≠ ≤ ≥`), the +left and right arrows only, super/subscript digits and a handful of subscript +letters, typographic quotes, en/em dashes, and a few symbols (`° ± × · µ ²³¹ …`). +Nothing else exists: no accented Latin at all, no Cyrillic, no CJK, no Arabic, +no emoji. + +The failure mode is worse than the coverage gap. `glyph_index` +(`src/raster.rs:1175`) returns `None` for an uncovered codepoint, and the paint +loop's `let … else { continue; }` (`src/raster.rs:1236-1238`) drops that +character **before** the advance is applied — so the glyph is deleted, not +substituted, and the following glyphs close up over the hole. There is no tofu +box, no fallback glyph, no warning, and no error. `"Müller"` rasterizes as +`"Mller"`; a fully non-Latin label rasterizes as nothing. The anchoring pass +sums advances through the same `glyph_index` filter (`src/raster.rs:1200-1204`), +so a centered or right-anchored label is positioned on its *shortened* width — +the loss is self-consistent and therefore invisible in the output. + +This violates §28 (no silent decisions) and is recorded as a known defect, not +a design intent. The minimum honest fix is a substitution glyph with a real +advance, so a coverage miss is *visible*; a warning surfaced at the Python +export boundary (where messages belong, §4) is the complete fix. Until then the +documented escape is `engine=xy.Engine.chromium`, and the user-facing statement +of the same limitation lives in `spec/api/styling.md` §"Native text coverage" — +these two must be amended together. The bound applies to the native raster +formats only; the SVG and PDF export paths have their own text contracts. + +## 3. FFI protocol — how it evolves without rewrites + +### 3.1 What's already right (keep as law) + +- **C ABI + ctypes, no PyO3**: no per-CPython builds; consumers install the + compiled wheel without a Rust or crate-registry dependency. +- **Caller-allocated buffers**: Python (NumPy) allocates outputs; Rust writes + into them and returns counts. No cross-language ownership for array data. +- **f64 in, f32 out** for geometry (offset-encoded, §16); u32 for indices. +- **Lockstep `ABI_VERSION`** in `src/lib.rs` + `_native.py`, checked at load, + hard error on mismatch — an old wheel never mis-calls a new lib. +- **The native core is the single implementation.** There is no pure-Python + fallback; every kernel is tested directly against the Rust core, and a + platform that can't load the core gets a clear ImportError, not a degrade. + +### 3.2 Evolution rules (the anti-rewrite discipline) + +- **E1 — additive by default**: new capability = new `xy_*` symbol. Existing + signatures are immutable once shipped in a release; changing one means a + new name (`xy_bin_2d_v2`) + ABI bump, old symbol kept for one minor cycle. +- **E2 — flags-struct for growth-prone kernels**: kernels we *know* will grow + options (tile fetch, channel binning) take a final `*const FcOpts` pointer + to a versioned plain-C struct (`{u32 size; ...}`); size-checked so old + callers pass smaller structs safely. This is how we avoid v2/v3 name churn + for the pyramid API specifically. +- **E3 — no callbacks across the ABI**: Rust never calls back into Python. + Progress/streaming = polling functions. Keeps the seam re-entrant and GIL- + trivial (all kernels release the GIL implicitly since ctypes calls do). +- **E4 — errors are return codes, never panics across FFI.** The panic shield + has landed: `lib.rs` defines `ffi_guard(sentinel, body)` — + `catch_unwind(AssertUnwindSafe(body)).unwrap_or(sentinel)` — and wraps every + entry point that does work in it (63 call sites across 61 of the 62 + `extern "C" fn`s; `xy_abi_version` returns a constant and needs no shield), + so any panic becomes that entry point's error sentinel instead of unwinding + across `extern "C"`. Output buffers may be partially written on that path, + exactly like the existing invalid-argument paths. Still outstanding: + sentinels are per-kernel ad hoc + (`0`, `usize::MAX`, `-1`/`-2`) rather than one documented negative error + enum; unifying them is a work item for the next ABI bump. +- **E5 — threading stays inside**: parallel kernels use `std::thread::scope` + inside the call; the ABI stays synchronous. General row scans cross over at + 512k values and scale to at most 18 workers. Zone maps cross earlier, at two + complete 65,536-row chunks, because chunks are independent and require no + merge; worker count is also capped by actual chunks. CodSpeed stays serial + because its simulator sums thread instructions rather than wall time. + Incremental build = handle + `xy_pyramid_append`, which mutates a live + pyramid in place under the registry lock. A polling entry point + (`xy_pyramid_poll`) for genuinely async builds is not built; if one is ever + needed it follows E3 (poll, never call back). + +### 3.3 Opaque handles (for tiles/streams) + +Arrays-in/arrays-out stops working when Rust must own long-lived state +(pyramid, append buffers). Pattern: + +```c +/* build: nonzero handle, or 0 on invalid arguments */ +uint64_t xy_pyramid_build(const double* x, const double* y, size_t len, + double x0, double x1, double y0, double y1, + uint32_t base_dim); +/* append: 1 applied; 0 on stale/busy handle, bad args, or a point outside + the pyramid's original domain (never partially mutates) */ +int32_t xy_pyramid_append(uint64_t handle, const double* x, + const double* y, size_t len); +/* count over a window: 1 ok, 0 on stale handle/bad args */ +int32_t xy_pyramid_count(uint64_t handle, double lo_x, double hi_x, + double lo_y, double hi_y, double* out_count); +/* compose window into a w×h grid: level used (>=0), -1 stale/bad args, + -2 window outresolves the pyramid (caller re-bins exactly and discloses) */ +int32_t xy_pyramid_compose(uint64_t handle, double lo_x, double hi_x, + double lo_y, double hi_y, + size_t w, size_t h, float* out); +/* free: 1 if it existed, 0 for stale/unknown */ +int32_t xy_pyramid_free(uint64_t handle); +``` + +Note this API took explicit bounds plus `base_dim` rather than the E2 +flags-struct; the option set stayed small enough that a versioned `FcOpts` +would have been ceremony. E2 still governs the channel-binning kernels. + +Handles are indices into a Rust-side registry (mutex-guarded slab), not raw +pointers — a stale/double-freed handle is an error code, not UB. Python wraps +each in an object with `__del__`/weakref finalizer. + +### 3.4 SIMD (contained in `src/simd.rs`) + +The cdylib builds for baseline x86-64 (SSE2), so hot scans never see 256-bit +registers unless we ask. `simd.rs` holds branch-free clones of selected +kernels compiled under `#[target_feature(enable = "avx2")]` (LLVM +autovectorizes them — no hand-written intrinsics unless a loop demonstrably +fails to vectorize) with runtime `is_x86_feature_detected!` dispatch and a +`XY_SIMD=0` kill switch. + +Rules, in priority order: + +1. **Bitwise parity is non-negotiable.** Only order-independent kernels are + eligible: integer counts, exact comparisons, truncation casts, min/max. + Float *accumulation* (zone-map sum/sum_sq) must stay scalar — vector + reassociation changes the result. Parity is enforced by fuzz tests in + `simd.rs` comparing SIMD vs scalar on hostile data. +2. **Only measured wins ship.** Every dispatch is justified by a before/after + number (via the kill switch); a kernel where the two-phase restructure + loses (M4's sequential bucket state machine, histogram's scatter-dominated + loop) is documented at its scalar definition and NOT dispatched. +3. **`unsafe` containment.** This module is the one exception to "unsafe only + in `lib.rs`": the `#[target_feature]` wrappers are unsafe to call, and + every call site sits behind a safe `try_*` fn that checks detection first. + Kernels code never writes `unsafe` — it calls `simd::try_*` and falls back. +4. **aarch64 needs no twin.** NEON is part of the aarch64 baseline, so the + scalar kernels already autovectorize at full width there. + +## 4. What Python keeps forever + +Ingest normalization (pandas/arrow/dtype coercion — ecosystem glue), +`ColumnStore`/zone-map bookkeeping (thin, O(chunks)), all spec emission, +tier/budget policy, channel *resolution* (mode inference, palette, warnings), +validation and error messages (the new bounds/bool hardening lives at this +layer and belongs there), widget/comm transport. This layer is the product's +personality; keeping it in Python is a feature. + +## 5. Streaming append (Phase-0 landed Python-side; Rust `stream.rs` later) + +**Landed (Phase-0, Python-side canonical):** `Column.append` grows an +amortized capacity buffer and extends zone maps incrementally (only chunks +at/after the old length recompute — the splice is bitwise identical to a +from-scratch ingest). `Figure.append(trace_id, x, y, color=, size=)` +validates atomically (line appends must continue the sorted series; +categorical channels and shared columns are rejected for now), frees the +trace's pyramid for lazy rebuild, exits any drill, and returns an `append` +message carrying a complete fresh payload — screen-bounded by construction +(§29), so the wire never needs deltas. The client rebuilds only the traces +named in `affected`, applies the follow policy (refit when at home, slide +when pinned to the live right edge, hold when inspecting history), and +refines tiered traces to its current window through the normal +stale-while-revalidate request path (§17). + +**Still future (`stream.rs`):** Rust-owned chunked append buffers with +zone maps computed on seal, and — the important one — appends marking +intersecting pyramid tiles dirty with lazy per-tile rebuild (bounded: a +stream touching one region rebuilds ~1 tile/level). Phase-0 instead frees +the whole pyramid on append, so a >2M-point stream pays a full pyramid +rebuild on its next far-out view — recorded, not hidden. ABI: +`xy_stream_new/append/seal/free` + the pyramid fetch reading through the +stream handle. + +## 6. Implementation order + +E4's panic shield and the `tiles.rs` pyramid handles (LOD phases 3-4) have +landed; the remainder, in order: + +1. E4 error enum — unify the ad-hoc per-kernel sentinels into one documented + negative enum (with the next ABI bump, cheap insurance). +2. `xy_bin_2d_channels` (LOD doc phase 1) — first FcOpts-style kernel. +3. Fold drill visible-count into `xy_range_indices` (one pass, count+idx). +4. `stats.rs`: `xy_quantiles` (+ box/violin composition) — unblocks rank-8 + box plots with Rust-grade interaction. +5. `stream.rs` append (after Arrow ingest lands). diff --git a/spec/design/wire-protocol.md b/spec/design/wire-protocol.md new file mode 100644 index 00000000..70cfad54 --- /dev/null +++ b/spec/design/wire-protocol.md @@ -0,0 +1,225 @@ +# Wire protocol — client ↔ Python + +Status: **shipped**. This document specifies the message catalog dispatched by +`xy.channel.handle_message` (`python/xy/channel.py`) and consumed by +`js/src/54_kernel.js`, plus the first-paint buffer layouts and the version +handshake. The transport envelopes that carry these messages are separate: +the anywidget comm (`python/xy/widget.py`), the `/_xy` socket.io namespace +([reflex-integration.md](reflex-integration.md) §2), and the `XYBF` binary +frame (`python/xy/_framing.py`, versioned in §7 below). + +The catalog does not vary by transport: where a host sends a given message, it +has the shape specified here, byte for byte. What varies is *which* messages a +host sends — the Reflex wrapper resolves `view_change` in the browser and never +emits it (§2). + +## 1. Dispatch contract + +`handle_message(fig, content, buffers=None, callbacks=ChannelCallbacks())` +returns either `None` or `(message, buffers)`, where `buffers` is a list of +binary attachments the reply's spec entries index into by position. + +- Non-dict `content`, an unknown `type`, a missing required field, or a value + that fails coercion returns `None`. Client-supplied data never raises; + exceptions from *user callbacks* do propagate. +- Replies are return values, not sends. Python-side callbacks therefore fire + before the reply leaves the process. +- The `buffers` argument is accepted and unused: no inbound message carries + binary payloads today. It exists so the length-prefixed framing path can + pass attachments through without a signature break. +- `append` is not a request kind — it is a server push (§4). + +`ChannelCallbacks` carries five optional hooks: `on_hover`, `on_click`, +`on_brush`, `on_select`, `on_view_change`. A host with none still gets every +wire reply. + +## 2. Requests (client → Python) + +Every request is a dict with a `type`. Coordinate fields are JSON numbers in +**data space**, not pixels. + +| `type` | Fields | Reply | +| --- | --- | --- | +| `view` | `x0`, `x1`, `px?`, `seq?` | `tier_update`, or nothing | +| `density_view` | `trace`, `x0`, `x1`, `y0`, `y1`, `w?`, `h?`, `seq?` | `density_update`, or nothing | +| `pick` | `trace`, `index`, `drill_seq?`, `seq?` | `pick_result` | +| `click` | `trace`, `index`, `drill_seq?` | none (`on_click`) | +| `view_change` | `x0`, `x1`, `y0`, `y1`, `source?` | none (`on_view_change`) | +| `select` | `x0`, `x1`, `y0`, `y1` | `selection` | +| `select_polygon` | `points` | `selection` | +| `select_clear` | — | `selection` (empty) | + +**`view`** — sent by `_scheduleViewRequest` when the chart holds any trace at +`tier === "decimated"` and a pan/zoom crossed what the shipped decimation can +serve. `x0`/`x1` are coerced with `float()` and must satisfy `x1 > x0`; `px` +defaults to `2048` and the client passes the rounded plot width. Only `line` +and `area` traces above `DECIMATION_THRESHOLD` are re-decimated; if that +produces no traces, there is no reply at all (silence, not an empty message). + +**`density_view`** — one request per density-tier trace, each naming its +`trace` id. `w` defaults to `512`, `h` to `384`; the client sends the rounded +plot width and height. A trace that is not in density mode yields +`{"traces": []}`, which is dropped rather than sent. + +**`pick`** — `trace` and `index` pass through `_integer_id`. `index` is a +*shipped-vertex* index, translated kernel-side to a canonical row when the +shipped copy dropped non-finite rows. `drill_seq`, when present, is the drill +subset version the client picked against; a mismatch resolves to `row: null` +rather than to a row in a dead index space. + +**`click`** — same fields and same `fig.pick` resolution as `pick`, minus +`seq`; it fires `on_click` and returns nothing. + +**`view_change`** — the four view edges plus a `source` string (default +`"view"`, stringified kernel-side). The client sends it rAF-coalesced and only +while the `view_change` interaction flag is set; the kernel returns +immediately when no `on_view_change` callback is wired. This is the one request +type a host may withhold: on the Reflex host it never reaches the kernel, +because `XYChart.jsx` intercepts the outgoing message and invokes the +`on_view_change` prop directly +(`python/reflex-xy/reflex_xy/assets/XYChart.jsx:164-169`) — that namespace +registers no Python-side view callback. + +**`select`** — box select. Edges are ordered by `lod.normalize_window` with +`require_area=False`, so a flipped or zero-area drag is still well-formed. + +**`select_polygon`** — `points` is a sequence of at least two-element +sequences; each is coerced to `[float, float]` for the `on_brush` payload. + +**`select_clear`** — no fields. Fires `on_select` with an empty `Selection` +and replies with the empty selection message. + +`px`, `w`, and `h` are untrusted: they pass through `lod.screen_shape`, which +rejects non-finite values and clamps to `[16, MAX_SCREEN_DIM]`. + +## 3. Replies (Python → client) + +**`tier_update`** — `{type, seq, traces}` plus one f32 buffer per column ref. +Each entry is `{id, x, y, base?}`, where each column ref is +`{buf, len, offset, scale}`: `buf` indexes the attachment list, and the client +recovers data space as `value * scale + offset`. The `x` offset re-centers on +the requested window midpoint so f32 precision follows the viewport. The +client drops the message unless `msg.seq === this.seq`. + +**`density_update`** — `{type, seq, traces}`. Each entry carries a `mode` that +states which representation this view resolved to: + +- `mode: "density"` — `{id, mode, tier, visible, reduction, binning, density}`. + `density` is `{buf, w, h, max, enc: "log-u8", x_range, y_range}` plus + optional `color` (a constant-channel color) and `sample` (the retained + point-sample overlay). `binning` is `"exact"` or `"pyramid-L"`. +- `mode: "points"` — the deep-zoom drill: + `{id, mode, tier: "direct", visible, reduction: "none", x_range, y_range, + x, y, color, size, density_val, lod_blend, density_colormap, drill_seq, + style}`. `x_range`/`y_range` are the window these points cover; the client + falls back to the density overview the moment the view leaves it. + +The client enforces `msg.seq` only when it is present, and additionally +accepts `msg.trace` and `msg.stale` for pending-request bookkeeping — no +current kernel path emits either field. + +**`pick_result`** — `{type, seq, row}`. `row` is `null` when the index is out +of range or `drill_seq` was stale; the reply ships regardless so the client +clears its hover state. A point row is +`{trace, index, x, y, x_kind, y_kind}` plus `color_value` or `color_category` +and `size_value` when those channels exist. A heatmap row is +`{trace, index, row, col}` plus `color_value` when the cell is finite. Picks +use their own client-side sequence (`_pickSeq`), not the view `seq` — sharing +one counter let a hover invalidate an in-flight `tier_update`. + +**`selection`** — `{type, traces, total}` plus one u32 buffer per trace. Each +entry is `{id, count, buf, drill_seq}`. Masks speak **shipped-vertex +positions** (`fig.to_shipped_indices`), so `count` is the wire mask length, +while `total` sums the canonical index counts; Python callbacks receive a +`Selection` over canonical rows. `on_brush` fires before the masks are +assembled and `on_select` after — that order is the invariant. An empty +`traces` list means "clear", and carries no buffers. + +## 4. Server push + +**`append`** — `{type: "append", affected: [trace_id], spec}` with a single +packed blob attachment. The kernel re-emits a complete fresh payload rather +than a delta, because every tier's payload is screen-bounded by construction. +The client swaps `spec` and the retained payload together, rebuilds only the +GPU traces named in `affected`, then re-requests its current view with +`delay: 0`. The widget also re-syncs its `spec`/`buffers` traits so a +re-rendered output shows the streamed state. + +## 5. First-paint buffer layout: packed vs split + +The first-paint payload is a data-less JSON spec plus encoded columns. The +spec's `columns` table is the addressing scheme, and it comes in two layouts: + +- **Packed** (`build_payload`) — one blob. Column entries carry a global + `byte_offset` into it. `u8` columns are followed by padding to the next + 4-byte boundary so later f32 columns stay aligned. This is the layout used + by static HTML export and by streaming-refresh reopen state. +- **Split** (`build_payload_split`) — one wire buffer per column. The spec + sets `buffer_layout: "split"`, and every column entry carries `buf` (its + index into the buffer list) with `byte_offset: 0`. The alignment padding for + a `u8` column is folded into that column's own buffer, and `len` still + counts only real values, so split is a byte-identical repack of packed. This + is what both live hosts ship at first paint — `FigureWidget` + (`python/xy/widget.py:76`) and the `/_xy` namespace + (`python/reflex-xy/reflex_xy/namespace.py:135`, `:197`) — with no join copy. + +Column entries otherwise carry `len`, an optional `dtype` (`"u8"`; absent +means f32), and, for offset-encoded geometry, `offset`/`scale`/`kind`. + +The client picks the layout from the spec, never from the shape of what +arrived, and **a disagreement is a fatal error, not a fallback**. +`payloadBuffers` throws when the spec says `split` and the transport delivered +one buffer, and equally when a buffer list arrives without split layout. +`_columnView` throws again per column when `Number.isInteger(meta.buf)` +disagrees with `Array.isArray(buffer)`, and rejects non-safe-integer or +negative `byte_offset`/`len`, a column extending past the payload, and a +misaligned start. Aligned spans stay zero-copy; only a legacy view whose +`byteOffset` is not a multiple of 4 pays one view-sized copy. + +## 6. Chunked base64 (standalone export only) + +The comm and socket.io transports carry binary attachments natively and never +base64. Standalone HTML export has no binary channel, so `xy.export` embeds +the packed blob as chunked base64: + +- The blob is sliced into `_B64_CHUNK_BYTES` = 48 MiB pieces. That size is + divisible by 3, so every chunk but the last encodes a multiple of 3 bytes + and its base64 carries no interior `=` padding — each chunk decodes + independently into an exact-length region of one preallocated buffer. +- Each chunk is emitted as its own `` + block. A script element's source is itself a JS string, so folding all + chunks into one block would rebuild the single-string length ceiling the + chunking removes. Base64 text (`[A-Za-z0-9+/=]`) contains no quote, + backslash, newline, or `<`, so it is quoted verbatim and can never close the + element. +- `xyDecodeB64(chunks, total)` allocates `Uint8Array(total)` and prefers + `setFromBase64`, falling back to an `atob` + `charCodeAt` loop into the + preallocated array. + +The reassembled bytes are identical to the source blob, which is what keeps +`spec.columns[i].byte_offset` valid end to end. Export warns above +`EMBED_WARN_BYTES` = 64 MiB, since base64 adds roughly a third. + +## 7. Versioning and compatibility + +Two independent version constants: + +- **Renderer/spec protocol.** `PROTOCOL_VERSION = 3` (`python/xy/config.py`) + rides every first-paint spec as `spec["protocol"]`; the client's + `PROTOCOL = 3` (`js/src/00_header.js`) is checked in the `ChartView` + constructor. A mismatch replaces the chart element with "update the xy + package and restart the kernel" and throws. Requests and replies carry no + version of their own — the handshake happens once, at first paint, before + any request is possible. +- **Transport frame.** `FRAME_MAGIC` `"XYBF"` with `FRAME_VERSION = 1` + versions the binary envelope separately, so the transport and the renderer + can evolve without coupling. + +Compatibility is structural rather than negotiated: the client is a committed +artifact (`python/xy/static/index.js`) shipped inside the same distribution as +the Python that produces payloads, and `reflex_xy` deliberately does not +package its own copy — `assets.register()` symlinks the client out of the +installed `xy` distribution, repairing a stale link if the install moved. The +JS that renders a payload is therefore always the build that shipped with the +Python that produced it. The protocol check exists for the case that survives +this: a browser holding a cached bundle against a restarted kernel. diff --git a/docs/engineering/matplotlib-compat-changelog.md b/spec/matplotlib/compat-changelog.md similarity index 97% rename from docs/engineering/matplotlib-compat-changelog.md rename to spec/matplotlib/compat-changelog.md index f40e2a1e..0a58aa0f 100644 --- a/docs/engineering/matplotlib-compat-changelog.md +++ b/spec/matplotlib/compat-changelog.md @@ -259,6 +259,12 @@ colorbar domains) fully cleared. Out-of-order frames (`left >= right`, `bottom >= top`) raise `ValueError` as in Matplotlib. This clears the two loud PDSH `subplots_adjust` cells (04.08 2×3 labeled grid, 04.10 5×5 zero-gap faces). +- The interactive modebar is now off by default, matching Matplotlib's inline + backend: `rcParams["toolbar"]` defaults to `"none"`, and panels draw the + on-chart controls only when it holds any other value (`"toolbar2"`, + `"toolmanager"`). `figure(toolbar=...)` — also reachable through + `subplots(..., toolbar=...)`, which forwards it to `figure` — overrides + rcParams for one figure. Future entries must identify the Matplotlib release/revision, inventory additions or removals, and any compatibility-level changes. diff --git a/docs/engineering/matplotlib-compat-matrix.md b/spec/matplotlib/compat-matrix.md similarity index 100% rename from docs/engineering/matplotlib-compat-matrix.md rename to spec/matplotlib/compat-matrix.md diff --git a/docs/engineering/matplotlib-compat.md b/spec/matplotlib/compat.md similarity index 87% rename from docs/engineering/matplotlib-compat.md rename to spec/matplotlib/compat.md index 802fac14..f410bd42 100644 --- a/docs/engineering/matplotlib-compat.md +++ b/spec/matplotlib/compat.md @@ -18,7 +18,7 @@ corpus in [`tests/pyplot/corpus/`](../../tests/pyplot/corpus/) covers representa calls from every family. This is 100% 2-D *chart-method* coverage; it is not a claim to reproduce Matplotlib's renderer, transforms, or full Artist graph. -The generated [method-by-method compatibility matrix](matplotlib-compat-matrix.md) +The generated [method-by-method compatibility matrix](compat-matrix.md) is sourced from that snapshot, executable corpus calls, and [`compatibility.json`](../../tests/pyplot/compatibility.json). CI fails if the generated matrix is stale, installs the released `matplotlib==3.11.0` wheel, @@ -64,7 +64,7 @@ dependency-free `triangles=` shorthand into Matplotlib's equivalent | `quiver`, `barbs`, `streamplot` | Native vector endpoint/arrowhead and bounded streamline kernels feeding one instanced segment mark. Barbs are a visual approximation: magnitude maps to a bounded tick count, not WMO 50/10/5 increments. Streamplot always uses the shim's own bounded fixed-step integrator (identical output with or without Matplotlib installed, but paths approximate Matplotlib's adaptive ones); `start_points`, `integration_direction`, array widths/colors and `num_arrows` are honored, and remaining non-default integration options fail loudly | | `tripcolor`, `triplot`, `tricontour`, `tricontourf` | Explicit topology or native dependency-free Delaunay triangulation; indexed geometry and isolines stay in Rust | | `pie` / `pie_label` | Native pie/donut tessellation and the Matplotlib 3.11 `PieContainer` (`values`, `fracs`, grouped text labels) | -| `axhline` / `axvline` / `axhspan` / `axvspan`, `text`, `annotate`, `table` | Fractional span bounds plus data/axes/figure text coordinates are supported. `annotate(arrowprops=)` draws real arrows in every output: offset-point text becomes an engine callout (arrow pinned from label to point across zoom), data-coordinate text an arrow annotation; date-string coordinates convert on datetime axes. Arrowstyles map to head/tail shapes (`->` open V, `-|>` filled, `\|-\|`/brackets bar caps, `fancy`/`simple`/`wedge` filled tapered shafts sized by the text's mutation scale) and `connectionstyle` arc3/angle3/angle become quadratic curves (corner rounding approximated); `alpha` dims the arrow only. `bbox=` becomes label box styles (fill/edge/round corners/`pad`) on the HTML label; static exporters keep the plain label. Text is unclipped like Matplotlib (`clip_on=False`), and axes-fraction text right of the axes box (x > 1, e.g. seaborn-style row titles) reserves right margin in every exporter. `rotation=90/270` renders vertical text in browser, PNG, and SVG with Matplotlib's rotate-then-align box semantics; other angles rotate in browser and SVG output only (native PNG draws them horizontally) | +| `axhline` / `axvline` / `axhspan` / `axvspan`, `text`, `annotate`, `table` | Fractional span bounds plus data/axes/figure text coordinates are supported. `annotate(arrowprops=)` draws real arrows in every output: offset-point text becomes an engine callout (arrow pinned from label to point across zoom), data-coordinate text an arrow annotation; date-string coordinates convert on datetime axes. Arrowstyles map to head/tail shapes (`->` open V, `-\|>` filled, `\|-\|`/brackets bar caps, `fancy`/`simple`/`wedge` filled tapered shafts sized by the text's mutation scale) and `connectionstyle` arc3/angle3/angle become quadratic curves (corner rounding approximated); `alpha` dims the arrow only. `bbox=` becomes label box styles (fill/edge/round corners/`pad`) on the HTML label; static exporters keep the plain label. Text is unclipped like Matplotlib (`clip_on=False`), and axes-fraction text right of the axes box (x > 1, e.g. seaborn-style row titles) reserves right margin in every exporter. `rotation=90/270` renders vertical text in browser, PNG, and SVG with Matplotlib's rotate-then-align box semantics; other angles rotate in browser and SVG output only (native PNG draws them horizontally) | | `from xy.pyplot import FacetGrid` (seaborn-shaped) | Row/column small multiples with seaborn's `map` contract (subset → activate panel → call the pyplot function), shared domains, edge-only axis labels, top-row column titles, and `margin_titles=True` rotated row titles. `hue=`/`palette=`, `col_wrap=`, `map_dataframe`, and `add_legend` fail loudly | | `xlabel` / `ylabel` / `title` / `suptitle` | Suptitles are retained in HTML and multi-panel PNG/SVG | | `legend()` | `loc`, columns, title/font size/colors, frame styling, `borderpad`, `labelspacing`, `fancybox`, `framealpha`, and `shadow` are retained across browser and static output. `loc='best'` chooses the least occupied corner from bounded samples of the current data | @@ -77,12 +77,13 @@ dependency-free `triangles=` shorthand into Matplotlib's equivalent | `twinx()`, `secondary_xaxis()`, `secondary_yaxis()` | second data axes and linked tick-only secondary axes with callable forward/inverse conversions. Secondary-axis ticks are evenly spaced conversions of the primary domain (not Matplotlib's secondary-unit locators) and currently reach the interactive HTML client only — PNG/SVG export does not draw them yet | | `fig, ax = plt.subplots()`; `plt.subplots(n, m, figsize=, dpi=, squeeze=, sharex=, sharey=)` | Grid renders as CSS-grid HTML and stitched PNG/SVG; shared axes use common domains and live linked pan/zoom. `Figure.subplots_adjust(left=, right=, top=, bottom=, wspace=, hspace=)` moves the SubplotParams frame: the grid resolves to explicit figure rectangles and every exporter (HTML, PNG, SVG) positions panels at those rectangles | | `fig.add_subplot(2, 2, 1)` / `add_subplot(221)` | | +| `plt.subplot_mosaic([['A','B'],['C','C']])` / `Figure.subplot_mosaic` | Row sequences (a list of equal-length label strings, or nested label lists) resolve to a uniform grid; each distinct label, in first-appearance order, binds to the next cell, returning `(fig, {label: Axes})` with `figsize=`/`dpi=` sizing the figure. Repeated labels do not span and `'.'` does not blank a cell — the grid keeps one axes per cell — and Matplotlib's single-string forms (`'AB;CC'`, newline-separated blocks) are not parsed into rows | | `gca` / `gcf` / `sca` / `figure(num)` / `close(...)` | matplotlib's implicit-state semantics | | `savefig('x.png' / '.svg' / '.html', dpi=)` | Browser-free PNG/SVG supports both single and multi-panel figures; file-like targets require an explicit `format=` and unsupported metadata/layout/export formats fail loudly | | `plt.show()` | notebooks: inline HTML display; scripts: opens the default browser | | Artists: `set_data` / `set_ydata` / `set_color` / `set_label` / `set_linewidth` / `remove` | mutating a handle rebuilds the chart on next render | | Colors | single letters, `C0`–`C9`, `tab:*`, gray `'0.5'`, RGB(A) tuples, any CSS color | -| `plt.cm.*` / `plt.colormaps[...]` / `cmap=` names | viridis, plasma, inferno, magma, cividis, gray, turbo, coolwarm, Blues, RdYlGn, RdGy, jet, rainbow, Spectral, aliases, and true `*_r` reversal (RdGy/jet render from 11-stop anchor tables sampled from Matplotlib 3.11, linearly interpolated) | +| `plt.cm.*` / `plt.colormaps[...]` / `cmap=` names | viridis, plasma, inferno, magma, cividis, gray, turbo, coolwarm, Blues, Purples, PuBu, RdBu, RdYlGn, RdGy, PiYG, PRGn, jet, rainbow, Spectral, binary, aliases, and true `*_r` reversal (RdGy/jet render from 11-stop anchor tables sampled from Matplotlib 3.11, linearly interpolated) | | `LinearSegmentedColormap.from_list` / `ListedColormap` | Python-side callables (`cmap(np.arange(cmap.N))` → RGBA) for scripts that colormap values themselves; they cannot be passed as `cmap=` to plotting calls (no engine table), which fails loudly | | `plt.colorbar()` / `fig.colorbar()` / `plt.clim()` / `plt.gci()` | Returns a live handle (`set_label`, `set_ticks`); with no mappable it uses the current image the way pyplot does. `ticks=`/`extend=` render in PNG and SVG (the HTML colorbar stays a minimal gradient without tick text); `clim` retargets the mappable's color window and any colorbar derived from it | | `rcParams` | Figure size/DPI, line width/marker size, image cmap/origin, axes color cycle, and all four `axes.spines.*` switches affect every exporter. Pyplot axes default to Matplotlib's four-sided box and each spine can be hidden independently. The chrome keys (axes face/edge/label/title styles, font family/size, tick colors/sizes, legend defaults, figure facecolor) reach the HTML renderer and multi-panel PNG stitching; single-chart PNG and SVG export currently render their own fixed chrome and ignore them. Unknown keys warn once | @@ -112,9 +113,13 @@ arrows. - Custom Matplotlib marker paths, arbitrary clipping graphs, and unsupported collection gradients are rejected rather than silently approximated. -- The shim's figure/axes bookkeeping adds ~10µs per figure over the - declarative API (measured: +9% at 10k points, +2% at 100k, +0.6% at 1M); - `tests/pyplot/test_perf_guardrail.py` gates this relationship in CI. +- The shim's figure/axes bookkeeping adds ~50µs of fixed per-figure cost over + the declarative API (measured 2026-07-14, M-series: +60% at 10k points, +26% + at 100k — fixed cost over an ~85µs baseline, not O(n) work). + `tests/pyplot/test_perf_guardrail.py` gates the relationship at 10k and 100k + with generous headroom (1.6x and 1.5x ceilings plus a 100µs absolute + allowance for CI timer jitter), to catch structural regressions rather than + to re-measure the margin. ## Boundaries (enforced by `tests/pyplot/test_boundaries.py`) @@ -133,4 +138,4 @@ python scripts/sync_matplotlib_compat.py ``` Review the snapshot and generated matrix diff as an API change. Release-level -changes are recorded in [the compatibility changelog](matplotlib-compat-changelog.md). +changes are recorded in [the compatibility changelog](compat-changelog.md). diff --git a/docs/engineering/matplotlib-shim-todo.md b/spec/matplotlib/shim-todo.md similarity index 97% rename from docs/engineering/matplotlib-shim-todo.md rename to spec/matplotlib/shim-todo.md index 670e2b51..dcdf2c03 100644 --- a/docs/engineering/matplotlib-shim-todo.md +++ b/spec/matplotlib/shim-todo.md @@ -102,7 +102,7 @@ The shim can be called complete for ordinary 2-D scripts when: - [x] Optional support for real Matplotlib objects is tested in a dedicated CI environment. - [x] Public compatibility boundaries and intentional exclusions are current - in both this document and `docs/engineering/matplotlib-compat.md`. + in both this document and `spec/matplotlib/compat.md`. ## P0 — make the compatibility claim measurable @@ -259,10 +259,14 @@ appear frequently in ordinary scripts and notebooks. - [x] `get_xlabel`, `get_ylabel`, `get_title`, `get_xaxis`, and `get_yaxis`. Evidence: `tests/pyplot/test_axes_helpers.py::test_ticklabel_minor_label_axis_and_legend_helpers` verifies label/title getters and axis proxy identity. - [x] `get_legend()` and `get_legend_handles_labels()`. Evidence: `tests/pyplot/test_axes_helpers.py::test_ticklabel_minor_label_axis_and_legend_helpers` verifies legend presence and labeled handles. - [x] `set_prop_cycle()` beyond the fixed default color sequence. Evidence: `tests/pyplot/test_axes_helpers.py::test_prop_cycle_setp_getp_rc_context_and_colormap_helpers` verifies per-Axes color cycle order. -- [x] `secondary_xaxis()` and `secondary_yaxis()` if secondary-axis layout is - promoted into supported scope; otherwise keep them explicitly excluded. Evidence: - `tests/pyplot/test_axes_helpers.py::test_subplot2grid_box_and_secondary_axes_contract` - verifies both helpers raise `NotImplementedError` with the compatibility-table error path. +- [x] `secondary_xaxis()` and `secondary_yaxis()`, promoted into supported scope + as tick-only linked axes. Both return a `SecondaryAxis` built from + `location` (`"top"`/`"bottom"`, `"left"`/`"right"`) and a `functions` + forward/inverse pair; only `transform=` raises `NotImplementedError`. + Evidence: `tests/pyplot/test_axes_helpers.py::test_subplot2grid_box_and_secondary_axes_contract` + calls both helpers, labels the returned objects, and asserts the built + chart carries `axis_options["xs1"]["side"] == "top"` and + `axis_options["ys2"]["side"] == "right"`. ### Image, property and convenience helpers @@ -348,7 +352,7 @@ method accepts the call. - [x] `barbs`: non-default increments, flags, rounding, empty-barb, flip, color and size options now fail loudly instead of being discarded. The rendered glyph remains a documented visual approximation (bounded tick - count, not WMO barb geometry) — see `docs/engineering/matplotlib-compat.md`. + count, not WMO barb geometry) — see `spec/matplotlib/compat.md`. - [x] `quiverkey`: coordinates, label positions, fonts and sizing. - [x] `streamplot`: always integrates with the shim's own bounded fixed-step kernel, so output no longer depends on whether Matplotlib is installed; @@ -416,7 +420,7 @@ method accepts the call. - [x] Add a useful typed public surface for `xy.pyplot`, `Axes`, `Figure`, common Artists, containers and return tuples; reduce broad `Any` usage. - [x] Add API documentation generated from the supported compatibility matrix. -- [x] Fix the stale `docs/engineering/chart-roadmap.md` rows that still call pie, vector +- [x] Fix the stale `spec/api/chart-roadmap.md` rows that still call pie, vector fields and irregular-grid families planned even though the shim exposes implementations. - [x] Document approximation levels: exact geometry, equivalent semantics, @@ -658,7 +662,10 @@ streamplot - Exporter chrome beyond backgrounds (fonts, tick/legend styling) stays fixed in single-chart PNG/SVG regardless of rcParams. -- `ax.set_facecolor()` does not exist; axes background is rc-at-creation. +- `ax.set_facecolor()` mutates the per-Axes plot background after creation, + but `set_facecolor(None)` is a silent no-op rather than a reset to the rc + default; restoring it requires passing `rcParams["axes.facecolor"]` back + explicitly. - `ax.patches` holds only pie wedges; bar rects live in containers. - `get_xticks()` on category/time axes falls back to linear ticks; tick density ignores figure size. diff --git a/docs/engineering/contributing.md b/spec/process/contributing.md similarity index 70% rename from docs/engineering/contributing.md rename to spec/process/contributing.md index 2ae8d7d6..2d898b59 100644 --- a/docs/engineering/contributing.md +++ b/spec/process/contributing.md @@ -81,8 +81,11 @@ numbers into docs or posts: make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs ``` -Use `BENCHMARK_KIND=line-decimation`, `install-footprint`, `core-2d`, -`scatter-native`, `kernel-native`, or `auto` for the other report shapes. The +`BENCHMARK_KIND` accepts `auto`, `scatter-vs`, `core-2d`, +`pyplot-vs-matplotlib`, `scatter-native`, `heatmap-native`, `kernel-native`, +`interaction-browser`, `dashboard-browser`, `workflow-native`, +`line-decimation`, `install-footprint`, and `transport-loopback`; the +authoritative list is `KNOWN_KINDS` in `scripts/verify_benchmark_report.py`. The verifier prints a compact report summary with the detected kind, row count, statuses, categories, backend, and git commit so CI logs remain self-describing. @@ -100,13 +103,23 @@ or anything that could become a public performance claim, run: make check-claims ``` -When you edit README snippets, `docs/engineering/api-examples.md`, or the Reflex example +When you edit README snippets, `spec/api/api-examples.md`, or the Reflex example dashboard registry/assets, run: ```bash make check-examples ``` +When you touch `python/xy/pyplot/` or the matplotlib compatibility corpus, run: + +```bash +make check-pyplot +``` + +When you change shim rendering performance, run `make check-pyplot-speed`, which +enforces the per-family 10x static-PNG target via +`benchmarks/bench_pyplot_vs_matplotlib.py` and requires the `.[bench]` extra. + When you touch standalone HTML export, path writes, user-facing text surfaces, tooltips, legends, or the browser client DOM code, run: @@ -140,9 +153,34 @@ For browser render smoke checks, pass a local Chrome/Chromium executable: make check-browser CHROMIUM=/path/to/chrome ``` -This runs the same split browser checks that CI names as `Browser lifecycle -smoke (Chromium)`, `Browser visual regression smoke (Chromium)`, and `Browser -interaction stress smoke (Chromium)`. +This runs six split browser checks, which CI runs as separate steps: + +| Check | CI step name | +| --- | --- | +| `render_smoke_nonumpy` | `Headless render smoke (stdlib + Chromium)` | +| `smoke_render` | `Real-Figure render smoke (numpy + Chromium)` | +| `reflex_lifecycle_smoke` | `Browser lifecycle smoke (Chromium)` | +| `visual_regression_smoke` | `Browser visual regression smoke (Chromium)` | +| `step_tier_smoke` | `Step tier-update smoke (Chromium)` | +| `interaction_stress_smoke` | `Browser interaction stress smoke (Chromium)` | + +The stdlib payload gate runs `scripts/render_smoke_nonumpy.py`. It hand-builds a +payload from stdlib `array` and `struct` in exactly the wire shape +`build_payload` emits, drives the standalone JS bundle in Chromium, and reads +back a lit-pixel count via `gl.readPixels`. It needs neither numpy nor PyPI, so +it covers the render client in a locked-down environment. + +The real-Figure gate runs `scripts/smoke_render.py`. It builds a standalone page +the same way `Chart.to_html` does from an actual Figure (line decimation plus +scatter), then counts non-transparent pixels via `gl.readPixels`. This is the +end-to-end Figure-to-browser path that the hand-built stdlib smoke cannot reach. + +The step tier gate runs `scripts/step_tier_smoke.py`. Step geometry is expanded +client-side after LOD, so both upload paths — the initial build and the +`tier_update` refinement that replaces vertex buffers on zoom — must run the +same expansion. It renders a decimated `step` chart, feeds the view a synthetic +`tier_update` exactly as the kernel would ship it, and asserts the re-uploaded +vertex stream still contains the step risers. The lifecycle gate runs `scripts/reflex_lifecycle_smoke.py`: every committed XY demo asset is loaded repeatedly, and each child chart must stay @@ -154,9 +192,13 @@ requires the rebuilt chart to remain nonblank. The iframe shell also exercises hash navigation, fast scrolling, resize and visibility events, a full iframe remount, an in-place iframe reload, and a hidden-boot/reveal pass where charts initialize at zero-sized iframe dimensions before becoming visible. The custom -chrome, business overview, and retention cohort assets are tracked as critical -reports in every shell phase. A blank, destroyed, shortened lifecycle, failed -context restore, or missing critical asset/phase pair is a failing browser gate. +chrome, business overview, retention cohort, and the two live drilldown assets +(`live_drilldown_10m.html`, `live_drilldown_100m.html`) are tracked as critical +reports in every shell phase. Critical assets are enforced per shell asset group +rather than across one flat asset set: the live drilldown assets form their own +group, and each group requires only its own critical assets in every phase. A +blank, destroyed, shortened lifecycle, failed context restore, or missing +critical asset/phase pair is a failing browser gate. The visual gate runs `scripts/visual_regression_smoke.py`. It is layout-aware: beyond global nonblank/color checks, it verifies title, plot, x-axis, and y-axis @@ -186,8 +228,12 @@ On macOS, pass the executable inside the app bundle, for example Use `make list-checks` to see check names, or `python scripts/verify_local.py --dry-run --full` to print commands without -running them. Browser checks are listed even before a Chrome path is configured; -dry-run output uses `` until you pass `--chromium` or `CHROMIUM=...`. +running them. `--dry-run --full` prints only the quick and full gates; browser +checks are appended only with `--browser`, which requires `--chromium PATH`. +`make list-checks` does show the browser checks before a Chrome path is +configured, marked `[requires chromium]`. To see a browser command rendered with +the `` placeholder, name it explicitly, for example +`python scripts/verify_local.py --dry-run --only reflex_lifecycle_smoke`. ## Pull Request Checklist @@ -230,7 +276,7 @@ Do not write broad claims like "faster than Plotly" without naming: - chart type - data size and shape -- backend (`native` or `numpy`) +- backend (always `native`; there is no NumPy fallback) - render target - whether browser time-to-first-render is included - whether the result is exact markers, decimated geometry, density, or adaptive @@ -238,5 +284,5 @@ Do not write broad claims like "faster than Plotly" without naming: Numeric multipliers such as "10x faster" or "5x smaller" need the same measured benchmark context. -When in doubt, phrase it as a measured row from `docs/engineering/benchmark.md`, not as a +When in doubt, phrase it as a measured row from `spec/benchmarks/results.md`, not as a universal product claim, and run `make check-claims` before publishing. diff --git a/docs/engineering/production-readiness.md b/spec/process/production-readiness.md similarity index 89% rename from docs/engineering/production-readiness.md rename to spec/process/production-readiness.md index 91a645e9..3d7a91d6 100644 --- a/docs/engineering/production-readiness.md +++ b/spec/process/production-readiness.md @@ -69,18 +69,28 @@ These must pass before publishing or making a broad performance claim. | HTML export safety | Inline JSON/script escaping, atomic path writes, hostile user strings, and browser client text-node insertion stay protected | `make check-security` | | Python tests | Native backend passes | `pytest -q` | | Python style | Library, tests, scripts, and benchmarks lint clean | `ruff check .` and `ruff format --check .` | -| Type surface | Shippable library is type-checkable and ships a full-package `py.typed` marker | `ty check python` | +| Matplotlib reference | The reviewed compatibility snapshot matches the pinned released matplotlib reference, and the `xy.pyplot` shim passes its interoperability and dual-engine corpus suites | `python scripts/sync_matplotlib_compat.py --check` and `pytest tests/pyplot` | | Rust core | Native kernels pass and lint clean | `cargo test` and `cargo clippy --all-targets -- -D warnings` | | Native ABI | C ABI can be loaded from the built core | `python scripts/abi_smoke.py` | | JavaScript | Committed bundles match source | `node js/build.mjs --check` | | Browser render | WebGL smoke reaches real pixels | `python scripts/render_smoke_nonumpy.py ` | | Accessibility / cross-browser | Semantic interaction checks plus tolerant WebGL/layout comparison pass in Chromium, Firefox, and WebKit | `make check-conformance` | | Real chart render | A real composed chart exports and paints in Chromium | `python scripts/smoke_render.py ` | +| Step tier update | A decimated `step` chart keeps its risers after a synthetic kernel `tier_update` replaces the vertex buffers | `python scripts/step_tier_smoke.py ` | +| Dashboard reliability | 10/20/50-chart dashboards stay nonblank under the render client's context governor | `python benchmarks/bench_dashboard.py --chart-counts 10,20,50 --chromium --json dashboard-smoke.json` then `python scripts/verify_benchmark_report.py dashboard-smoke.json --kind dashboard-browser` | | sdist | Source archive contains required source/bundles, benchmark regression harness/baseline, release docs/tests/scripts, the Reflex example app, `PKG-INFO` version/dependencies matching `pyproject.toml`, no duplicate members, and no generated junk | `python scripts/verify_sdist.py dist/*.tar.gz` | | Native wheel | Platform wheel contains package-only files, exactly one native library, `METADATA` version/dependencies matching `pyproject.toml`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, and is tagged non-pure | `python scripts/verify_wheel.py dist/*.whl --expect-native` | | Fallback wheel | No-toolchain wheel contains package-only files, `METADATA` version/dependencies matching `pyproject.toml`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, is pure, and contains no native library | `python scripts/verify_wheel.py dist/*.whl --expect-pure` | | Wheel size | Platform wheel remains small enough for notebook installs | CI budget: 15 MB | -| Benchmark artifact | JSON benchmark reports carry schema, environment, categories, row status, and finite non-negative metrics; native reports must declare the native backend | `python scripts/verify_benchmark_report.py benchmark.json --kind scatter-vs`; repeat for line, install, core-2D, native, interaction, dashboard, and workflow artifacts | +| Benchmark artifact | JSON benchmark reports carry schema, environment, categories, row status, and finite non-negative metrics; native reports must declare the native backend | `python scripts/verify_benchmark_report.py benchmark.json --kind scatter-vs`; repeat for line, install, core-2D, pyplot-vs-matplotlib, native, interaction, dashboard, and workflow artifacts | + +Type checking is **advisory, not release-blocking**. CI runs `ty check python` +and reports findings without failing the build, and `scripts/verify_local.py` +registers the same check with `advisory=True`, so `make check-full` prints +warnings for type findings rather than failing. Promoting it to a hard gate is +tracked in the Hardening Backlog. The full-package `py.typed` marker is a hard +gate, but it is enforced by `make check-api` +(`scripts/check_public_api.py`), not by the type checker. ## Standalone HTML Safety @@ -98,8 +108,11 @@ reports, and sharing a single file, but it has a clear security contract: replace the target after the full document is flushed, so failed writes do not corrupt the previous standalone artifact. - The standalone file emits a defensive `Content-Security-Policy` meta tag that - blocks network fetches, workers, objects, forms, and external images while - allowing the inline scripts/styles required by single-file export. + blocks network fetches, external worker scripts, objects, forms, and external + images, and pins `base-uri 'none'`, while allowing the inline scripts/styles + required by single-file export. Workers are restricted to `blob:` URLs so the + bundled density re-bin worker can boot from its own inlined source; no + external worker script can load. - The browser client inserts user-facing text with `textContent` or text nodes; HTML parser sinks such as `innerHTML` are reserved for fixed internal icons, not titles, labels, legends, categories, or tooltips. @@ -121,10 +134,13 @@ production-facing push: | Changed surface | Focused gate | |---|---| | README/API prose, examples, public benchmark wording | `make check-docs` | -| README snippets, `docs/engineering/api-examples.md`, Reflex chart registry/assets | `make check-examples` | +| README snippets, `spec/api/api-examples.md`, Reflex chart registry/assets | `make check-examples` | | Public validation, error messages, builder rollback, LOD/drill mutation boundaries, chart/widget caching | `make check-errors` | | Public exports, lazy import mappings, component factories, public annotations | `make check-api` | | Import-time budget, `xy.__init__`, dependency boundaries, widget/export/backend import boundaries | `make check-import` | +| `xy.pyplot` shim behavior, matplotlib interoperability, reference corpus | `make check-pyplot` | +| Reviewed matplotlib compatibility snapshot (`spec/matplotlib/compat-matrix.md`) | `python scripts/sync_matplotlib_compat.py --check` | +| `xy.pyplot` speed margin against matplotlib | `make check-pyplot-speed` | | Standalone HTML export, path writes, user text, tooltips, legends, browser DOM insertion | `make check-security` | | Benchmark harness code, environment metadata, report schema, regressions | `make check-benchmark-harness` | | Generated benchmark JSON artifacts | `make check-benchmark-report BENCHMARK_JSON=benchmark.json BENCHMARK_KIND=scatter-vs` | @@ -147,9 +163,12 @@ wording: make check-docs ``` -The browser gates are split into three app-facing checks that match the CI step +The browser gates are split into app-facing checks that match the CI step names: `Browser lifecycle smoke (Chromium)`, `Browser visual regression smoke -(Chromium)`, and `Browser interaction stress smoke (Chromium)`. The Reflex +(Chromium)`, `Step tier-update smoke (Chromium)`, `Browser interaction stress +smoke (Chromium)`, and `Browser dashboard reliability smoke (Chromium)`. +`make check-browser` runs all of these except the dashboard reliability smoke, +which runs in CI only. The Reflex lifecycle smoke remounts every committed XY iframe asset, the visual regression smoke screenshots generated representative chart families plus every committed XY Reflex gallery asset except the Plotly comparison page, @@ -198,7 +217,7 @@ verify those exact files rather than rebuilding locally: make check-artifacts SDIST=/path/to/xy.tar.gz WHEEL=/path/to/xy.whl ``` -Use this after editing README snippets, `docs/engineering/api-examples.md`, or the Reflex +Use this after editing README snippets, `spec/api/api-examples.md`, or the Reflex dashboard chart registry/assets: ```bash @@ -349,7 +368,7 @@ Before tagging a release: package-only: docs, tests, benchmarks, scripts, and `examples/reflex/` are sdist-only. - Confirm the wheel size budget is still below 15 MB. -- Confirm README examples and `docs/engineering/api-examples.md` run against the tagged API. +- Confirm README examples and `spec/api/api-examples.md` run against the tagged API. - Confirm package metadata uses measured, scoped language rather than broad "faster/best" positioning. - Confirm performance claims mention chart type, mode, backend, data size, and @@ -364,7 +383,7 @@ Safe claims: - The native backend is substantially faster than pure Python/NumPy for kernel work covered by the benchmarks. - Current 2D core chart benchmarks beat Plotly on the measured payload-prep, - payload-size, and TTFR rows documented in `docs/engineering/benchmark.md`. + payload-size, and TTFR rows documented in `spec/benchmarks/results.md`. Claims that need qualification: diff --git a/docs/engineering/security-audit-2026-07-06.md b/spec/process/security-audit-2026-07-06.md similarity index 65% rename from docs/engineering/security-audit-2026-07-06.md rename to spec/process/security-audit-2026-07-06.md index bdcb7da4..f3a8ed06 100644 --- a/docs/engineering/security-audit-2026-07-06.md +++ b/spec/process/security-audit-2026-07-06.md @@ -31,6 +31,17 @@ Fix: Strict nonce/hash CSP still requires a host wrapper that serves the JS bundle as a separate asset. +#### Status as of 2026-07-20 (XY-SEC-2026-01) + +`worker-src` is no longer `'none'`. It was relaxed to `worker-src blob:` on +2026-07-08 in commit b353dea ("Standalone density re-bin in a Web Worker"), so +the standalone density re-bin worker can boot from a Blob URL of its own +bundled source. No external worker script can load under that directive. The +shipped policy is `_STANDALONE_CSP` in `python/xy/export.py`; every other +directive listed above is unchanged. `tests/test_static_client_security.py` +asserts the directive is exactly `worker-src blob:`, and +`docs/guides/serving-csp-and-offline-use.md` documents it for host operators. + ### XY-SEC-2026-02: Legend swatches accepted raw CSS paint strings Severity: medium. @@ -65,6 +76,23 @@ Fix: - Threaded the option through `Figure.to_png()` and composed chart `to_png()`. - Added tests proving `--no-sandbox` only appears when explicitly requested. +#### Status as of 2026-07-20 (XY-SEC-2026-03) + +The last bullet no longer holds as written. Commit 8cda831 ("Stabilize Chromium +PNG export in CI"), landed 2026-07-06, added an automatic fallback: +`html_to_png` launches sandboxed first, and if that attempt produces no +screenshot it rebuilds the argv with `--no-sandbox` inserted and re-runs before +raising. The error message on total failure reports both attempts. +`_browser_session` mirrors this for the persistent path, retrying +`ChromiumSession(..., sandbox=False)` on `ChromiumError`. So `--no-sandbox` can +appear without the caller requesting it. + +Sandboxing remains the default and the first attempt; the escape hatch and the +threading through `Figure.to_png()` are unchanged. The fallback is an accepted +residual risk taken to keep CI and container rasterization working where the +sandbox cannot initialize. Follow-up pending: make the fallback opt-in (or at +minimum warn on the downgrade) so a silent sandbox loss is observable. + ### XY-SEC-2026-04: Pyramid native-boundary validation was weaker than other kernels Severity: low/medium robustness. @@ -117,13 +145,30 @@ Fix: - CodSpeed and benchmark workflows assert native backend before performance runs. +#### Status as of 2026-07-20 (Confirmed Controls) + +The Rust core is no longer std-only, so the "Cargo dependency tree is empty" +control above and the `cargo tree --locked` evidence line below are both +superseded. Commit c3c867b, landed 2026-07-11, added one direct dependency to +`Cargo.toml` — `png = "0.18.1"`, for the native raster encoder's fdeflate fast +path — which pulls in eight transitive crates: `bitflags`, `crc32fast`, +`cfg-if`, `fdeflate`, `simd-adler32`, `flate2`, `miniz_oxide`, and `adler2`. +`Cargo.lock` therefore holds nine third-party packages plus `xy-core`. + +The tree is still shallow and single-rooted, but "no third-party Rust crates" +is no longer an accurate standing control. Follow-up pending: add `cargo audit` +(or `cargo deny`) to CI now that a third-party tree exists, matching the +`pip-audit` coverage already run on the Python side. + ## Tooling Evidence - `uv tool run pip-audit --progress-spinner off .`: no known vulnerabilities. - `uv tool run pip-audit --progress-spinner off -r requirements.txt` from `examples/reflex/`: no known vulnerabilities; local editable `xy` is skipped because it is not a PyPI package. -- `cargo tree --locked`: only `xy-core`, no third-party Rust crates. +- `cargo tree --locked`: only `xy-core`, no third-party Rust crates. (True on + the audit date only; superseded by the 2026-07-20 status note above, which + records `png` plus eight transitive crates.) - `node js/build.mjs --check`: static JS bundles fresh. - `make check-security`: passed. - `make check-ci`: passed. @@ -159,3 +204,25 @@ Fix: with Bun installed. - This audit did not perform browser fuzzing, GPU-driver fuzzing, native memory sanitizer runs, or a hosted-app penetration test. + +#### Status as of 2026-07-20 (Residual Risks) + +The first two bullets above understate the current exposure. "Keep Chromium's +sandbox enabled" is caller-side advice that the library does not enforce end to +end, and unsandboxed launches are not confined to CI smoke scripts: since commit +8cda831 the public export path downgrades itself automatically. `html_to_png` +re-runs with `--no-sandbox` inserted when the sandboxed launch produces no +screenshot (`python/xy/export.py:509-526`, the retry itself at `:511-519` and +the two-attempt error assembly through `:526`), and `_browser_session` retries +`ChromiumSession(..., sandbox=False)` on `ChromiumError` +(`python/xy/export.py:926-931`). Neither path emits a warning, so the downgrade +is silent. See [the 2026-07-20 status note under +XY-SEC-2026-03](#status-as-of-2026-07-20-xy-sec-2026-03) above and +`spec/api/export.md` §7. + +Read that way, `sandbox=True` is a preference, not a guarantee: rendering +untrusted HTML through `to_png()` can execute unsandboxed on a host where the +sandbox cannot initialize. Container/worker isolation is therefore the load- +bearing control, not the sandbox flag. Follow-up pending (same item as +XY-SEC-2026-03): make the fallback opt-in, or at minimum warn on the downgrade, +so a sandbox loss is observable. diff --git a/tests/test_check_regressions.py b/tests/test_check_regressions.py index cce8a444..4e2aaba2 100644 --- a/tests/test_check_regressions.py +++ b/tests/test_check_regressions.py @@ -7,9 +7,11 @@ import pytest +ROOT = Path(__file__).resolve().parents[1] + def _load_regression_module(): - path = Path(__file__).resolve().parents[1] / "scripts" / "check_regressions.py" + path = ROOT / "scripts" / "check_regressions.py" spec = importlib.util.spec_from_file_location("check_regressions", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) @@ -21,6 +23,40 @@ def _load_regression_module(): check_regressions = _load_regression_module() +def test_committed_metrics_document_matches_baseline_contract() -> None: + expected = set(json.loads((ROOT / "benchmarks/baseline.json").read_text())["metrics"]) + lines = (ROOT / "spec/benchmarks/metrics.md").read_text(encoding="utf-8").splitlines() + documented = { + cells[0] + for line in lines + if line.startswith("| ") + if len(cells := [cell.strip() for cell in line.strip("|").split("|")]) == 2 + if cells[0] not in {"metric", "---"} + } + + assert documented == expected + assert any(line.startswith("Source CI reports: commit ") for line in lines) + + +def test_report_provenance_uses_latest_timestamp() -> None: + first = { + "environment": { + "generated_at_utc": "2026-07-20T23:32:01Z", + "git": {"commit": "abc123"}, + } + } + second = { + "environment": { + "generated_at_utc": "2026-07-20T23:32:08Z", + "git": {"commit": "abc123"}, + } + } + + assert check_regressions._report_provenance(first, second) == ( + "Source CI reports: commit `abc123`; latest measurement `2026-07-20T23:32:08Z`." + ) + + def test_flatten_accepts_schema_v2_scatter_report() -> None: scatter = { "schema_version": 2, diff --git a/tests/test_components.py b/tests/test_components.py index 58da3b6d..7ef7f6e3 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1918,7 +1918,7 @@ def test_selection_payload(): def test_chart_styles_prop_is_the_documented_per_slot_mechanism() -> None: - """docs/engineering/styling.md's fourth mechanism: `styles={slot: {...}}` on xy.chart — + """spec/api/styling.md's fourth mechanism: `styles={slot: {...}}` on xy.chart — slot-validated, CSS-validated, merged with per-component `style=`.""" xs = np.arange(6.0) chart = xy.chart( diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py index 063dc466..47f59f24 100644 --- a/tests/test_docs_examples.py +++ b/tests/test_docs_examples.py @@ -2,19 +2,21 @@ import ast import json +import re from pathlib import Path +from urllib.parse import unquote import pytest ROOT = Path(__file__).resolve().parents[1] -ENGINEERING_DOCS = ROOT / "docs" / "engineering" -API_EXAMPLES = ENGINEERING_DOCS / "api-examples.md" +SPEC_DOCS = ROOT / "spec" +API_EXAMPLES = SPEC_DOCS / "api" / "api-examples.md" README = ROOT / "README.md" CONTRIBUTING = ROOT / "CONTRIBUTING.md" SECURITY = ROOT / "SECURITY.md" -BENCHMARK_DOC = ENGINEERING_DOCS / "benchmark.md" -PRODUCTION_DOC = ENGINEERING_DOCS / "production-readiness.md" -REFLEX_SHAPED_API_DOC = ENGINEERING_DOCS / "design" / "reflex-shaped-api.md" +BENCHMARK_DOC = SPEC_DOCS / "benchmarks" / "results.md" +PRODUCTION_DOC = SPEC_DOCS / "process" / "production-readiness.md" +REFLEX_SHAPED_API_DOC = SPEC_DOCS / "design" / "reflex-shaped-api.md" EXPECTED_QUICK_REFERENCE = { "Line": ("xy.line_chart", "xy.line"), "Scatter": ("xy.scatter_chart", "xy.scatter"), @@ -29,6 +31,34 @@ "Heatmap": ("xy.heatmap_chart", "xy.heatmap"), } +LOCAL_MARKDOWN_LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)") + + +def test_spec_local_links_resolve() -> None: + broken: list[str] = [] + for path in sorted(SPEC_DOCS.rglob("*.md")): + in_fence = False + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if line.lstrip().startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + for match in LOCAL_MARKDOWN_LINK_RE.finditer(line): + raw_target = match.group(1).strip() + if raw_target.startswith("<") and raw_target.endswith(">"): + raw_target = raw_target[1:-1] + target = raw_target.split(maxsplit=1)[0] + if target.startswith(("#", "/", "http://", "https://", "mailto:")): + continue + target_path = unquote(target.split("#", 1)[0].split("?", 1)[0]) + if not target_path: + continue + resolved = (path.parent / target_path).resolve() + if not resolved.is_relative_to(ROOT) or not resolved.exists(): + broken.append(f"{path.relative_to(ROOT)}:{line_number}: {target}") + assert not broken, "broken local spec links:\n" + "\n".join(broken) + def _python_examples(path: Path = API_EXAMPLES) -> list[tuple[str, str]]: examples: list[tuple[str, str]] = [] @@ -364,7 +394,7 @@ def test_benchmark_docs_name_ci_report_artifacts() -> None: "benchmark-report", "regression-benchmark-report", "docs/benchmark_ci.md", - "docs/engineering/benchmark_metrics.md", + "spec/benchmarks/metrics.md", "scatter.json", "kernel.json", "compact summary", @@ -396,7 +426,7 @@ def test_docs_name_benchmark_harness_shortcut() -> None: readme = " ".join(README.read_text(encoding="utf-8").split()) production = " ".join(PRODUCTION_DOC.read_text(encoding="utf-8").split()) contributing = " ".join( - (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8").split() + (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8").split() ) for text in (readme, production, contributing): @@ -410,7 +440,7 @@ def test_docs_name_claim_guardrail_shortcut() -> None: readme = " ".join(README.read_text(encoding="utf-8").split()) production = " ".join(PRODUCTION_DOC.read_text(encoding="utf-8").split()) contributing = " ".join( - (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8").split() + (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8").split() ) for text in (readme, production, contributing): @@ -470,7 +500,7 @@ def test_docs_name_split_browser_hardening_gates() -> None: readme = " ".join(README.read_text(encoding="utf-8").split()) production = " ".join(PRODUCTION_DOC.read_text(encoding="utf-8").split()) contributing = " ".join( - (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8").split() + (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8").split() ) step_names = [ diff --git a/tests/test_figure.py b/tests/test_figure.py index 918e9df2..8cd467c2 100644 --- a/tests/test_figure.py +++ b/tests/test_figure.py @@ -2020,7 +2020,7 @@ def fake_run(args, **kwargs): # --------------------------------------------------------------------------- -# Mark-level styling (docs/engineering/styling.md#styling-the-marks): CSS linear-gradient +# Mark-level styling (spec/api/styling.md#styling-the-marks): CSS linear-gradient # fills, rounded corners + stroke borders on the rect family, smooth curves. # --------------------------------------------------------------------------- diff --git a/tests/test_lod.py b/tests/test_lod.py index f3532c08..e8b641c4 100644 --- a/tests/test_lod.py +++ b/tests/test_lod.py @@ -198,9 +198,7 @@ def add_encoded(self, column): # type: ignore[no-untyped-def] def test_lod_architecture_doc_names_shared_extension_points() -> None: - text = (ROOT / "docs" / "engineering" / "design" / "lod-architecture.md").read_text( - encoding="utf-8" - ) + text = (ROOT / "spec" / "design" / "lod-architecture.md").read_text(encoding="utf-8") for marker in ( "ViewportRequest.from_client", diff --git a/tests/test_python_floor.py b/tests/test_python_floor.py index c35b4ba6..08ea4d11 100644 --- a/tests/test_python_floor.py +++ b/tests/test_python_floor.py @@ -26,7 +26,7 @@ def _write_floor_repo( readme: str = "Install with Python 3.11+.\n", production_readiness: str = "The current contract is Python 3.11+ only.\n", ) -> None: - root.joinpath("docs", "engineering").mkdir(parents=True) + root.joinpath("spec", "process").mkdir(parents=True) root.joinpath("pyproject.toml").write_text( "[project]\n" f'requires-python = "{requires_python}"\n' @@ -36,7 +36,7 @@ def _write_floor_repo( encoding="utf-8", ) root.joinpath("README.md").write_text(readme, encoding="utf-8") - root.joinpath("docs", "engineering", "production-readiness.md").write_text( + root.joinpath("spec", "process", "production-readiness.md").write_text( production_readiness, encoding="utf-8", ) diff --git a/tests/test_static_client_security.py b/tests/test_static_client_security.py index 5970b5b0..18289272 100644 --- a/tests/test_static_client_security.py +++ b/tests/test_static_client_security.py @@ -687,7 +687,7 @@ def test_client_renders_mark_level_styling() -> None: """Gradient fills (premultiplied, currentColor-aware), rounded corners + stroke borders on both rect-family GPU programs, and curve:"smooth" monotone-cubic densification are first-class mark styling - (docs/engineering/styling.md#styling-the-marks).""" + (spec/api/styling.md#styling-the-marks).""" required = ( "xyGradSample(", # gradient sampler shared by area + rect shaders "u_gradMode", diff --git a/tests/test_type_surface.py b/tests/test_type_surface.py index a067aa86..4fc0d58a 100644 --- a/tests/test_type_surface.py +++ b/tests/test_type_surface.py @@ -208,9 +208,7 @@ def test_chart_dom_slots_are_public_styling_contract() -> None: assert len(components.CHART_DOM_SLOTS) == len(set(components.CHART_DOM_SLOTS)) assert all(slot == slot.lower() and " " not in slot for slot in components.CHART_DOM_SLOTS) - design = (ROOT / "docs" / "engineering" / "design" / "reflex-shaped-api.md").read_text( - encoding="utf-8" - ) + design = (ROOT / "spec" / "design" / "reflex-shaped-api.md").read_text(encoding="utf-8") for slot in components.CHART_DOM_SLOTS: assert f"`{slot}`" in design diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index 98f38c63..4d01f5c9 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -321,7 +321,7 @@ def test_ci_workflow_rejects_missing_regression_gate(tmp_path: Path) -> None: path.write_text( workflow.replace( " python3 scripts/check_regressions.py --scatter scatter.json --kernel kernel.json \\\n" - " --transport transport.json --emit-md docs/engineering/benchmark_metrics.md\n", + " --transport transport.json --emit-md spec/benchmarks/metrics.md\n", "", ), encoding="utf-8", diff --git a/tests/test_verify_local.py b/tests/test_verify_local.py index 4dd890bb..0ef3bcf6 100644 --- a/tests/test_verify_local.py +++ b/tests/test_verify_local.py @@ -20,7 +20,7 @@ def _load_verify_local_module(): verify_local = _load_verify_local_module() ROOT = Path(__file__).resolve().parents[1] -ENGINEERING_DOCS = ROOT / "docs" / "engineering" +SPEC_DOCS = ROOT / "spec" def test_default_selection_is_quick_checks_only() -> None: @@ -480,7 +480,7 @@ def test_contributor_setup_builds_native_core_and_docs_use_it() -> None: contributor_docs = ( (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8"), - (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8"), + (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8"), (ROOT / "README.md").read_text(encoding="utf-8"), (ROOT / "docs" / "api-reference" / "contributing.md").read_text(encoding="utf-8"), ) @@ -606,8 +606,8 @@ def test_makefile_exposes_claim_guardrail_shortcut() -> None: def test_contributor_docs_name_full_gate_toolchain_requirements() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -618,19 +618,19 @@ def test_contributor_docs_name_full_gate_toolchain_requirements() -> None: def test_docs_name_example_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): assert "make check-examples" in text - assert "docs/engineering/api-examples.md" in contributing + assert "spec/api/api-examples.md" in contributing assert "Reflex example" in contributing def test_docs_name_docs_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") production_inline = " ".join(production.split()) @@ -642,8 +642,8 @@ def test_docs_name_docs_verification_shortcut() -> None: def test_docs_name_security_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -654,8 +654,8 @@ def test_docs_name_security_verification_shortcut() -> None: def test_docs_name_error_safety_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -669,8 +669,8 @@ def test_docs_name_error_safety_verification_shortcut() -> None: def test_docs_name_api_surface_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") readme_inline = " ".join(readme.split()) @@ -682,8 +682,8 @@ def test_docs_name_api_surface_verification_shortcut() -> None: def test_docs_name_import_budget_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -696,8 +696,8 @@ def test_docs_name_import_budget_verification_shortcut() -> None: def test_docs_name_ci_workflow_verification_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -708,8 +708,8 @@ def test_docs_name_ci_workflow_verification_shortcut() -> None: def test_docs_name_claim_guardrail_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): @@ -720,8 +720,8 @@ def test_docs_name_claim_guardrail_shortcut() -> None: def test_docs_name_benchmark_harness_shortcut() -> None: - contributing = (ENGINEERING_DOCS / "contributing.md").read_text(encoding="utf-8") - production = (ENGINEERING_DOCS / "production-readiness.md").read_text(encoding="utf-8") + contributing = (SPEC_DOCS / "process" / "contributing.md").read_text(encoding="utf-8") + production = (SPEC_DOCS / "process" / "production-readiness.md").read_text(encoding="utf-8") readme = (ROOT / "README.md").read_text(encoding="utf-8") for text in (contributing, production, readme): diff --git a/tests/test_verify_sdist.py b/tests/test_verify_sdist.py index 406ac0aa..b3b243be 100644 --- a/tests/test_verify_sdist.py +++ b/tests/test_verify_sdist.py @@ -35,7 +35,7 @@ "# xy\n\n" "## Stable vs. Experimental\n\n" "Python 3.11+ package import is documented here.\n" - "See docs/engineering/api-examples.md for examples.\n" + "See spec/api/api-examples.md for examples.\n" "Run make check-examples after editing example docs.\n" + ("readme padding\n" * 100) ) API_EXAMPLES_MD = ( @@ -50,7 +50,7 @@ BENCHMARK_MD = ( "# Benchmark\n\n" "The docs/benchmark_ci.md numbers land in the benchmark-report artifact.\n" - "The docs/engineering/benchmark_metrics.md regression table, scatter.json, and kernel.json " + "The spec/benchmarks/metrics.md regression table, scatter.json, and kernel.json " "land in the regression-benchmark-report artifact.\n" + ("benchmark padding\n" * 100) ) PRODUCTION_READINESS_MD = ( @@ -69,7 +69,7 @@ "## Pull Request Checklist\n\n" "Run make check-full, make check-sdist, make check-wheel, and " "make check-benchmark-report.\n" - "Run make check-examples for README snippets, docs/engineering/api-examples.md, and " + "Run make check-examples for README snippets, spec/api/api-examples.md, and " "the Reflex example app.\n\n" "## Performance Claims\n\n" "Claims need benchmark context.\n" + ("contributing padding\n" * 100) @@ -185,13 +185,13 @@ def _write_sdist( data = ENTRIES_JS.encode("utf-8") elif name == "README.md": data = README_MD.encode("utf-8") - elif name == "docs/engineering/api-examples.md": + elif name == "spec/api/api-examples.md": data = API_EXAMPLES_MD.encode("utf-8") - elif name == "docs/engineering/benchmark.md": + elif name == "spec/benchmarks/results.md": data = BENCHMARK_MD.encode("utf-8") - elif name == "docs/engineering/production-readiness.md": + elif name == "spec/process/production-readiness.md": data = PRODUCTION_READINESS_MD.encode("utf-8") - elif name == "docs/engineering/contributing.md": + elif name == "spec/process/contributing.md": data = CONTRIBUTING_MD.encode("utf-8") elif name == "examples/reflex/README.md": data = REFLEX_README_MD.encode("utf-8") @@ -288,14 +288,57 @@ def test_verify_sdist_rejects_partial_type_marker(tmp_path: Path) -> None: def test_verify_sdist_rejects_missing_production_docs_or_tooling(tmp_path: Path) -> None: sdist = tmp_path / "xy-0.0.1.tar.gz" - _write_sdist( - sdist, omit={"docs/engineering/production-readiness.md", "scripts/verify_local.py"} - ) + _write_sdist(sdist, omit={"spec/process/production-readiness.md", "scripts/verify_local.py"}) with pytest.raises(AssertionError, match="production-readiness"): verify_sdist.verify_sdist(str(sdist)) +def test_verify_sdist_requires_every_spec_subdirectory(tmp_path: Path) -> None: + """A pinned member per group is not enough — the group itself must survive.""" + sdist = tmp_path / "xy-0.0.1.tar.gz" + _write_sdist(sdist, omit={"spec/matplotlib/compat.md"}) + + with pytest.raises(AssertionError, match="spec/matplotlib/compat"): + verify_sdist.verify_sdist(str(sdist)) + + +@pytest.mark.parametrize("subdir", verify_sdist.SPEC_SUBDIRS) +def test_require_spec_layout_rejects_empty_subdirectory(subdir: str) -> None: + files = {f"spec/{name}/doc.md" for name in verify_sdist.SPEC_SUBDIRS} + files |= { + "spec/assets/benchmark-snapshot.svg", + "spec/assets/launch-benchmark-comparison.svg", + } + verify_sdist._require_spec_layout(files) + + files.discard(f"spec/{subdir}/doc.md") + with pytest.raises(AssertionError, match=subdir): + verify_sdist._require_spec_layout(files) + + +def test_require_spec_layout_rejects_missing_asset_snapshots() -> None: + files = {f"spec/{name}/doc.md" for name in verify_sdist.SPEC_SUBDIRS} + files.add("spec/assets/benchmark-snapshot.svg") + + with pytest.raises(AssertionError, match="spec/assets"): + verify_sdist._require_spec_layout(files) + + +def test_require_spec_layout_ignores_non_markdown_group_members() -> None: + """A group left holding only stray non-markdown files is still empty.""" + files = {f"spec/{name}/doc.md" for name in verify_sdist.SPEC_SUBDIRS} + files |= { + "spec/assets/benchmark-snapshot.svg", + "spec/assets/launch-benchmark-comparison.svg", + } + files.discard("spec/design/doc.md") + files.add("spec/design/notes.txt") + + with pytest.raises(AssertionError, match="design"): + verify_sdist._require_spec_layout(files) + + def test_verify_sdist_rejects_missing_benchmark_harness(tmp_path: Path) -> None: sdist = tmp_path / "xy-0.0.1.tar.gz" _write_sdist(sdist, omit={"benchmarks/bench_vs.py", "benchmarks/environment.py"}) @@ -364,7 +407,7 @@ def test_verify_sdist_rejects_stale_api_examples_doc(tmp_path: Path) -> None: sdist = tmp_path / "xy-0.0.1.tar.gz" _write_sdist( sdist, - replacements={"docs/engineering/api-examples.md": "# API Examples\n" + ("padding\n" * 200)}, + replacements={"spec/api/api-examples.md": "# API Examples\n" + ("padding\n" * 200)}, ) with pytest.raises(AssertionError, match="api-examples"): @@ -375,7 +418,7 @@ def test_verify_sdist_rejects_stale_benchmark_doc(tmp_path: Path) -> None: sdist = tmp_path / "xy-0.0.1.tar.gz" _write_sdist( sdist, - replacements={"docs/engineering/benchmark.md": "# Benchmark\n" + ("padding\n" * 200)}, + replacements={"spec/benchmarks/results.md": "# Benchmark\n" + ("padding\n" * 200)}, ) with pytest.raises(AssertionError, match="benchmark"): @@ -395,8 +438,7 @@ def test_verify_sdist_rejects_stale_production_readiness_doc(tmp_path: Path) -> _write_sdist( sdist, replacements={ - "docs/engineering/production-readiness.md": "# Production Readiness\n" - + ("padding\n" * 200) + "spec/process/production-readiness.md": "# Production Readiness\n" + ("padding\n" * 200) }, ) @@ -408,7 +450,7 @@ def test_verify_sdist_rejects_stale_contributing_doc(tmp_path: Path) -> None: sdist = tmp_path / "xy-0.0.1.tar.gz" _write_sdist( sdist, - replacements={"docs/engineering/contributing.md": "# Contributing\n" + ("padding\n" * 200)}, + replacements={"spec/process/contributing.md": "# Contributing\n" + ("padding\n" * 200)}, ) with pytest.raises(AssertionError, match="contributing"): diff --git a/tests/test_verify_wheel.py b/tests/test_verify_wheel.py index 39bd4240..98ef5323 100644 --- a/tests/test_verify_wheel.py +++ b/tests/test_verify_wheel.py @@ -394,7 +394,7 @@ def test_verify_wheel_rejects_unexpected_native_artifact(tmp_path: Path) -> None @pytest.mark.parametrize( "extra_name", [ - "docs/engineering/api-examples.md", + "spec/api/api-examples.md", "tests/test_docs_examples.py", "benchmarks/bench_vs.py", "examples/reflex/reflex_xy_app/reflex_xy_app.py",