diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efc7653..ac0e9e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,11 @@ jobs: - name: Dashboard-controls apply + slice-id stability if: ${{ !cancelled() }} run: python tools/ci_live_check.py --base-url http://localhost:8098 --spec tests/fixtures/live_dashboard_controls.json + # A waterfall bridge on a virtual dataset made for it: built and read back on + # 6.1.0; refused before 6.1.0, where the plain waterfalls are built instead. + - name: Waterfall bridge apply + round trip + if: ${{ !cancelled() }} + run: python tools/ci_live_waterfall.py --base-url http://localhost:8098 --spec tests/fixtures/live_waterfall.json # A chart added in Superset: re-apply takes it off the dashboard (4.1.4 and 5.0.0 # merge chart links on import), never deletes it, and restores match their backups. - name: UI-added chart taken off on re-apply @@ -86,6 +91,16 @@ jobs: - name: Locked text visible on the rendered dashboard if: ${{ !cancelled() && steps.visual.outcome == 'success' }} run: python tools/ci_live_visible.py --base-url http://localhost:8098 + # Markdown blocks at the height advise --fix gives them show every line with + # Windows-style scrollbars on, and a block size.markdown-fit calls cut off is. + - name: Markdown blocks fit at the height advise gives them + if: ${{ !cancelled() && steps.visual.outcome == 'success' }} + run: python tools/ci_live_markdown_fit.py --base-url http://localhost:8098 + # The type sizes the readability rules model (table and pivot cells, titles, + # ECharts labels, big numbers by height) are what this release draws. + - name: Type sizes match the readability model + if: ${{ !cancelled() && steps.visual.outcome == 'success' }} + run: python tools/record_readability_measurements.py --check --base-url http://localhost:8098 # Saved queries captured from Superset's own frontend, so CSV reports work. - name: Chart queries saved for reports if: ${{ !cancelled() && steps.visual.outcome == 'success' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 193e8a9..433573b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,140 @@ while the major version is 0, minor bumps may include breaking changes and say s ## Unreleased +### Upgrading + +- The design brain is version 8. `advise` now flags a table whose height can't show every row its `row_limit` allows, as the smoke check after `apply` does when the data fills it; before, it passed a table that showed half. A `--design strict` or `advise --strict` gate can newly fail on such a table. To keep the old threshold for tables that rarely reach their `row_limit`, set `table_visible_ratio: 0.5` in `design.yaml` or a standards file. +- The design brain is version 11. `advise` now warns when a markdown block's text runs past the block's bottom edge (`size.markdown-fit`), so a `--design strict` or `advise --strict` gate can newly fail on such a block; `advise --fix` raises it. `layout.markdown-height` now fixes a one-line block to the height its line takes, 1.6 units for a line of text up to 2.4 for a `#` heading, instead of 2. +- A spec with `conditional_formatting` on a big number is refused by chartwright 0.5.0 and earlier (`extra_forbidden`); upgrade CI before committing one. +- The design brain is version 14. Its new warn rule, `chart.color-contrast`, can fail `advise --strict` and `--design strict` on a spec that passed before: a colour rule painting text or a trendline too pale to read. + +### Added + +- `advise` asks every axis chart to name its axes (design brain 17). `chart.axis-titles` notes a chart with no `y_axis_title` (the measure and its unit) or no `x_axis_title` (the dimension); Superset draws neither unless the chart sets it, and a screenshot or export can't show a tooltip. The heatmap, which has no title setting, and the waterfall, whose steps name themselves, are left out. The brief now tells authors to title every axis. + +- `layout.thin-subtabs` notes a sub-tab holding two charts or fewer besides its KPIs: readers click for little, and the sub-tabs read better as sections of one page under header rows. + +- The value-label design default now covers grouped bars: a bar chart with several metrics side by side gets `show_value` when its bars (rows times metrics) fit the same 12-bar and spacing limits as a single series. Stacked metrics and grouped series stay unlabelled. + +- `advise` checks that each markdown block is tall enough for its text (`size.markdown-fit`). Superset cuts the rest off inside the block: macOS shows no scrollbar there, so a screenshot shows the text stopping mid-line. The check estimates the text's height from its headings, paragraphs, lists, tables, quotes, code and line breaks, the block's width and the dashboard's own CSS, in a window 1440 px wide, measured against what 4.1.4, 5.0.0 and 6.1.0 draw. A wider window wraps fewer lines, so a height that fits there fits wider too. + - It warns when letters are cut off on every release and says by how much ("it needs ~16 units at 12/12 in a 1440 px wide window, has 15"); `--fix` raises the block to a height that shows every line on every release, Windows scrollbars included. + - A block whose last line or padding only reaches the edge gets an info: it scrolls a few px, and Windows draws a scrollbar in it. That covers short strips: a line of text needs 1.6 units, a `##` heading 2.2. + - It reads the text sizes and spacing a dashboard's CSS gives its markdown: font sizes, line heights, weights, letter and word spacing, margins, padding and borders set on `.dashboard-markdown` elements, `h4 + p` and `p:last-child` rules included, in cascade order over Superset's own. CSS that could change a block's height in a way it doesn't read (another font family, attribute selectors and pseudo-elements, `@media` rules) is named in the finding, as is a theme. + - It checks blocks in layout rows, tabs and sub-tabs, and header and footer rows, and allows for the filter bar, which narrows the page on a dashboard with native filters. A header or footer row a standard owns is reported, never resized, and a block holding an image gets no fix, since its height is unknown. +- `conditional_formatting` on `big_number_total`: colour the number by its value, such as a balance-closure KPI in red outside ±2% and green inside (`[{"operator": "<", "target": -0.02, "color": "red"}, {"operator": ">", "target": 0.02, "color": "red"}]`). The rules take the operators, thresholds and colours of table and pivot rules; a named colour takes its darker text shade, since the number is text. Superset 4.1.4, 5.0.0 and 6.1.0 all colour it, so no release is refused or warned. Decompile reads the rules back, and `plan` reports a rule changed in Superset. + - When several rules match, the last one's colour wins, as in Superset. + - `apply_to`, `paint` and a `metric` other than the chart's own are refused on a big number's rule, and so is `conditional_formatting` on `big_number_trend`, which Superset can't colour (its `trend_color` colours the line). + - Superset never colours a value of exactly 0; apply's data check warns when a coloured big number is 0 and a rule would have coloured it. +- Design brain 14: + - `chart.color-contrast` (warn): a colour too pale to read on the white card, below 3:1 for a big number's rule or a trendline and below 4.5:1 for a table or pivot text rule. + - `narrative.kpi-thresholds` (info): a coloured big number whose subtitle and description don't state its thresholds. + - `chart.format-bands` checks a big number's rules for overlapping bands. A single band on a big number is fine: colouring only the exceptions is enough. +- Table cell bars per column. `cell_bars` takes a list of labels as well as true or false: `"cell_bars": ["Revenue", "Change"]` draws bars on those columns only. `color_by_sign` colours bars by sign, green above zero and red below (Superset 6.0+; 4.1.4 and 5.0.0 colour only the negative bars, red), for the whole table or the labels listed, so a change shows its sign while revenue bars stay neutral; `"color_by_sign": false` turns off the sign colours Superset draws by default. `absolute_bars` sizes bars by absolute value from the cell's left edge (Superset's "Align +/-"), so a rise and a fall of the same size draw the same bar. Each compiles to Superset's own table-wide and per-column settings; `decompile` reads any mix of them back as Superset draws it, and `plan` reports a change made in the UI. chartwright 0.5.0 and earlier refuse a spec that uses them. +- `check`, `apply` and `plan` warn when a table asks for cell bars beside colour rules on Superset before 6.1.0, which draws no cell bar on a table with any colour rule. +- Colour rules on tables, pivots and big numbers take `>=`, `<=`, `!=` and `between_inclusive` (a range with its bounds taken in), besides `<`, `>`, `=` and `between`. Every supported release has these comparators, so they draw the same everywhere. `decompile` reads them back from rules made in Superset, where before they were dropped as losses. +- `chartwright advise` checks type sizes against floors for reading a dashboard on a laptop (design brain 16): `readability.table-text` (table and pivot cells 14 px, their headers 12 px, rows 24 px tall), `readability.chart-text` (axis labels and legends 12 px, chart titles 14 px) and `readability.kpi-text` (a big number's value 24 px, its subtitle or comparison 12 px). Text Superset draws smaller by default, such as its 12 px table cells, is an info naming the CSS selector or theme token to set; text the dashboard's own CSS or theme sets smaller is a warn, so `--design strict` can newly block. Superset sizes a big number's text by the card's height, so `advise --fix` raises such a card, and the cards beside it, to the height where its text reads: a card with a subtitle to 5 units, a trendline card with a comparison to 6. The floors are design parameters (`min_cell_text_px` and four more) that `design.yaml` and a standard can tune or lock. `check` and `advise --profile` read a named theme's tokens and ECharts overrides. + +### Fixed + +- A horizontal bar chart's axis titles now sit where they belong. Superset lays the titles out for a vertical chart and then swaps the axes, so the value-axis title ran past the right end of the bottom axis and was cut off, and the category title sat on top of the category labels. The value title is now centred under the bottom axis, and the category title clears labels of about 8 characters. + +- `advise` and the smoke check after `apply` now ask the same height question of tables and pivots, so they no longer disagree: + - a table: advise names the height smoke names once the data fills `row_limit` (a 25-row table at 13 units now gets "raise height to ~21" from both); + - a pivot: `size.pivot-window` no longer reads `row_limit` as the pivot's rows. `row_limit` caps the query's records, which are rows times columns, so a 5-row cause by month pivot was told it showed "~0 of 1000 rows" and to grow to 331 units. Offline, the rule now checks only what the spec fixes (its header rows, totals row and first row) and says so; `advise --profile` counts each row dimension's values and gives the height smoke gives; + - messages name what takes the room: the card frame, each header row, the totals row and room for a horizontal scrollbar. +- Height estimates count what a pivot draws in every layout: a transposed pivot is sized by its column dimension, metrics laid out as rows multiply its rows, `row_subtotals` adds its subtotal rows, and a pivot with no row dimension is its totals row. A table's `show_totals` row now takes a row of height too, in the checks and in the page size `advise --fix` fills. +- The smoke check queries a table or pivot without a `row_limit` for as many rows as the chart does (10,000 for a pivot, 1,000 for a table), so a large pivot's height check counts all its rows. +- `apply`'s data check queries each chart the way the dashboard does when it loads: with the default of every native filter in the chart's scope. Before, it queried every chart unfiltered, so a table holding one set of rows per value of a required filter counted every set; the check then asked for more height on charts that fit, and could name a chart empty, or not, wrongly. As in Superset, a filter's default reaches a chart only when the chart is in the filter's scope and the chart's dataset has a column of that name. + - Applied: a select's default values, or with `default_to_first` its first value, which the check reads with the query Superset's filter sends (its pre-filter and the defaults of the filters it depends on included); a range's bounds; a time range, on the chart's time axis or main time column; a time grain, on the chart's time axis. + - Each chart's result names the defaults it ran with (`with the dashboard's filter defaults: Carrier = All carriers`), and lists them under `filter_defaults` in the apply report, present only when there are some. A default the check can't apply is named as not applied: a time column filter's, or a first value it could not read. + - A chart the dashboard shows empty on load, such as one under a default time range the data doesn't reach, is now named empty. +- `decompile` (and so `adopt`) no longer reports the values an untouched table stores as settings it can't carry: HTML rendered, columns fixed in place, paging in the browser, no percentage metrics, and bars coloured by sign. A table saved in Superset reads back without a loss; each of these is still named when changed. +- `check`, `apply` and `plan` (and the MCP tools) warn when a table or pivot uses something its Superset release draws differently. Before, the dashboard came out different from the spec with no word. On 4.1.4, 5.0.0 and 6.0.0: + - a table rule's `apply_to` paints the rule's own column instead; + - `paint: "text"` fills the cell with the dark text shade instead, under the cell's own dark text, so the value is hard to read; + - each band fades by the value's distance from its threshold instead of one solid colour (an `=` rule stays solid); + - a dark `color` fill keeps the cell's dark text, where 6.1 turns it white. + + On 4.1.4 and 5.0.0, a table's `hidden` columns show. Each warning says what that release draws. +- `decompile` names a colour rule that Superset 6.1 fades by distance (its rule editor turns the gradient on by default). The spec paints a rule one solid colour, so the next apply changes how it looks. +- The design brain is version 15: `chart.format-bands` reads the bounds of the new operators, so `>= 80` and `<= 80` share 80 and `> 80` and `<= 80` share nothing. +### Added + +- A sketch can hold section headers and notes. A legend letter stands for a chart's name as before, or for a header (`{"header": "Monthly sales", "size": "large"}`) or a markdown block (`{"markdown": "Source: the ledger", "height": 1.6}`), drawn like a chart. A header is one line: drawn across the whole sketch it titles the band below it, as a header row does in `rows`; drawn above or below charts it sits in their column, and alone in a narrower slot it gets a column of its own. A markdown block takes its drawn width and height, or its own `height` in fifths of a unit. `decompile` (and so `adopt` and `plan`) reads headers and text blocks in a section of stacked charts back into its sketch, where it used to flatten the columns. Specs without blocks compile to the same bytes. See [the layout guide](docs/LAYOUT-GUIDE.md#section-headers-and-notes). + +### Fixed + +- `decompile` (and so `adopt`) reads columns of stacked charts side by side, such as two columns of two charts each, back as one row of columns. Before, the sketch it drew for them split into one row per level of the stacks, so a copy laid them out as rows. +### Added + +- A waterfall chart (`"type": "waterfall"`, Superset's Waterfall): steps that add up to a running total, such as a bridge from last year's revenue through each driver to this year's. Compiled, decompiled, compared by `plan` and checked by `apply` on 4.1.4, 5.0.0 and 6.1.0. + - `x_column` holds the steps (or a time column, one bar per `time_grain` period), `metric` the one measure they add, and `groupby` an optional breakdown per x value. + - `increase_color`, `decrease_color` and `total_color` take green, amber, red or any `#RRGGBB`, and replace Superset's stock green, red and grey on every release. `show_value`, `show_legend`, `number_format`, axis titles, `x_label_rotation` (0, 45 or 90) and `x_label_format` set the rest. + - On Superset 6.1.0 or later, `steps` lists a bridge's values in its own order and `closing` names the row that closes it, drawn last as the running total in the total colour: `"steps": ["FY2025", "Price", "Volume"], "closing": "FY2026"`. The first step rises from zero; `"opening": "FY2025"` draws the opening row as a total too, on a dashboard whose theme keeps zero on the value axis (`{"echartsOptionsOverridesByChartType": {"waterfall": {"yAxis": {"scale": false}}}}` in its JSON), which `check` and `apply` verify (`waterfall_opening_axis`). Older releases draw a running total after every step, so `check`, `apply` and `plan` refuse `steps` there. `total_label`, `increase_label` and `decrease_label` name the bars on 6.1.0 and warn before it. + - `apply`'s data check says when a bridge's opening, step or closing has no row, a value isn't in `steps`, or the closing row doesn't equal the steps added up (Superset draws the sum either way). +- A box plot (`"type": "box_plot"`, Superset's Box Plot): a measure's distribution in each group, its median, quartiles, whiskers and outliers, such as the daily sales of each month. Compiled, decompiled, compared by `plan` and checked by `apply` on 4.1.4, 5.0.0 and 6.1.0. + - `distribute_across` holds the columns whose rows are the observations (a time column at `time_grain`, e.g. a day each), `groupby` the columns that get a box each, `metrics` the measure. + - `whiskers` is `"tukey"` (Superset's default), `"min_max"` (no outliers) or two percentiles such as `[5, 95]`, which every release takes. + - `number_format`, `x_label_format`, axis titles, `x_label_rotation` and `color_scheme` set the rest; `row_limit` is a Superset 6.0.0 control, and warns before it, where a dashboard's query still stops at it but Explore leaves it out. + - `apply`'s data check says when the observations reach the row limit (10,000 unset on Superset 6.x), so the boxes miss some. +- The design brain is version 9. A waterfall gets `chart.waterfall-additive` (warn: `AVG`, `MIN`, `MAX` or `COUNT_DISTINCT` steps don't add up), `chart.waterfall-steps` (warn: past ~12 bars), `chart.waterfall-order` (info: a categorical waterfall without `steps` reads A to Z), `chart.waterfall-colors` (info for Superset's stock colours, whose green is 2.2:1 against the panel; warn for a colour under 3:1) and `chart.waterfall-axis-titles` (info: Superset puts the titles over wide or rotated tick labels). `advise --fix` fills a waterfall's `show_value` when it has at most 12 bars, and `number_format` `,.0f` on a count. A box plot gets `chart.box-plot-observations` (warn: grouped by a period at that period's grain, each box holds a few observations) and `chart.box-plot-groups` (warn, with `--profile`: past ~20 boxes). `chart.ordinal-order` reads a box plot's groups, and with `--profile` stands down for a column the dataset reports numeric or temporal. The brief's chart-choice guide says when a waterfall is the right form, and when a box plot beats a line of P50 and P90. +- `tools/extract_panel_contract.py` extracts the waterfall's and the box plot's options for the params contract from their control panels at each release; `tools/ci_live_waterfall.py` builds a bridge on a virtual dataset in CI. + +### Fixed + +- `decompile` reads Superset's stored defaults for data zoom and the series limit (`zoomable` false on time-axis and mixed charts, no `series_limit` on a pivot) as unset; before, `params not preserved` named them on charts saved in Explore. A zoomable chart or a series limit is still named. +- `plan` reads a `trend_color` written as the hex of a named shade (`#1B7F3B` for green) as equal to the dashboard, which decompile reads back as the name; before, the chart showed as changed after every apply. The waterfall's colours compare the same way. +- `plan` is clean straight after `apply` for a chart height in fifths of a unit and for an aggregate written as custom SQL. A height of 4.6 (as `absorb` writes after a resize in the UI) read back rounded to 5, and `SQL(MAX(col)) AS Label` read back as `MAX(col) AS Label`, so every such chart showed as changed after each apply. `decompile` now reads both back as written; custom SQL typed in Superset's metric popover still reads back as `MAX(col) AS Label`. An aggregate written with spaces (`SUM( qty )`, which is also the label Superset shows) reads back as written too, so a table's settings keyed by that label (number format, header, alignment, a rule painting it) are no longer dropped. + +- `plan` reads alike the spellings `apply` stores alike, so none of them shows as a change after apply: Superset's own defaults written out (`"time_range": "No filter"`, `"number_format": "SMART_NUMBER"`, `"x_label_format": "smart_date"`, `"x_label_rotation": 0`, a table's `"date_format": "smart_date"`), `COUNT(*) AS N` and `SQL(COUNT(*)) AS N`, a named colour's hex (`"#1B7F3B"` for a green trendline, `"#ACE1C4"` for a green colour rule on a table or pivot), a height between fifths, an empty `metrics` or `groupby` list on a table, a filter's pre-filter time range of `"No filter"`, and blank text. `decompile` leaves a big number's `SMART_NUMBER` format out, as it does on every other chart. +- `plan` compares a table's `hidden` labels as a set. Decompile reads them back in the order Superset stores them, not the spec's, so a table hiding two or more columns showed as changed after every apply. + +### Added + +- `sort_by` on a categorical bar ranks it by something other than its first metric, largest first, on every supported release. `"total"` ranks stacked or grouped bars by their sum. Before, Superset drew several series in name order: two stacked metrics on 4.1.4 and 5.0.0, and a grouped bar on every release. A metric (`"sort_by": "SUM(revenue)"`) ranks by its value and orders the query too, so a `row_limit` keeps the top bars by it; it need not be drawn, so two charts can share one order. `advise` warns when a `row_limit` cuts a bar ranked by `"total"`, since the query is still ordered by the first metric. +- A threshold on a horizontal bar: `annotations` there now say what Superset draws, a line standing upright at its value (a 1.0x threshold, a 4-hour limit), seen on every supported release, and the live CI fixture builds one. +- Heatmap axes: `x_label_every` and `y_label_every` label every Nth column or row, counted from the first, so an hour axis with `"x_label_every": 6` reads 0, 6, 12, 18 instead of Superset's uneven automatic spacing. `left_margin` leaves room, in px, left of the row labels. Every supported release takes all three; `decompile` reads them back and `plan` reports a change made in the UI. + +### Changed + +- Documented: no supported release gives one line of a mixed chart its own width or dash. The plugin has no such control and its transform passes none, and 6.1's ECharts Options replace the series list whole. Set a ghost line apart by colour with `dashboard.label_colors`, or draw a fixed level as an annotation. +- Design brain 12: `advise --fix` fills `left_margin: 16` on every heatmap (`default.heatmap-label-room`). On Superset 6.1.0 a heatmap's longest row label lost its first letters at the card's edge, and a theme can't reach the heatmap there. Write `left_margin` yourself, or ignore the rule for a chart, to keep your own value. + ### Fixed +- `decompile` names a bar ranked by its series' minimum, maximum or average, or ranked smallest first; before, apply quietly put the largest-first ranking back. - `decompile` (and so `adopt`) names more chart settings it can't carry when they differ from Superset's defaults: big-number font sizes and a hidden trendline, a smooth or step line, horizontal bars on a time axis, a pie's radius and hidden labels, a funnel's labels, tooltip and percentage calculation, treemap labels, a heatmap's axis sort, legend, margins, label intervals and value bounds, a histogram's normalize setting, and any chart's currency format. Before, they were dropped without a note. +### Upgrading from 0.5 + +- A spec with a mixed chart's `kind: "area"`, a trendline's `rolling_type` or `y_axis_truncate`, a big number's `date_format` or a heatmap's `x_order` or `y_order` is refused by chartwright 0.5.0 and earlier (`literal_error` or `extra_forbidden`); upgrade CI before committing one. +- The design brain is version 10. `data.rolling-window-span` is a new warn-level rule, so `--design strict` can newly block a rolling trendline KPI whose time range can't hold its window. `advise --fix` fills `compare_suffix` as "vs prior 12 months" where a 12-step rolling window compares with the 12 steps before, and, with column types (`--profile`), `date_format` on a big number of a date column. It also removes a search box it filled on a table whose rows all show (below), so a spec it fixed before shows that change in its diff. + +### Added + +- Filled areas on a mixed chart: `"kind": "area"` on query `a` or `b`, with `opacity` for the fill (0 to 1; Superset's default is 0.2). A line draws over an area whichever query holds it, and at opacity 1 the area's edge disappears into its fill, so solar output under a net-load line reads as one solid shape; give it a light colour with `label_colors`. `decompile` reads Superset's "Area chart" box back as `kind: "area"`. +- A rolling window on a trendline KPI: `rolling_type` (`sum`, `mean`, `std`, or `cumsum` for a running total) with `rolling_periods`, the window in time-grain steps. At P1M, `"rolling_type": "sum", "rolling_periods": 12` shows a trailing-12-month total, and `compare_lag: 12` compares it with the 12 months before. Every point is a whole window; set `rolling_min_periods` to also draw the partial windows of the first steps. + - `advise` warns when the chart's time range, or a defaulted dashboard time filter it loads with, is too short to hold the window and the comparison (`data.rolling-window-span`), and apply's data check warns from the time buckets the chart really has. +- A big number shown as a date: `"date_format": "%a %-d %b %Y"` on a `big_number_total` shows `MAX(updated_at)` as "Sat 3 Oct 2026", for a tile that says how fresh the data is. It works on a date or timestamp metric and on a number of epoch milliseconds, and replaces `number_format`. With column types (`advise --profile`), `advise --fix` fills it on a big number of a date column's `MIN` or `MAX`, which Superset shows by default as its day alone ("Tue 31"). +- Heatmap axis order: `x_order` (left to right) and `y_order` (top to bottom), each `a_to_z`, `z_to_a`, `value_asc` or `value_desc`. `"y_order": "a_to_z"` puts the first label on top, such as a cohort triangle's oldest cohort; omitted, the labels keep running A to Z from the bottom up. On Superset 6.1.0 a value order ranks each label by its total; older releases order by the cells, so `check` warns there. `decompile` reads the order back. +- A trendline KPI fitted to its values: `"y_axis_truncate": true` draws the trendline from its lowest value to its highest instead of up from zero, so a trailing-12-month total that moves a few percent shows the movement rather than a flat line over a solid block. `decompile` reads Superset's unticked "Start y-axis at 0" back as it. +- Apply's data check names the values a chart's `IN` filter lists that return no rows, on a column the chart groups or draws by: a line chart that lists four zones where two have data now says which two it draws nothing for. A series limit that drops values on purpose, or a time column, is left alone. + +### Changed + +- The design brain adds no page-size picker, pager or search box to a table whose rows all show in its panel. `advise --fix` now fills a search box only on a raw table that pages, where the box sits in the bar the page-size picker already draws, and removes one it filled on a table whose rows all show. `advise` names a `page_length` or `search_box` you wrote on such a table (`size.table-chrome`, info), since Superset draws a page-size picker for any page size, even over one page; set `page_length` to 0 or leave it out to show every row with no picker. + +### Fixed + +- `decompile` (and so `adopt`) names more chart settings it can't carry when they differ from Superset's defaults: big-number font sizes and a hidden trendline, a smooth or step line, horizontal bars on a time axis, a pie's radius and hidden labels, a funnel's labels, tooltip and percentage calculation, treemap labels, a heatmap's legend, margins, label intervals and value bounds, a histogram's normalize setting, and any chart's currency format. Before, they were dropped without a note. + +- `decompile` (and so `adopt` and `plan`) no longer lists a mixed chart's untouched query B settings (marker size, time comparison type, metric truncation, rolling window, series sort), which a chart saved in Superset stores beside query A's, as settings it can't carry, nor a trendline KPI's untouched Force date format. It reads the older `echarts_timeseries_line` and `echarts_timeseries_bar` series types, which Superset draws as a straight line, as `kind: "line"`, with no loss. - `standards apply` no longer adds a header or footer row that the body already holds at its edge (the last rows for a footer, the first for a header). A dashboard built in the UI has no header or footer, so `decompile` and `adopt` read its legal line as a body row, and the standard's footer was then added below it, showing it twice. `--claim` moves such a row into the header or footer and records it; `explain` and the apply summary name it. +- `chart.format-bands` read an `=` rule as covering every value above its target, so a pivot with `= 1` in red and `> 5` in green was reported as overlapping bands. + +- `decompile` (and so `adopt`) no longer reports colour rules kept on a trendline KPI that was switched from a Big Number in Superset as a loss: the trendline never draws them. + ## 0.5.0 (2026-10-04) Take over a dashboard built in the UI where it is, with everything the first apply resets listed up front. Save each chart's query so Superset's CSV reports work on the charts the tool builds. Decompile names the settings it can't carry and keeps stacked charts; backups record their instance, prune themselves and clean up after a restore. diff --git a/README.md b/README.md index 5e5f35e..f79756b 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ as-is. automatically. Dashboards Chartwright did not create are never overwritten; to bring a hand-built one under a spec, decompile it and build it at a new slug. -- **What a spec can express**: 15 chart types, metrics as you write them, +- **What a spec can express**: 17 chart types, metrics as you write them, per-chart filters, a native filter bar, tabs, markdown notes, a header and footer shown on every tab, and layouts you can draw as ASCII sketches. - **Environment promotion**: specs name their data (connection, schema, diff --git a/chartwright/absorb.py b/chartwright/absorb.py index 4c7d8df..22d298c 100644 --- a/chartwright/absorb.py +++ b/chartwright/absorb.py @@ -22,7 +22,7 @@ import json from dataclasses import asdict, dataclass, field -from .compiler import ROW_UNITS_PER_SPEC_UNIT, _position +from .compiler import _position, spec_units from .spec import DashboardSpec @@ -38,12 +38,6 @@ def to_json(self) -> str: return json.dumps({"stage": "absorb", **asdict(self)}, indent=2) -def _exact_units(row_units: int) -> float | int: - """Superset row units (8px) -> spec units (40px), exactly representable.""" - units = row_units / ROW_UNITS_PER_SPEC_UNIT - return int(units) if units.is_integer() else round(units, 1) - - def absorb_heights(spec: DashboardSpec, spec_data: dict, live_position: dict) -> tuple[dict, AbsorbReport]: """Pure core: returns (patched copy of spec_data, report). spec_data is the raw JSON the spec file holds; only charts[*].height values are touched.""" @@ -82,7 +76,7 @@ def absorb_heights(spec: DashboardSpec, spec_data: dict, live_position: dict) -> live_h = have.get("height") if not isinstance(live_h, int) or live_h == want["height"]: continue - charts_by_name[chart.name]["height"] = _exact_units(live_h) + charts_by_name[chart.name]["height"] = spec_units(live_h) report.absorbed.append({ "chart": chart.name, "from_px": want["height"] * 8, diff --git a/chartwright/apply.py b/chartwright/apply.py index b703fa2..9570467 100644 --- a/chartwright/apply.py +++ b/chartwright/apply.py @@ -785,7 +785,7 @@ def _auto_restore(reason: str) -> None: report.stage = "smoke" results: list[SmokeResult] = smoke(spec, resolution, client) - report.smoke_results = [asdict(s) for s in results] + report.smoke_results = [s.as_dict() for s in results] report.warnings.extend(f"{s.chart}: {s.detail}" for s in results if s.warning) if any(not s.ok for s in results): return report diff --git a/chartwright/client.py b/chartwright/client.py index 98de5f5..cdcda14 100644 --- a/chartwright/client.py +++ b/chartwright/client.py @@ -278,16 +278,19 @@ def delete_chart(self, chart_id: int) -> None: # -- themes (6.0.0+) ---------------------------------------------------- def themes(self) -> list[dict]: - """Every theme on the instance: id, uuid and theme_name, from the theme list API - (ThemeRestApi, resource "theme", superset/themes/api.py at 6.0.0 and 6.1.0). - Themes are few, so all pages are read and names matched here: theme_name has no - unique constraint (models/core.py class Theme), and the API's only name search is - a substring match over name and JSON alike (themes/filters.py ThemeAllTextFilter).""" + """Every theme on the instance: id, uuid, theme_name and json_data (the tokens the + design brain's readability rules read), from the theme list API (ThemeRestApi, + resource "theme", superset/themes/api.py at 6.0.0 and 6.1.0, whose list_columns + hold json_data, :110-120). Themes are few, so all pages are read and names matched + here: theme_name has no unique constraint (models/core.py class Theme), and the + API's only name search is a substring match over name and JSON alike + (themes/filters.py ThemeAllTextFilter).""" out: list[dict] = [] page = 0 while page < 100: # 10,000 themes: a runaway guard, not an expected ceiling rows = self.get("/api/v1/theme/", q={ - "columns": ["id", "uuid", "theme_name"], "page": page, "page_size": 100, + "columns": ["id", "uuid", "theme_name", "json_data"], "page": page, + "page_size": 100, })["result"] out += rows if len(rows) < 100: diff --git a/chartwright/compiler.py b/chartwright/compiler.py index 5d8f634..431cd13 100644 --- a/chartwright/compiler.py +++ b/chartwright/compiler.py @@ -10,6 +10,7 @@ import io import json +import re import uuid import zipfile from decimal import Decimal @@ -19,11 +20,16 @@ from . import ids from .resolver import Resolution from .spec import ( + BRIDGE_TOTAL, DEFAULT_ROW_LIMIT, DEFAULT_TIME_GRAIN, + FORMAT_RANGE_OPERATORS, FUNNEL_LABEL_TYPES, HEATMAP_DEFAULT_SCHEME, + OPENING_KEY_GAP, + OPENING_OTHER_KEY, PIVOT_ORDER, + TICK_LAYOUTS, DashboardSpec, DividerBlock, HeaderBlock, @@ -31,6 +37,9 @@ _AxisChart, _ColorSchemeMixin, _SeriesDisplay, + hex_to_rgb, + metric_label, + named_hex, parse_metric, row_items, without_superset_defaults, @@ -43,6 +52,20 @@ # ponytail: 1 spec grid unit -> 5 superset row units (1 row unit ~ 8px). # Calibration knob; validated against rendered dashboards. ROW_UNITS_PER_SPEC_UNIT = 5 + + +def grid_rows(height: float) -> int: + """Spec height units (40 px) -> the whole Superset grid rows (8 px) compile writes.""" + return int(round(height * ROW_UNITS_PER_SPEC_UNIT)) + + +def spec_units(rows: int) -> float | int: + """Superset grid rows -> spec height units, exactly: a whole unit stays an int, any + other is a fifth (one decimal), so grid_rows(spec_units(n)) == n.""" + units = rows / ROW_UNITS_PER_SPEC_UNIT + return int(units) if units.is_integer() else round(units, 1) + + FOOTER_PREFIX = "sdc-footer-" # layout.footer rows compile to ROW-sdc-footer-; decompile keys on it HEADER_PREFIX = "sdc-header-" # layout.header rows compile to ROW-sdc-header-; decompile keys on it @@ -62,6 +85,8 @@ "funnel": "funnel", "treemap": "treemap_v2", "mixed": "mixed_timeseries", + "waterfall": "waterfall", + "box_plot": "box_plot", } # 'bar' and 'timeseries_bar' share a viz_type; the compiler marks categorical # bars in params so the decompiler can tell them apart (x column not temporal @@ -73,11 +98,19 @@ def _yaml(data: dict) -> bytes: return yaml.safe_dump(data, sort_keys=True, default_flow_style=False, allow_unicode=True).encode() +# Every ad-hoc metric the compiler writes carries this optionName prefix; Superset's +# metric popover names its own metric__ and keeps a stored one +# (src/explore/components/controls/MetricControl/AdhocMetric.js:84-88 at 4.1.4 and +# 5.0.0, AdhocMetric.ts:122-126 at 6.1.0), so decompile can tell the tool's metrics +# from ones built in the UI. +METRIC_OPTION_PREFIX = "metric_sdc_" + + def _metric_payload(metric: str, slug: str, chart_name: str) -> str | dict: adhoc = parse_metric(metric) if adhoc is None: return metric - option = "metric_sdc_" + uuid.uuid5(ids.NAMESPACE, f"{slug}/chart/{chart_name}/metric/{metric}").hex[:12] + option = METRIC_OPTION_PREFIX + uuid.uuid5(ids.NAMESPACE, f"{slug}/chart/{chart_name}/metric/{metric}").hex[:12] label = adhoc["label"] or metric if adhoc.get("sql") is not None: # Custom SQL (the metric popover's "Custom SQL" tab), in every release. @@ -106,32 +139,53 @@ def _metric_payload(metric: str, slug: str, chart_name: str) -> str | dict: } -# Spec operator -> Superset's Comparator value. A range is '< x <' in every -# supported release (types.ts at 4.1.4/5.0.0/6.1.0); the literal "between" -# matches no comparator, and getColorFormatters' default case colours nothing. -FORMAT_OPERATOR = {"<": "<", ">": ">", "=": "=", "between": "< x <"} +# Spec operator -> Superset's Comparator value (superset-ui-chart-controls types.ts: +# 4.1.4 :434-446, 6.1.0 :456-467). A range is '< x <' in every supported release; the +# literal "between" matches no comparator, and getColorFormatters' default case colours +# nothing. Each one has its case in getColorFunction at every release (getColorFormatters.ts +# '≥' 4.1.4/5.0.0 :113, 6.0.0 :122, 6.1.0 :141; '≤ x ≤' :153, :162, :191) and its entry +# in the rule popover (FormattingPopoverContent.tsx:55-60 at 4.1.4, constants.ts:26-31 at 6.1.0). +FORMAT_OPERATOR = {"<": "<", ">": ">", "=": "=", ">=": "≥", "<=": "≤", "!=": "≠", + "between": "< x <", "between_inclusive": "≤ x ≤"} -def _format_rule_payload(rule) -> dict: +def _band_payload(rule, column: str) -> dict: out = { - "column": rule.metric, + "column": column, "colorScheme": rule.paint_hex(), "operator": FORMAT_OPERATOR[rule.operator], - # A rule is a solid band. Left unset, Superset fades '<' / '>' / range - # colours by distance from the threshold (getColorFormatters.getOpacity); - # useGradient is honoured from 6.x and ignored by 4.1.4/5.0.0. + # A rule is a solid band. Left unset, Superset fades every colour but '='s by + # distance from the threshold (getColorFormatters.getOpacity); useGradient is read + # from 6.1.0 (getColorFormatters.ts:273-276) and ignored by 4.1.4, 5.0.0 and 6.0.0, + # which fade regardless (versions.GATED_FIELDS warns). "useGradient": False, } - if rule.operator == "between": + if rule.operator in FORMAT_RANGE_OPERATORS: out["targetValueLeft"] = rule.target_left out["targetValueRight"] = rule.target_right else: out["targetValue"] = rule.target + return out + + +def _big_number_rule_payload(rule, chart) -> dict: + """A big number's rule, as its panel stores it (BigNumberTotal/controlPanel.ts + conditional_formatting: 4.1.4 and 5.0.0 :94, 6.1.0 :85). The column is the metric's + label, the one numeric column the panel offers; the number is painted from every + rule whose column is set (getColorFormatters.ts 4.1.4 and 5.0.0 :199, 6.1.0 :309) + and always solid, since the plugin asks for no alpha (BigNumberTotal/transformProps.ts + 4.1.4 and 5.0.0 :99, 6.1.0 :121).""" + return _band_payload(rule, metric_label(chart.metric)) + + +def _format_rule_payload(rule) -> dict: + out = _band_payload(rule, rule.metric) if rule.paint == "text": out["objectFormatting"] = "TEXT_COLOR" if rule.apply_to: # The 6.1 table's "apply to": another column's key, or the whole row - # (TableChart.tsx reads columnFormatting; ObjectFormattingEnum.ENTIRE_ROW). + # (TableChart.tsx:984-1007 at 6.1.0 reads columnFormatting; ObjectFormattingEnum.ENTIRE_ROW). + # Older tables paint only rule.column's own cells (TableChart.tsx:763 at 4.1.4). out["columnFormatting"] = "ENTIRE_ROW" if rule.apply_to == "row" else rule.apply_to return out @@ -140,20 +194,43 @@ def _format_rule_payload(rule) -> dict: # 5.0.0 and 6.1.0; customColumnName from 6.0.0, read at TableChart.tsx:806, :859 at 6.1.0). COLUMN_CONFIG_KEYS = {"number_formats": "d3NumberFormat", "column_align": "horizontalAlign", "column_widths": "columnWidth", "column_headers": "customColumnName"} +# The cell-bar switches (spec.BAR_SWITCHES): the key each column stores (TableColumnConfig, +# plugin-chart-table/src/types.ts 4.1.4 and 5.0.0 :47-49, 6.1.0 :99-103) and the table-wide +# control it falls back to (Table controlPanel.tsx, all three releases). +BAR_SWITCH_KEYS = {"cell_bars": "showCellBars", "color_by_sign": "colorPositiveNegative", + "absolute_bars": "alignPositiveNegative"} +BAR_SWITCH_PARAMS = {"cell_bars": "show_cell_bars", "color_by_sign": "color_pn", + "absolute_bars": "align_pn"} def _column_config(chart) -> dict: """Per-column table display: hidden columns, d3 number formats, alignment, minimum - widths and header text, merged per label.""" + widths, header text and the cell-bar switches a list turns on, merged per label.""" cfg: dict = {} for label in chart.hidden: cfg.setdefault(label, {})["visible"] = False for field, key in COLUMN_CONFIG_KEYS.items(): for label, value in getattr(chart, field).items(): cfg.setdefault(label, {})[key] = value + for field, key in BAR_SWITCH_KEYS.items(): + if isinstance(getattr(chart, field), list): + for label in getattr(chart, field): + cfg.setdefault(label, {})[key] = True return cfg +def _bar_switches(chart, p: dict) -> None: + """The table-wide half of each cell-bar switch. A list turns the table's off and its + columns' on (_column_config). A written Superset default (color_by_sign true, + absolute_bars false) arrives unset, so a spec without the switches emits nothing.""" + for field, key in BAR_SWITCH_PARAMS.items(): + value = getattr(chart, field) + if isinstance(value, bool): + p[key] = value + elif value is not None and field != "absolute_bars": + p[key] = False # align_pn is off unless written: its list needs no table-wide key + + def _adhoc_filters(chart) -> list[dict]: return _adhoc_filter_list(chart.filters) @@ -176,6 +253,19 @@ def _adhoc_filter_list(filters) -> list[dict]: return out +# Heatmap axis order -> sort_x_axis / sort_y_axis (sortAxisChoices, Heatmap/controlPanel.tsx +# :26-31 at 4.1.4 and 5.0.0, :28-33 at 6.1.0). Both axes are ECharts category axes, whose +# first label sits at the left and at the BOTTOM (Heatmap/transformProps.ts:227-240 at 4.1.4, +# :431-447 at 6.1.0, no inverse), so the y order, written top to bottom, is reversed here. +# 6.1.0 sorts each axis itself (sortAxisValues, transformProps.ts:88-145); before, the axes +# list labels as the query's ORDER BY returns them (buildQuery.ts:39-48 at 4.1.4, :39-52 at +# 5.0.0 and 6.0.0), which versions.py warns about for a value order. +HEATMAP_X_SORT = {"a_to_z": "alpha_asc", "z_to_a": "alpha_desc", + "value_asc": "value_asc", "value_desc": "value_desc"} +HEATMAP_Y_SORT = {"a_to_z": "alpha_desc", "z_to_a": "alpha_asc", + "value_asc": "value_desc", "value_desc": "value_asc"} + + def _pin_big_number_fonts(p: dict) -> None: """Pin the number and subtitle sizes to the plugin's own documented defaults (proportions of the card). Left unset, 6.1 renders a subtitle fed @@ -209,14 +299,42 @@ def metric(m: str): p["subheader"] = chart.subtitle if chart.number_format: p["y_axis_format"] = chart.number_format + if chart.date_format: + # "Date format" and "Force date format" (BigNumberTotal/controlPanel.ts:66 and + # :80 at 4.1.4 and 5.0.0, :57 and :71 at 6.1.0). transformProps formats the number + # with the time format when the metric is temporal or a string, or when forced + # (transformProps.ts:87-92 at 4.1.4 and 5.0.0, :109-114 at 6.1.0); forced, a + # number of epoch milliseconds shows as a date too. + p["time_format"] = chart.date_format + p["force_timestamp_formatting"] = True _pin_big_number_fonts(p) + if chart.conditional_formatting: + p["conditional_formatting"] = [ + _big_number_rule_payload(r, chart) for r in chart.conditional_formatting + ] elif t == "big_number_trend": p["metric"] = metric(chart.metric) p["x_axis"] = chart.time_column p["time_grain_sqla"] = chart.time_grain or DEFAULT_TIME_GRAIN p["show_trend_line"] = True - p["start_y_axis_at_zero"] = True - p["rolling_type"] = "None" + # "Start y-axis at 0" (BigNumberWithTrendline/controlPanel.tsx:96 at 4.1.4 and 5.0.0, + # :107 at 6.1.0, default true), drawn as the trendline's yAxis scale: + # !startYAxisAtZero (transformProps.ts:219 at 4.1.4 and 5.0.0, :326 at 6.1.0), so + # false fits the line to its values. Always written, true unless y_axis_truncate. + p["start_y_axis_at_zero"] = not chart.y_axis_truncate + # The Advanced Analytics rolling window (BigNumberWithTrendline controlPanel.tsx + # :179-228 at 4.1.4 and 5.0.0, :252-301 at 6.1.0), applied to the trendline by + # rollingWindowOperator in buildQuery, so transformProps reads the number and + # compare_lag's point from the rolled series (transformProps.ts:110-128 at 4.1.4 + # and 5.0.0, :189-210 at 6.1.0). The backend drops the first min_periods - 1 rows + # (utils/pandas_postprocessing/rolling.py:99-100, the same at all three tags), so + # min_periods = rolling_periods keeps only whole windows; Superset's own 0 draws + # the partial windows of the first steps too. + p["rolling_type"] = chart.rolling_type or "None" + if chart.rolling_periods is not None: + p["rolling_periods"] = chart.rolling_periods + p["min_periods"] = (chart.rolling_periods if chart.rolling_min_periods is None + else chart.rolling_min_periods) if chart.number_format: p["y_axis_format"] = chart.number_format _pin_big_number_fonts(p) @@ -301,6 +419,36 @@ def metric(m: str): "hasCustomLabel": False, } p["order_desc"] = chart.category_sort == "desc" + elif chart.sort_by: + # Largest first, as the default ranking: on top of the bottom-up horizontal axis. + ascending = chart.orientation == "horizontal" + sort_metric = chart.sort_metric() + if sort_metric is None: + # "total": the plugin sorts the categories by the sum of their series + # (SortSeriesType.Sum, utils/series.ts sortRows: 4.1.4 :200-208, 6.1.0 + # :564-572), reading x_axis_sort from 6.0.0 (Timeseries/transformProps.ts + # 6.1.0 :335-336) and x_axis_sort_series at 4.1.4 and 5.0.0 (:243-246, + # :246-249). "sum" is no label, so sortOperator.ts adds no sort of its own. + p["x_axis_sort"] = "sum" + p["x_axis_sort_series"] = "sum" + p["x_axis_sort_series_ascending"] = ascending + else: + # A metric: sortOperator.ts sorts the rows by it (no groupby, all three + # releases). As the "Sort query by" metric it also orders the query, so a + # row limit keeps the top bars by it (normalizeOrderBy, descending), and + # one that is not drawn is queried but never drawn: extractExtraMetrics.ts:35 + # adds it because its label is x_axis_sort, and extractSeries skips it. + payload = metric(sort_metric) + p["x_axis_sort"] = payload["label"] if isinstance(payload, dict) else payload + p["timeseries_limit_metric"] = payload + if chart.several_series(): + # Several series at 4.1.4 and 5.0.0: the plugin re-sorts the rows by + # x_axis_sort_series, whose panel default is the category name, unless + # it is unset (isDefined, utils/series.ts extractSeries, 4.1.4 + # :296-307, 5.0.0 :302-313). Null keeps sortOperator's order there; + # 6.0.0 dropped the control. + p["x_axis_sort_series"] = None + p["x_axis_sort_asc"] = ascending else: # Rankings read sorted by their measure, not by label order. The # horizontal axis renders bottom-up, so ascending puts the largest @@ -328,8 +476,11 @@ def metric(m: str): p[f"groupby{suffix}"] = [series.groupby] if series.groupby else [] p[f"row_limit{suffix}"] = chart.row_limit or DEFAULT_ROW_LIMIT[t] p["adhoc_filters_b"] = _adhoc_filters(chart) - p["seriesType"] = chart.a.kind - p["seriesTypeB"] = chart.b.kind + # An area is a line series with its "Area chart" box ticked (area / areaB, read + # by transformSeries as areaStyle; MixedTimeseries/transformProps.ts:387-390 and + # :434-437 at 4.1.4, :466-469 and :540-543 at 6.1.0). + p["seriesType"] = SERIES_TYPE[chart.a.kind] + p["seriesTypeB"] = SERIES_TYPE[chart.b.kind] p["yAxisIndex"] = 0 if chart.a.axis == "primary" else 1 p["yAxisIndexB"] = 0 if chart.b.axis == "primary" else 1 p["y_axis_format"] = chart.number_format or "SMART_NUMBER" @@ -339,6 +490,12 @@ def metric(m: str): # markerEnabled / markerEnabledB (createCustomizeSection, all three releases); # emitted only when set, so pre-feature bundles stay byte-identical. for suffix, series in (("", chart.a), ("B", chart.b)): + if series.kind == "area": + p[f"area{suffix}"] = True + if series.opacity is not None: + # The fill is opacity x this (transformSeries areaStyle, Timeseries/ + # transformers.ts at 4.1.4, 5.0.0 and 6.1.0); the edge line stays opaque. + p[f"opacity{suffix}"] = series.opacity if series.markers: p[f"markerEnabled{suffix}"] = True if series.show_value: @@ -408,8 +565,7 @@ def metric(m: str): column_config = _column_config(chart) if column_config: p["column_config"] = column_config - if chart.cell_bars is not None: - p["show_cell_bars"] = chart.cell_bars + _bar_switches(chart, p) if chart.date_format: p["table_timestamp_format"] = chart.date_format elif t == "pivot_table": @@ -450,8 +606,8 @@ def metric(m: str): p["normalize_across"] = chart.normalize_across p["legend_type"] = "continuous" p["linear_color_scheme"] = chart.color_scheme or HEATMAP_DEFAULT_SCHEME - p["sort_x_axis"] = "alpha_asc" - p["sort_y_axis"] = "alpha_asc" + p["sort_x_axis"] = HEATMAP_X_SORT[chart.x_order or "a_to_z"] + p["sort_y_axis"] = HEATMAP_Y_SORT[chart.y_order or "z_to_a"] p["show_legend"] = chart.show_legend p["row_limit"] = chart.row_limit or DEFAULT_ROW_LIMIT[t] # Heatmap controlPanel.tsx, all three releases; emitted only when set. @@ -461,6 +617,18 @@ def metric(m: str): p["show_percentage"] = False # the control's default is true if chart.number_format: p["y_axis_format"] = chart.number_format # the cell values' format + # xscale_interval / yscale_interval: transformProps hands interval N - 1 to the + # category axis, so N labels every Nth value from the first (Heatmap/transformProps.ts + # 4.1.4 :231,238, 5.0.0 :234,241, 6.1.0 :436,445). left_margin is the grid's left + # edge, labels inside it (containLabel, :166-169, :168-171, :354-357); 'auto' is 0. + # On 6.1.0 Heatmap.tsx:25 renders without vizType, so a theme's per-type + # overrides never reach it and these are the only way to set the axes. + if chart.x_label_every is not None: + p["xscale_interval"] = chart.x_label_every + if chart.y_label_every is not None: + p["yscale_interval"] = chart.y_label_every + if chart.left_margin is not None: + p["left_margin"] = chart.left_margin elif t == "histogram": p["column"] = chart.column p["bins"] = chart.bins @@ -488,6 +656,10 @@ def metric(m: str): p["label_type"] = chart.label_type if chart.number_format: p["number_format"] = chart.number_format + elif t == "waterfall": + _waterfall_params(chart, p, metric) + elif t == "box_plot": + _box_plot_params(chart, p, ds, metric) if isinstance(chart, _SeriesDisplay): _series_display_params(chart, p, metric) @@ -552,6 +724,8 @@ def time_binding(chart, ds) -> tuple[str, str] | None: return ("axis", chart.time_column) if chart.type == "mixed" and mixed_time_axis(chart, ds): return ("axis", chart.x_column) + if chart.type == "waterfall" and waterfall_time_axis(chart, ds): + return ("axis", chart.x_column) if ds.main_dttm_col: return ("granularity", ds.main_dttm_col) return None @@ -612,18 +786,35 @@ def _y_title_layout(p: dict) -> None: p["y_axis_title_position"] = "Top" +# A horizontal bar's titles. transformProps lays them out for a vertical chart, then swaps the +# axes and the bottom and left padding (Timeseries/transformProps.ts 4.1.4 and 5.0.0 :530-531, +# 6.1.0 :993-994). So x_axis_title names the category axis, drawn rotated left of the bars at +# nameGap from the axis line, through the column of category labels; y_axis_title names the +# value axis along the bottom. Placed 'Top' (nameLocation 'end', :526 and :973), the value +# title lands past the axis's right end and is cut off; 'Left' centres it under the axis, and +# getPadding's left offset becomes the bottom one (Timeseries/transformers.ts getPadding). +HBAR_CATEGORY_TITLE_GAP = 64 # clears category labels of about 8 characters at 12 px +HBAR_VALUE_TITLE_GAP = 30 # clears the value labels under the axis + + def _y_axis_params(chart, p: dict) -> None: """Axis titles, bounds, truncation and log scale: titleControls (sections/chartTitle.tsx) and the panels' Y Axis section, at 4.1.4, 5.0.0 and 6.1.0. transformProps passes y_axis_bounds to ECharts as the axis min and max, so a bound applies on every release.""" + horizontal = getattr(chart, "orientation", None) == "horizontal" if chart.x_axis_title: p["x_axis_title"] = chart.x_axis_title # Clear of the tick labels; rotated labels hang lower. 0 (6.1.0's default margin) # draws the title over the labels. - p["x_axis_title_margin"] = 50 if chart.x_label_rotation else 30 + p["x_axis_title_margin"] = (HBAR_CATEGORY_TITLE_GAP if horizontal + else 50 if chart.x_label_rotation else 30) if chart.y_axis_title: p["y_axis_title"] = chart.y_axis_title - _y_title_layout(p) + if horizontal: + p["y_axis_title_margin"] = HBAR_VALUE_TITLE_GAP + p["y_axis_title_position"] = "Left" + else: + _y_title_layout(p) if chart.y_axis_min is not None or chart.y_axis_max is not None: p["y_axis_bounds"] = [chart.y_axis_min, chart.y_axis_max] if chart.y_axis_truncate: @@ -689,6 +880,197 @@ def mixed_time_axis(chart, ds) -> bool: return bool(temporal or (temporal is None and chart.time_grain)) +def waterfall_time_axis(chart, ds) -> bool: + """Whether a waterfall's steps are periods of a time column, by the same test as a + mixed chart's. A bridge's own order (steps) is always categorical.""" + return chart.steps is None and mixed_time_axis(chart, ds) + + +# A bridge in its own order (WaterfallChart.steps) draws its bars in the order of an x +# axis the compiler writes: a CASE over the step column giving each step a key, the +# closing row its own name. The plugin groups and sorts the rows by the x axis and then +# by the Breakdowns column (Waterfall/buildQuery.ts:27-35, orderby every column +# ascending, all three releases); with the step column as the breakdown, each bar is +# labelled by its step and the closing row, matched by total_label, is drawn as the +# running total under its x value: its name (Waterfall/transformProps.ts at 6.1.0, +# :240-254 and :326-330). show_total: false keeps the plugin from adding a running total +# after every step (:117-134), a 6.1.0 control: older plugins always add one. +STEPS_ORDER_LABEL = "Bridge order" +STEPS_LABEL = "Bridge step" # an opened bridge's breakdown, apart from the step column's name +STEPS_OTHER_KEY = "0zzz" # values steps doesn't list: after every listed step, before the closing +_PLAIN_COLUMN = re.compile(r"[a-z_][a-z0-9_]*") + + +def _sql_text(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _sql_column(column: str) -> str: + # A lowercase identifier as it is; anything else double-quoted (ANSI SQL). + return column if _PLAIN_COLUMN.fullmatch(column) else '"' + column.replace('"', '""') + '"' + + +def _step_keys(steps: list[str], opening: str | None) -> tuple[list[str], str]: + """Each step's order key and the key of a value steps doesn't list. Without an + opening: '0000', '0001', ... and '0zzz', all starting with '0', which sorts before a + letter or a digit 1-9 in every common collation (the spec holds closing to that). + With one: 'FY2025 000', ... and 'FY2025 9999', after the opening's own name and, + as the spec checks (spec.sorts_before), before the closing's.""" + if opening is None: + return [f"0{i:03d}" for i in range(len(steps))], STEPS_OTHER_KEY + return ([f"{opening}{OPENING_KEY_GAP}{i:03d}" for i in range(len(steps))], + opening + OPENING_OTHER_KEY) + + +def steps_order_sql(column: str, steps: list[str], closing: str, opening: str | None = None) -> str: + """The bridge's x axis: a key per step, the opening's and the closing's own names.""" + keys, other = _step_keys(steps, opening) + whens = [f"WHEN {_sql_text(opening)} THEN {_sql_text(opening)}"] if opening is not None else [] + whens += [f"WHEN {_sql_text(s)} THEN {_sql_text(k)}" for s, k in zip(steps, keys)] + whens.append(f"WHEN {_sql_text(closing)} THEN {_sql_text(closing)}") + return f"CASE {_sql_column(column)} {' '.join(whens)} ELSE {_sql_text(other)} END" + + +def bridge_totals_sql(column: str, opening: str, closing: str) -> str: + """An opened bridge's breakdown: both total rows marked BRIDGE_TOTAL (total_label), + each step by its own name.""" + col = _sql_column(column) + return (f"CASE WHEN {col} IN ({_sql_text(opening)}, {_sql_text(closing)}) " + f"THEN {_sql_text(BRIDGE_TOTAL)} ELSE {col} END") + + +_SQL_TEXT = r"'(?:[^']|'')*'" +_SQL_COLUMN = r"[a-z_][a-z0-9_]*|\"(?:[^\"]|\"\")+\"" +_STEPS_SQL = re.compile( + rf"CASE (?P{_SQL_COLUMN}) (?P(?:WHEN {_SQL_TEXT} THEN {_SQL_TEXT} )+)" + rf"ELSE (?P{_SQL_TEXT}) END", re.S) + + +def _text(lit: str) -> str: + return lit[1:-1].replace("''", "'") + + +def parse_steps_order_sql(sql: str) -> tuple[str, list[str], str, str | None] | None: + """(column, steps, closing, opening or None) from an x axis steps_order_sql wrote, + or None for any other SQL.""" + m = _STEPS_SQL.fullmatch(sql or "") + if not m: + return None + pairs = [(_text(a), _text(b)) for a, b in + re.findall(rf"WHEN ({_SQL_TEXT}) THEN ({_SQL_TEXT})", m.group("whens"))] + if len(pairs) < 2 or pairs[-1][0] != pairs[-1][1]: + return None + closing = pairs[-1][0] + opening = pairs[0][0] if pairs[0][0] == pairs[0][1] and len(pairs) > 2 else None + steps = [s for s, _ in pairs[1 if opening is not None else 0:-1]] + if (m.group("other"), [k for _, k in pairs[1 if opening is not None else 0:-1]]) != ( + _sql_text(_step_keys(steps, opening)[1]), _step_keys(steps, opening)[0]): + return None + column = m.group("column") + if column.startswith('"'): + column = column[1:-1].replace('""', '"') + return column, steps, closing, opening + + +def _waterfall_params(chart, p: dict, metric) -> None: + """Waterfall/controlPanel.tsx at 4.1.4, 5.0.0 and 6.1.0; the labels and show_total + are 6.1.0 controls. Emitted only when set, but for the query keys.""" + if chart.steps: + p["x_axis"] = {"expressionType": "SQL", "label": STEPS_ORDER_LABEL, + "sqlExpression": steps_order_sql(chart.x_column, chart.steps, + chart.closing, chart.opening)} + if chart.opening is None: + p["groupby"] = [chart.x_column] + p["total_label"] = chart.closing + else: + # Both ends are total rows: a total at the first index adds to the running + # total, any later one shows it (transformProps.ts:241-250 at 6.1.0). + p["groupby"] = [{"expressionType": "SQL", "label": STEPS_LABEL, + "sqlExpression": bridge_totals_sql(chart.x_column, chart.opening, + chart.closing)}] + p["total_label"] = BRIDGE_TOTAL + p["show_total"] = False + else: + p["x_axis"] = chart.x_column + p["groupby"] = [chart.groupby] if chart.groupby else [] + if chart.time_grain: + # Written as the spec says: the backend buckets a temporal x axis only and + # leaves a categorical one as it is (seen on 4.1.4 and 6.1.0). + p["time_grain_sqla"] = chart.time_grain + if chart.total_label: + p["total_label"] = chart.total_label + p["metric"] = metric(chart.metric) + p["row_limit"] = chart.row_limit or DEFAULT_ROW_LIMIT["waterfall"] + # The colour pickers store {r, g, b, a}; transformProps paints rgbToHex(r, g, b). + for field in ("increase_color", "decrease_color", "total_color"): + if getattr(chart, field): + p[field] = hex_to_rgb(named_hex(getattr(chart, field))) + for field in ("increase_label", "decrease_label"): + if getattr(chart, field): + p[field] = getattr(chart, field) + if chart.show_value: + p["show_value"] = True + if chart.show_legend: + p["show_legend"] = True # the control's default is false + if chart.number_format: + p["y_axis_format"] = chart.number_format + if chart.x_axis_title: + p["x_axis_label"] = chart.x_axis_title + if chart.y_axis_title: + p["y_axis_label"] = chart.y_axis_title + if chart.x_label_rotation is not None: + p["x_ticks_layout"] = TICK_LAYOUTS[chart.x_label_rotation] + if chart.x_label_format: + p["x_axis_time_format"] = chart.x_label_format + + +# whiskers -> whiskerOptions, the Box Plot panel's choices (BoxPlot/controlPanel.ts, 4.1.4 +# and 5.0.0 :83-99, 6.1.0 :84-102). The select is free-form, and the boxplot operator +# reads any 'lo/hi percentiles' (operators/boxplotOperator.ts:28, all three releases), so +# [5, 95] works on 4.1.4 too, whose choices don't list it (seen rendering). +WHISKER_OPTIONS = {"tukey": "Tukey", "min_max": "Min/max (no outliers)"} + + +def whisker_options(whiskers) -> str: + if isinstance(whiskers, list): + return f"{whiskers[0]}/{whiskers[1]} percentiles" + return WHISKER_OPTIONS[whiskers] + + +def _box_plot_params(chart, p: dict, ds, metric) -> None: + """BoxPlot/controlPanel.ts at 4.1.4, 5.0.0 and 6.1.0: the observations are the rows + of `columns` (Distribute across), a box per `groupby` value, and the boxplot + post-processing turns them into quartiles (BoxPlot/buildQuery.ts:29-57). row_limit + is a 6.0.0 control.""" + p["columns"] = list(chart.distribute_across) + p["groupby"] = list(chart.groupby) + p["metrics"] = [metric(m) for m in chart.metrics] + # Written even as Tukey: without it buildQuery adds no boxplot post-processing. + p["whiskerOptions"] = whisker_options(chart.whiskers) + if chart.time_grain: + # buildQuery buckets a column by the grain only if temporal_columns_lookup marks + # it, a control Explore fills from the dataset (BoxPlot/buildQuery.ts:38-50). + p["time_grain_sqla"] = chart.time_grain + lookup = {c: True for c in chart.distribute_across if ds.is_temporal(c) is not False} + if lookup: + p["temporal_columns_lookup"] = lookup + if chart.row_limit: + p["row_limit"] = chart.row_limit + if chart.number_format: + p["number_format"] = chart.number_format + if chart.x_label_format: + p["date_format"] = chart.x_label_format + if chart.x_label_rotation is not None: + p["x_ticks_layout"] = TICK_LAYOUTS[chart.x_label_rotation] + # sections.titleControls, as the axis charts write them (_y_axis_params). + if chart.x_axis_title: + p["x_axis_title"] = chart.x_axis_title + p["x_axis_title_margin"] = 50 if chart.x_label_rotation else 30 + if chart.y_axis_title: + p["y_axis_title"] = chart.y_axis_title + _y_title_layout(p) + + def _x_label_params(chart: _AxisChart, p: dict) -> None: # Emitted only when set, so pre-feature bundles stay byte-identical. # x_axis_time_format and xAxisLabelRotation: every chart panel at 4.1.4, 5.0.0 and 6.1.0. @@ -718,6 +1100,8 @@ def _x_label_params(chart: _AxisChart, p: dict) -> None: # Chart types whose 6.1.0 control panel declares echart_options (scatter's does not). _ECHART_OPTIONS_TYPES = ("timeseries_line", "timeseries_bar", "timeseries_area", "mixed") +# A mixed query's kind -> its seriesType (EchartsTimeseriesSeriesType, Timeseries/types.ts). +SERIES_TYPE = {"bar": "bar", "line": "line", "area": "line"} def _has_bars(chart) -> bool: @@ -756,7 +1140,7 @@ def _chart_meta(spec: DashboardSpec, name: str, cuuid, width: int, height: float "uuid": str(cuuid), "sliceName": name, "width": width, - "height": int(round(height * ROW_UNITS_PER_SPEC_UNIT)), + "height": grid_rows(height), # Placeholder the importer requires and remaps via uuid. "chartId": 100000 + counter[0], } @@ -777,9 +1161,7 @@ def _rows_into(pos: dict, rows, spec: DashboardSpec, parents: list[str], prefix: # ROW (dashboard/util/isValidChild.ts, all three releases). kind = "HEADER" if isinstance(entry, HeaderBlock) else "DIVIDER" node_id = f"{kind}-{prefix}{i + 1}" - meta = ({"text": entry.header, "headerSize": HEADER_SIZE[entry.size], - "background": BACKGROUND[entry.background]} - if kind == "HEADER" else {}) + meta = _header_meta(entry) if kind == "HEADER" else {} pos[node_id] = {"type": kind, "id": node_id, "children": [], "parents": parents, "meta": meta} row_ids.append(node_id) continue @@ -799,7 +1181,7 @@ def _rows_into(pos: dict, rows, spec: DashboardSpec, parents: list[str], prefix: "meta": { "code": item.markdown, "width": spec.resolved_item_width(item), - "height": int(round((item.height or 4) * ROW_UNITS_PER_SPEC_UNIT)), + "height": grid_rows(item.height or 4), }, } continue @@ -842,20 +1224,59 @@ def _sketch_chart_node(pos, spec, sc, width, parents, counter) -> str: return chart_id -def _sketch_into(pos, parsed_rows, spec, parents: list[str], prefix: str, counter: list[int]) -> list[str]: - """Emit ROW / COLUMN / CHART nodes from a parsed sketch (chartwright/sketch.py).""" - from .sketch import SketchColumn +def _header_meta(entry: HeaderBlock) -> dict: + return {"text": entry.header, "headerSize": HEADER_SIZE[entry.size], + "background": BACKGROUND[entry.background]} + + +def _sketch_block_node(pos, holder, sb, width, node_id, parents) -> str: + """Emit one MARKDOWN or HEADER node from a sketch block; a markdown block's own + height wins over the drawn one, as a chart's does.""" + entry = holder.sketch_block(sb) + if sb.kind == "header": + meta = _header_meta(entry) + node_id = f"HEADER-{node_id}" + else: + meta = {"code": entry.markdown, "width": width, + "height": int(round(holder.sketch_block_height(sb) * ROW_UNITS_PER_SPEC_UNIT))} + node_id = f"MARKDOWN-{node_id}" + pos[node_id] = {"type": "HEADER" if sb.kind == "header" else "MARKDOWN", "id": node_id, + "children": [], "parents": parents, "meta": meta} + return node_id + + +def _sketch_into(pos, holder, spec, parents: list[str], prefix: str, counter: list[int]) -> list[str]: + """Emit ROW / COLUMN / CHART nodes from a parsed sketch (chartwright/sketch.py), and the + MARKDOWN and HEADER nodes of its blocks. A header across the whole sketch is a HEADER + between rows, as in rows; any other header sits in a COLUMN, the one place beside a + ROW's other children Superset takes a header (dashboard/util/isValidChild.ts :75 and + :97 at 4.1.4, 5.0.0 and 6.1.0): its stack's, or one of its own.""" + from .sketch import SketchBlock, SketchColumn row_ids: list[str] = [] - for i, srow in enumerate(parsed_rows): + for i, srow in enumerate(holder.parsed_sketch()): + band = srow.header_band + if band is not None: + node_id = f"HEADER-{prefix}{i + 1}" + pos[node_id] = {"type": "HEADER", "id": node_id, "children": [], "parents": parents, + "meta": _header_meta(holder.sketch_block(band))} + row_ids.append(node_id) + continue row_id = f"ROW-{prefix}{i + 1}" child_ids: list[str] = [] for j, child in enumerate(srow.children): - if isinstance(child, SketchColumn): - col_id = f"COLUMN-{prefix}{i + 1}-{j + 1}" + slot = f"{prefix}{i + 1}-{j + 1}" + if isinstance(child, SketchColumn) or ( + isinstance(child, SketchBlock) and child.kind == "header"): + col_id = f"COLUMN-{slot}" + items = child.children if isinstance(child, SketchColumn) else [child] + # A list of its own per node: a shared one is a YAML alias in the bundle. col_children = [ + _sketch_block_node(pos, holder, sc, child.width, f"{slot}-{k + 1}", + [*parents, row_id, col_id]) + if isinstance(sc, SketchBlock) else _sketch_chart_node(pos, spec, sc, child.width, [*parents, row_id, col_id], counter) - for sc in child.children + for k, sc in enumerate(items) ] pos[col_id] = { "type": "COLUMN", @@ -865,6 +1286,9 @@ def _sketch_into(pos, parsed_rows, spec, parents: list[str], prefix: str, counte "meta": {"background": "BACKGROUND_TRANSPARENT", "width": child.width}, } child_ids.append(col_id) + elif isinstance(child, SketchBlock): + child_ids.append( + _sketch_block_node(pos, holder, child, child.width, slot, [*parents, row_id])) else: child_ids.append( _sketch_chart_node(pos, spec, child, child.width, [*parents, row_id], counter) @@ -892,9 +1316,7 @@ def _position(spec: DashboardSpec) -> dict: if spec.layout.rows: grid_children = _rows_into(pos, spec.layout.rows, spec, ["ROOT_ID", "GRID_ID"], "sdc-", counter) elif spec.layout.sketch: - grid_children = _sketch_into( - pos, spec.layout.parsed_sketch(), spec, ["ROOT_ID", "GRID_ID"], "sdc-", counter - ) + grid_children = _sketch_into(pos, spec.layout, spec, ["ROOT_ID", "GRID_ID"], "sdc-", counter) else: tabs_id = "TABS-sdc-1" tab_ids: list[str] = [] @@ -909,7 +1331,7 @@ def tab_node(tab_id: str, title: str, children: list[str], parents: list[str]) - def content_into(tab, parents: list[str], prefix: str) -> list[str]: into = _sketch_into if tab.sketch else _rows_into - return into(pos, tab.parsed_sketch() if tab.sketch else tab.rows, spec, parents, prefix, counter) + return into(pos, tab if tab.sketch else tab.rows, spec, parents, prefix, counter) for k, tab in enumerate(spec.layout.tabs or []): tab_id = f"TAB-sdc-{k + 1}" diff --git a/chartwright/dashdiff.py b/chartwright/dashdiff.py index 2b64565..4a4cf3c 100644 --- a/chartwright/dashdiff.py +++ b/chartwright/dashdiff.py @@ -14,13 +14,19 @@ import yaml from .client import SupersetClient -from .compiler import compile_bundle, filter_id -from .decompile import decompile_live +from .compiler import _metric_payload, compile_bundle, filter_id, grid_rows, spec_units +from .decompile import _metric_to_spec, decompile_live from .spec import ( - DATASET_FILTER_TYPES, DEFAULT_ROW_LIMIT, DEFAULT_TIME_GRAIN, DashboardSpec, load_spec, row_items, - without_superset_defaults, + BAR_SWITCHES, DATASET_FILTER_TYPES, DEFAULT_ROW_LIMIT, DEFAULT_TIME_GRAIN, FORMAT_COLOR_HEX, + FORMAT_TEXT_HEX, DashboardSpec, load_spec, named_hex, row_items, without_superset_defaults, ) +# Colour fields that take green, amber or red or a #RRGGBB: a name and its own hex paint +# the same, and decompile reads that hex back as the name, so plan compares the hex. +NAMED_COLOUR_FIELDS = ("trend_color", "increase_color", "decrease_color", "total_color") +# A table's fields that list labels (hidden columns, the cell-bar switches). +TABLE_LABEL_LISTS = ("hidden", *BAR_SWITCHES) + # Dashboard settings `plan` compares one by one (dashboard_settings_changed). DASHBOARD_SETTINGS = ( "color_scheme", "description", "certified_by", "certification_details", "published", @@ -110,23 +116,95 @@ def _normalize_tags(holder: dict) -> None: holder.pop("tags", None) +def _canonical_metric(metric: str) -> str: + """A metric as decompile reads back what compile writes for it, so two spellings of + one stored metric compare equal: `SQL(COUNT(*)) AS N` and `COUNT(*) AS N`, or + `SUM(x) AS N` and `SUM(x) AS N`, which differ only in a key Superset never shows.""" + return _metric_to_spec(_metric_payload(metric, "", ""), [], "") or metric + + +def _canonical_metrics(chart: dict) -> None: + """Every metric a chart names (spec.chart_metrics), in its canonical spelling.""" + for holder in (chart, chart.get("a"), chart.get("b")): + if not isinstance(holder, dict): + continue + for key in ("metric", "series_limit_metric"): + if isinstance(holder.get(key), str): + holder[key] = _canonical_metric(holder[key]) + if holder.get("metrics"): + holder["metrics"] = [_canonical_metric(m) for m in holder["metrics"]] + if chart["type"] == "table" and chart.get("sort_by") and not chart.get("columns"): + chart["sort_by"] = _canonical_metric(chart["sort_by"]) # a metric in aggregate mode + + +def _canonical_colours(chart: dict) -> None: + """A named shade's own hex reads as the name: compile paints both alike, and + decompile names the shade (_rgb_to_spec, _format_to_spec, _big_number_rules_to_spec).""" + text_names = {v.upper(): k for k, v in FORMAT_TEXT_HEX.items()} + cell_names = {v.upper(): k for k, v in FORMAT_COLOR_HEX.items()} + if isinstance(chart.get("trend_color"), str): + chart["trend_color"] = text_names.get(chart["trend_color"].upper(), chart["trend_color"]) + # A big number's rules colour the number, which is text: a name paints its text shade. + number = chart.get("type") == "big_number_total" + for rule in chart.get("conditional_formatting") or []: + names = text_names if number or rule.get("paint") == "text" else cell_names + rule["color"] = names.get(str(rule["color"]).upper(), rule["color"]) + + +def _drop_blank_text(model, data: dict) -> None: + """Blank text in an optional field (a description of " ") reads as unset, as + decompile reads it back: Superset shows nothing for it either.""" + for key, value in list(data.items()): + f = type(model).model_fields.get(key) + if isinstance(value, str) and not value.strip() and f is not None and not f.is_required(): + del data[key] + + def _normalize(spec: DashboardSpec) -> dict: """Canonical form for comparison: validated model dump with every compiler default materialized, so spec-with-defaults-omitted and decompiled-with-defaults-present compare equal. Chart identity is the name, so chart list order is canonicalized by name. A written Superset default - compares equal to the omitted field, which is all decompile can read back.""" + compares equal to the omitted field, which is all decompile can read back, and + any other spelling compile stores alike compares as the one decompile reads.""" data = without_superset_defaults(spec).model_dump(exclude_none=True, by_alias=True) - for chart in data["charts"]: + _drop_blank_text(spec.dashboard, data["dashboard"]) + for model, chart in zip(spec.charts, data["charts"]): + _drop_blank_text(model, chart) chart["width"] = spec.resolved_item_width(chart["name"]) - chart["height"] = spec.resolved_height(chart["name"]) + # On the grid compile writes: whole 8 px rows, read back in fifths (4.65 -> 4.6). + chart["height"] = spec_units(grid_rows(spec.resolved_height(chart["name"]))) if chart["type"] in DEFAULT_ROW_LIMIT: chart.setdefault("row_limit", DEFAULT_ROW_LIMIT[chart["type"]]) + if chart["type"] == "table": + # Label lists (hidden, the bar switches) are per-column settings, kept in + # column_config by label: their order changes nothing, and decompile reads + # them back in column_config's order, not the author's. + order = {k: i for i, k in enumerate(model.labels())} + for key in TABLE_LABEL_LISTS: + if isinstance(chart.get(key), list): + chart[key] = sorted(chart[key], key=lambda k: order.get(k, len(order))) if chart["type"] in ("timeseries_line", "timeseries_bar", "timeseries_area", "timeseries_scatter", "big_number_trend"): chart.setdefault("time_grain", DEFAULT_TIME_GRAIN) + for field in NAMED_COLOUR_FIELDS: + if chart.get(field): + chart[field] = named_hex(chart[field]) + if chart["type"] == "table": + # An empty list compiles as the omitted one, which is how decompile reads it. + for key in ("metrics", "groupby"): + if chart.get(key) == []: + del chart[key] + _canonical_metrics(chart) + _canonical_colours(chart) _normalize_tags(chart) data["charts"].sort(key=lambda c: c["name"]) + for model, f in zip(spec.filters, data.get("filters") or []): + _drop_blank_text(model, f) + if f.get("time_range") == "No filter": + # A pre-filter over every time is no pre-filter, which is how decompile reads it. + f.pop("time_range") + f.pop("time_column", None) _normalize_tags(data["dashboard"]) # Owners are names in a spec and ids live; plan compares them as ids, apart. data["dashboard"].pop("owners", None) @@ -141,18 +219,38 @@ def norm_rows(model_rows, data_rows) -> None: ditem["width"] = spec.resolved_item_width(mitem) ditem.setdefault("height", 4) - def sketch_as_rows(holder) -> list[list]: + def sketch_as_rows(holder) -> list: """Sketch -> rows of chart names, a stacked COLUMN as a nested list, so a sketch spec and its live state compare equal: decompile reads a dashboard - with columns back as a sketch, and one without as rows of the same names.""" + with columns back as a sketch, and one without as rows of the same names. + Blocks read as they do in rows: a header across the sketch is a header row, + a markdown block a {markdown, width, height} item; a header elsewhere sits in + its column (alone, a column of its own) with the column's width.""" + from .sketch import SketchBlock, SketchColumn + + def item(sc, width: int): + if not isinstance(sc, SketchBlock): + return sc.name + entry = holder.sketch_block(sc) + if sc.kind == "header": + return {**entry.model_dump(), "width": width} + return {"markdown": entry.markdown, "width": width, + "height": holder.sketch_block_height(sc)} + rows = [] for srow in holder.parsed_sketch(): + band = srow.header_band + if band is not None: + rows.append(holder.sketch_block(band).model_dump()) + continue row = [] for child in srow.children: - if hasattr(child, "children"): - row.append([sc.name for sc in child.children]) + if isinstance(child, SketchColumn): + row.append([item(sc, child.width) for sc in child.children]) + elif isinstance(child, SketchBlock) and child.kind == "header": + row.append([item(child, child.width)]) else: - row.append(child.name) + row.append(item(child, child.width)) rows.append(row) return rows diff --git a/chartwright/decompile.py b/chartwright/decompile.py index 444edcd..84d59d9 100644 --- a/chartwright/decompile.py +++ b/chartwright/decompile.py @@ -18,17 +18,22 @@ import yaml from .compiler import ( - BACKGROUND, COLUMN_CONFIG_KEYS, CONTRIBUTION_VALUES, FOOTER_PREFIX, HEADER_PREFIX, HEADER_SIZE, - LEGEND_TYPES, ROW_UNITS_PER_SPEC_UNIT, SDC_BAR_MARKER, STACK_VALUES, VIZ_TYPE, + BACKGROUND, BAR_SWITCH_KEYS, COLUMN_CONFIG_KEYS, CONTRIBUTION_VALUES, FOOTER_PREFIX, + FORMAT_OPERATOR, HEADER_PREFIX, HEADER_SIZE, HEATMAP_X_SORT, HEATMAP_Y_SORT, LEGEND_TYPES, + METRIC_OPTION_PREFIX, ROW_UNITS_PER_SPEC_UNIT, SDC_BAR_MARKER, STACK_VALUES, VIZ_TYPE, + WHISKER_OPTIONS, bridge_totals_sql, parse_steps_order_sql, spec_units, ) from .spec import ( - ADHOC_AGGREGATES, DEPENDENCY_PARENT_TYPES, FORMAT_COLOR_HEX, FORMAT_TEXT_HEX, FUNNEL_LABEL_TYPES, - HEATMAP_DEFAULT_SCHEME, HEX_COLOUR_RE, PIVOT_ORDER, TREND_DEFAULT_HEX, FilterOp, LabelType, - PivotAggregate, SequentialScheme, metric_label, row_items, + ADHOC_AGGREGATES, BRIDGE_TOTAL, DEPENDENCY_PARENT_TYPES, FORMAT_COLOR_HEX, + FORMAT_RANGE_OPERATORS, FORMAT_TEXT_HEX, FUNNEL_LABEL_TYPES, HEATMAP_DEFAULT_SCHEME, + HEX_COLOUR_RE, PIVOT_ORDER, SUPERSET_DEFAULTS, TICK_LAYOUTS, TREND_DEFAULT_HEX, + WATERFALL_DEFAULT_HEX, FilterOp, LabelType, PivotAggregate, SequentialScheme, metric_label, + parse_metric, row_items, ) REVERSE_VIZ = {v: k for k, v in VIZ_TYPE.items() if k != "bar"} # echarts_timeseries_bar -> timeseries_bar _FILTER_OPS = set(get_args(FilterOp)) +_FORMAT_OPERATOR_BACK = {v: k for k, v in FORMAT_OPERATOR.items()} | {"between": "between"} # Cosmetic / behavioral params we knowingly discard without a loss entry. _IGNORABLE = { @@ -57,7 +62,18 @@ "left_margin", "show_percentage", "show_values", "value_bounds", "xscale_interval", "yscale_interval", "column", "bins", "normalize", "show_value", "slice_id", "url_params", "percent_calculation_type", - "show_tooltip_labels", "tooltip_label_type", SDC_BAR_MARKER, + "show_tooltip_labels", "tooltip_label_type", "show_total", "x_ticks_layout", "zoomable", + "series_limit", "series_limit_metric", SDC_BAR_MARKER, + # A mixed chart's query B twins of the above (createCustomizeSection 'B', + # createAdvancedAnalyticsSection and createQuerySection '_b', MixedTimeseries + # controlPanel.tsx at 4.1.4, 5.0.0 and 6.1.0), with the same stored defaults. + "markerSizeB", "comparison_type_b", "truncate_metric_b", "rolling_type_b", + "sort_series_typeB", + # The big numbers' "Force date format" (BigNumberTotal/controlPanel.ts and + # BigNumberWithTrendline/controlPanel.tsx, default false at all three tags). + "force_timestamp_formatting", + # A table's stored defaults (below). + "allow_render_html", "allow_rearrange_columns", "server_pagination", "percent_metrics", } # Of the keys above, those a user changes from the value an untouched chart stores: @@ -99,6 +115,20 @@ "bottom_margin": ("auto",), "left_margin": ("auto",), "xscale_interval": (-1,), "yscale_interval": (-1,), "value_bounds": ([None, None], [], None), "normalize": (False,), + "markerSizeB": (6,), "comparison_type_b": ("values",), "truncate_metric_b": (True,), + "rolling_type_b": (None, "None", ""), "sort_series_typeB": ("sum",), + "force_timestamp_formatting": (False, None), + # Waterfall/controlPanel.tsx: a closing total (show_total, 6.1.0 :134) and Superset's + # own tick layout (x_ticks_layout, 4.1.4 :130, 6.1.0 :210). + "show_total": (True,), "x_ticks_layout": ("auto",), + # sharedControls.tsx: data zoom off (zoomable, 6.1.0 :422) and no series limit + # (series_limit and series_limit_metric, the box plot's at all three releases). + "zoomable": (False,), "series_limit": (None, "", 0), "series_limit_metric": (None, "", [], {}), + # Table controlPanel.tsx: HTML rendered and columns fixed (4.1.4 :449-454, :433-438; + # 5.0.0 :468-473, :452-457; 6.1.0 :543-548, :527-532), client-side paging (4.1.4 + # :317-324, 5.0.0 :351-358, 6.1.0 :387-394), no percentage metrics. + "allow_render_html": (True,), "allow_rearrange_columns": (False,), + "server_pagination": (False,), "percent_metrics": ([], None), } # A currency format has no default: unset reads as {} or every part empty. _UNSET_WHEN_EMPTY = {"currency_format"} @@ -106,15 +136,19 @@ _STORED_DEFAULT_WHEN = { "only_total": lambda p: bool(p.get("show_value")) and bool(p.get("stack")), "comparison_type": lambda p: bool(p.get("time_compare")), + "comparison_type_b": lambda p: bool(p.get("time_compare_b")), "x_axis_title_margin": lambda p: bool(p.get("x_axis_title")), "y_axis_title_margin": lambda p: bool(p.get("y_axis_title")), "y_axis_title_position": lambda p: bool(p.get("y_axis_title")), "innerRadius": lambda p: bool(p.get("donut")), + # Percentage metrics join the query in aggregate mode only (Table buildQuery.ts: 4.1.4 + # :131-142, 5.0.0 :120-131, 6.1.0 :129-146), so a raw table's leftovers are inert. + "percent_metrics": lambda p: not (p.get("query_mode") == "raw" or p.get("all_columns")), } # Spec chart types with a color_scheme field (spec._ColorSchemeMixin). _COLOR_SCHEME_TYPES = ("timeseries_line", "timeseries_bar", "timeseries_area", "timeseries_scatter", - "bar", "pie", "histogram", "funnel", "treemap", "mixed") + "bar", "pie", "histogram", "funnel", "treemap", "mixed", "box_plot") _REVERSE_HEADER_SIZE = {v: k for k, v in HEADER_SIZE.items()} _REVERSE_OPACITY = {"opacityLow": "low", "opacityMedium": "medium", "opacityHigh": "high"} @@ -125,6 +159,43 @@ "xAxisLabelInterval"} +def _bar_switch(p: dict, key: str, own: dict[str, bool], labels: list[str], default: bool): + """One cell-bar switch as the spec writes it, from the table-wide param `key` and each + column's own value (`own`), which Superset reads first: the table-wide bool while no + column of `labels` differs from it (None when the chart leaves it unset), else the + labels it is on for (False when none). A column's value that matches the table's, or + sits on a column that draws no bar, changes nothing and is dropped. A stored null is + a false: only an absent key takes the panel's default (applyDefaultFormData).""" + table = bool(p[key]) if key in p else default + if all(own.get(k, table) == table for k in labels): + return table if key in p else None + return [k for k in labels if own.get(k, table)] or False + + +def _bar_switches_to_spec(p: dict, out: dict, switches: dict[str, dict]) -> None: + """cell_bars, color_by_sign and absolute_bars from show_cell_bars, color_pn and align_pn + and the columns' own showCellBars, colorPositiveNegative and alignPositiveNegative. A + stored Superset default reads as unset (color_pn true, align_pn false: the panel's + defaults); show_cell_bars reads back as stored, as it always has. Colour and alignment + count only on the columns that draw a bar, and are read back only where they can show, + so the spec validates.""" + if out.get("columns"): + capable = list(out["columns"]) + else: + capable = [metric_label(m) for m in out.get("metrics") or []] + bars = _bar_switch(p, "show_cell_bars", switches["cell_bars"], capable, True) + if bars is not None: + out["cell_bars"] = bars + barred = [] if bars is False else capable if bars in (None, True) else bars + for spec_field, key, default in (("color_by_sign", "color_pn", True), + ("absolute_bars", "align_pn", False)): + value = _bar_switch(p, key, switches[spec_field], barred, default) + if value is True and not barred: + value = None # asks for bars the table doesn't draw: it shows nothing + if value is not None and value != default: + out[spec_field] = value + + def _x_labels_to_spec(p: dict, out: dict, losses: list, name: str) -> None: # Superset's own defaults (adaptive format, 0 degrees, auto spacing) map to nothing. if p.get("x_axis_time_format") not in (None, "", "smart_date"): @@ -155,12 +226,26 @@ def _x_labels_to_spec(p: dict, out: dict, losses: list, name: str) -> None: "timeseries_area": (True, "stream", "expand")} # others: True, "stream" _CONTRIBUTION_SPEC = {v: k for k, v in CONTRIBUTION_VALUES.items()} _PIVOT_ORDER_SPEC = {v: k for k, v in PIVOT_ORDER.items()} +_HEATMAP_X_ORDER = {v: k for k, v in HEATMAP_X_SORT.items()} # alpha_asc -> a_to_z +_HEATMAP_Y_ORDER = {v: k for k, v in HEATMAP_Y_SORT.items()} # alpha_asc -> z_to_a _LABEL_TYPES = set(get_args(LabelType)) _PIVOT_AGGREGATES = set(get_args(PivotAggregate)) _SEQUENTIAL_SCHEMES = set(get_args(SequentialScheme)) _COLUMN_CONFIG_SPEC = {v: k for k, v in COLUMN_CONFIG_KEYS.items()} # d3NumberFormat -> number_formats +# SortSeriesType (superset-ui-chart-controls types.ts, 4.1.4 :503, 6.1.0 :549): what a bar +# with several series sorts its categories by. +_SERIES_SORTS = ("name", "sum", "min", "max", "avg") + + +def _stored_label(m) -> str | None: + """A stored metric's label, as getMetricLabel reads it: a saved metric is its name.""" + if isinstance(m, str): + return m + return m.get("label") if isinstance(m, dict) else None + + def _number(v) -> float | int | None: """A stored bound or size as a number; None for null, "" or junk.""" if isinstance(v, bool) or v in (None, ""): @@ -279,20 +364,167 @@ def _line_style_to_spec(p: dict, out: dict, spec_type: str) -> set[str]: return {"markerEnabled", "markerSize", "area", "opacity"} -def _rgb_to_spec(colour) -> str | None: - """A color_picker {r, g, b} as the spec writes it: a named shade, a hex, or None - for Superset's own default teal.""" +def _rolling_to_spec(p: dict, out: dict, losses: list, name: str) -> None: + """A trendline KPI's rolling window. rollingWindowOperator reads a missing window as + 1 and missing min periods as 0 (ensureIsInt, operators/rollingWindowOperator.ts at + 4.1.4, 5.0.0 and 6.1.0), and a cumulative sum reads neither.""" + kind = p.get("rolling_type") + if kind in (None, "", "None"): + return + if kind not in ("sum", "mean", "std", "cumsum"): + losses.append(Loss(name, f"rolling_type {kind!r} not preserved (no rolling window on re-apply)")) + return + out["rolling_type"] = kind + if kind == "cumsum": + return + periods = _number(p.get("rolling_periods")) + periods = periods if isinstance(periods, int) and periods >= 1 else 1 + out["rolling_periods"] = periods + least = _number(p.get("min_periods")) + least = least if isinstance(least, int) and least >= 0 else 0 + if least > periods: + losses.append(Loss(name, f"min_periods {least} above the {periods}-step window not " + f"preserved (whole windows only on re-apply)")) + elif least != periods: + out["rolling_min_periods"] = least + + +def _rgb_to_spec(colour, default: str = TREND_DEFAULT_HEX) -> str | None: + """A colour picker's {r, g, b} as the spec writes it: a named shade, a hex, or None + for Superset's own default (a trendline's teal unless another is given).""" if not isinstance(colour, dict): return None try: hexed = "#{:02X}{:02X}{:02X}".format(*(int(colour[k]) for k in "rgb")) except (KeyError, TypeError, ValueError): return None - if hexed == TREND_DEFAULT_HEX: + if hexed == default: return None return {v.upper(): k for k, v in FORMAT_TEXT_HEX.items()}.get(hexed, hexed) +_TICK_LAYOUT_SPEC = {v: k for k, v in TICK_LAYOUTS.items()} | {"staggered": 45} # drawn at 45° + + +def _tick_layout_to_spec(p: dict, out: dict, losses: list, name: str) -> None: + # "staggered" is rotated -45 degrees exactly like "45°" (transformProps.ts, waterfall + # and box plot, all three releases), so it reads back as 45. + layout = p.get("x_ticks_layout") + if layout in _TICK_LAYOUT_SPEC: + out["x_label_rotation"] = _TICK_LAYOUT_SPEC[layout] + elif layout not in (None, "", "auto"): + losses.append(Loss(name, f"x_ticks_layout {layout!r} not preserved (auto on re-apply)")) + + +def _waterfall_to_spec(p: dict, out: dict, losses: list, name: str, metric_one) -> set[str] | None: + """A waterfall's params into out; the params it read, or None when the chart can't + be a spec chart. A bridge's x axis is the steps_order_sql the compiler writes.""" + m = metric_one(p.get("metric")) + x = p.get("x_axis") + groupby = [g for g in (p.get("groupby") or []) if isinstance(g, str)] + if m is None or not x: + losses.append(Loss(name, "waterfall needs metric + x_axis; chart skipped")) + return None + out["metric"] = m + read = {"x_axis_label", "y_axis_label", "increase_color", "decrease_color", "total_color", + "increase_label", "decrease_label", "total_label", "show_legend", "x_ticks_layout"} + bridge = parse_steps_order_sql(x.get("sqlExpression")) if isinstance(x, dict) else None + if bridge is not None: + column, steps, closing, opening = bridge + raw = p.get("groupby") or [] + if opening is None: + ours = groupby == [column] and p.get("total_label") == closing + else: + ours = (len(raw) == 1 and isinstance(raw[0], dict) and p.get("total_label") == BRIDGE_TOTAL + and raw[0].get("sqlExpression") == bridge_totals_sql(column, opening, closing)) + if not ours or p.get("show_total") is not False: + losses.append(Loss(name, "waterfall bridge order changed outside the spec (breakdown, " + "show_total or total_label); chart skipped")) + return None + out.update({"x_column": column, "steps": steps, "closing": closing}) + if opening is not None: + out["opening"] = opening + read.add("show_total") + elif isinstance(x, str): + out["x_column"] = x + if len(groupby) > 1: + losses.append(Loss(name, f"multiple breakdowns {groupby}; kept first only")) + if groupby: + out["groupby"] = groupby[0] + if p.get("time_grain_sqla"): + out["time_grain"] = p["time_grain_sqla"] + if isinstance(p.get("total_label"), str) and p["total_label"].strip(): + out["total_label"] = p["total_label"] + else: + losses.append(Loss(name, f"waterfall x axis {x!r} is SQL the spec can't express; chart skipped")) + return None + for field, stock in WATERFALL_DEFAULT_HEX.items(): + colour = _rgb_to_spec(p.get(field), stock) + if colour: + out[field] = colour + for field in ("increase_label", "decrease_label"): + if isinstance(p.get(field), str) and p[field].strip(): + out[field] = p[field] + if p.get("show_value") is True: + out["show_value"] = True + if p.get("show_legend") is True: + out["show_legend"] = True + if p.get("y_axis_format") not in (None, "", "SMART_NUMBER"): + out["number_format"] = p["y_axis_format"] + for key, field in (("x_axis_label", "x_axis_title"), ("y_axis_label", "y_axis_title")): + if isinstance(p.get(key), str) and p[key].strip(): + out[field] = p[key] + _tick_layout_to_spec(p, out, losses, name) + if p.get("x_axis_time_format") not in (None, "", "smart_date"): + out["x_label_format"] = p["x_axis_time_format"] + if p.get("row_limit"): + out["row_limit"] = p["row_limit"] + return read + + +_WHISKER_SPEC = {v: k for k, v in WHISKER_OPTIONS.items()} +_PERCENTILES = re.compile(r"^(\d+)/(\d+) percentiles$") # the boxplot operator's own reading + + +def _box_plot_to_spec(p: dict, out: dict, losses: list, name: str, metric_one) -> set[str] | None: + """A box plot's params into out; the params it read, or None when the chart can't be + a spec chart (an observation or group the spec can't name: a SQL column).""" + ms = [m for m in (metric_one(m) for m in (p.get("metrics") or [])) if m] + across, groupby = p.get("columns") or [], p.get("groupby") or [] + if not ms or not across: + losses.append(Loss(name, "box plot needs metrics + distribute across; chart skipped")) + return None + if not all(isinstance(c, str) for c in [*across, *groupby]): + losses.append(Loss(name, "box plot over SQL columns the spec can't express; chart skipped")) + return None + out.update({"metrics": ms, "distribute_across": list(across)}) + if groupby: + out["groupby"] = list(groupby) + whiskers = p.get("whiskerOptions") or "Tukey" + percentiles = _PERCENTILES.match(whiskers) if isinstance(whiskers, str) else None + if whiskers in _WHISKER_SPEC: + if whiskers != "Tukey": + out["whiskers"] = _WHISKER_SPEC[whiskers] + elif percentiles and int(percentiles.group(1)) < int(percentiles.group(2)) <= 100: + out["whiskers"] = [int(percentiles.group(1)), int(percentiles.group(2))] + else: + losses.append(Loss(name, f"whisker option {whiskers!r} not preserved (Tukey on re-apply)")) + if p.get("time_grain_sqla"): + out["time_grain"] = p["time_grain_sqla"] + if p.get("number_format") not in (None, "", "SMART_NUMBER"): + out["number_format"] = p["number_format"] + if p.get("date_format") not in (None, "", "smart_date"): + out["x_label_format"] = p["date_format"] + for key in ("x_axis_title", "y_axis_title"): + if isinstance(p.get(key), str) and p[key].strip(): + out[key] = p[key] + _tick_layout_to_spec(p, out, losses, name) + if p.get("row_limit"): + out["row_limit"] = p["row_limit"] + return {"columns", "whiskerOptions", "date_format", "x_axis_title", "y_axis_title", + "x_ticks_layout"} + + def _annotations_to_spec(p: dict, out: dict, losses: list, name: str) -> None: """FORMULA annotation layers -> spec annotations; any other layer (events, intervals, other charts' series) is named as a loss.""" @@ -398,19 +630,37 @@ def losses_json(self) -> list[dict]: """dataset_uuid -> {'database','schema','table'} or None.""" -def _format_to_spec(cf: dict) -> dict | None: - """Superset conditional_formatting entry -> FormatRule dict, None if outside surface.""" - text = cf.get("objectFormatting") == "TEXT_COLOR" - palette = FORMAT_TEXT_HEX if text else FORMAT_COLOR_HEX +def _band_colour(cf: dict, palette: dict[str, str]) -> str | None: + """A rule's colour as the spec writes it: a named shade of `palette` reads back as + its name, any other #RRGGBB as itself; None for anything else (a 6.1 theme token).""" scheme = cf.get("colorScheme") if isinstance(cf.get("colorScheme"), str) else "" - # A named shade for this paint reads back as its name; any other #RRGGBB as itself. color = {v.upper(): k for k, v in palette.items()}.get(scheme.upper()) if color is None and HEX_COLOUR_RE.fullmatch(scheme): color = scheme.upper() - # '< x <' is Superset's range comparator; the bare "between" older - # chartwright builds wrote (and Superset never matched) reads back the same. - op = {"< x <": "between"}.get(cf.get("operator"), cf.get("operator")) - if not color or not cf.get("column") or op not in ("<", ">", "=", "between"): + return color + + +def _band_targets(cf: dict, op: str) -> dict | None: + """A rule's threshold(s) as spec fields, None when Superset's entry lacks them.""" + if op in FORMAT_RANGE_OPERATORS: + left, right = cf.get("targetValueLeft"), cf.get("targetValueRight") + return None if left is None or right is None else {"target_left": left, "target_right": right} + return None if cf.get("targetValue") is None else {"target": cf["targetValue"]} + + +def _band_operator(cf: dict) -> str | None: + # Superset's comparators back to the spec's operators; the bare "between" older + # chartwright builds wrote (and Superset never matched) reads back as between too. The + # half-open ranges ('≤ x <', '< x ≤') have no spec operator and stay outside. + return _FORMAT_OPERATOR_BACK.get(cf.get("operator")) + + +def _format_to_spec(cf: dict) -> dict | None: + """Superset conditional_formatting entry -> FormatRule dict, None if outside surface.""" + text = cf.get("objectFormatting") == "TEXT_COLOR" + color = _band_colour(cf, FORMAT_TEXT_HEX if text else FORMAT_COLOR_HEX) + op = _band_operator(cf) + if not color or not cf.get("column") or op is None: return None if cf.get("toTextColor") or cf.get("objectFormatting") not in (None, "BACKGROUND_COLOR", "TEXT_COLOR"): return None # legacy text flag / cell bars: outside the surface @@ -420,16 +670,29 @@ def _format_to_spec(cf: dict) -> dict | None: target_col = cf.get("columnFormatting") or ("ENTIRE_ROW" if cf.get("toAllRow") else None) if target_col: rule["apply_to"] = "row" if target_col == "ENTIRE_ROW" else target_col - if op == "between": - rule["target_left"] = cf.get("targetValueLeft") - rule["target_right"] = cf.get("targetValueRight") - if rule["target_left"] is None or rule["target_right"] is None: - return None - else: - rule["target"] = cf.get("targetValue") - if rule["target"] is None: - return None - return rule + targets = _band_targets(cf, op) + return None if targets is None else rule | targets + + +def _big_number_rules_to_spec(p: dict, losses: list, name: str) -> list[dict]: + """A big number's conditional_formatting as BigNumberFormatRule dicts. The plugin + paints the number from every rule whose column is set, whichever column it names, + always solid and never by cell or row (BigNumberViz.tsx; getColorFormatters.ts + 4.1.4 and 5.0.0 :199, 6.1.0 :309), so the column, the gradient and the cell options + say nothing about the number; a rule without a column paints nothing.""" + rules = [] + for cf in p.get("conditional_formatting") or []: + cf = cf if isinstance(cf, dict) else {} + color, op = _band_colour(cf, FORMAT_TEXT_HEX), _band_operator(cf) + targets = _band_targets(cf, op) if op else None + if not cf.get("column"): + losses.append(Loss(name, f"conditional format with no column, which Superset never " + f"paints, dropped: {cf}")) + elif color is None or targets is None: + losses.append(Loss(name, f"conditional format not representable, dropped: {cf}")) + else: + rules.append({"operator": op, "color": color, **targets}) + return rules def _rules_to_spec(p: dict, losses: list, name: str, labels: set[str], @@ -454,6 +717,12 @@ def _rules_to_spec(p: dict, losses: list, name: str, labels: set[str], "no longer queries, dropped")) else: rules.append(rule) + # 6.1's popover turns the gradient on for a new rule (FormattingPopoverContent.tsx + # :250-252 at 6.1.0); a rule is a solid band in the spec. An '=' rule is solid either way. + if cf.get("useGradient") is True and rule["operator"] != "=": + losses.append(Loss(name, f"conditional format on {rule['metric']!r} fades by " + "distance from its threshold (gradient), not preserved: " + "apply paints it one solid colour")) return rules @@ -482,10 +751,23 @@ def _metric_to_spec(m, losses: list[Loss], chart: str) -> str | None: if agg in ADHOC_AGGREGATES and col: if m.get("hasCustomLabel") and m.get("label"): return f"{agg}({col}) AS {m['label']}" + # The label is the series name Superset shows; compile stores the spec's + # own spelling there (`SUM( sales )`), so that spelling reads back. + spelled = m.get("label") + parsed = parse_metric(spelled) if isinstance(spelled, str) else None + if parsed and parsed.get("sql") is None and not parsed["label"] \ + and (parsed["aggregate"], parsed["column"]) == (agg, col): + return spelled return f"{agg}({col})" if m.get("expressionType") == "SQL": sql = (m.get("sqlExpression") or "").strip() match = re.fullmatch(r"(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX)\(\s*(\*|\w+)\s*\)", sql, re.I) + if match and str(m.get("optionName") or "").startswith(METRIC_OPTION_PREFIX): + # The tool's own metric. It writes an aggregate as SQL only for `AGG(*)`, in + # exactly that spelling (compile _metric_payload); any other was the spec's + # `SQL(MAX(col)) AS Label` and reads back as written, so `plan` matches it. + # One built in the UI still reads as the simpler AGG(col). + match = match if sql in {f"{a}(*)" for a in ADHOC_AGGREGATES} else None if match: base = f"{match.group(1).upper()}({match.group(2)})" if m.get("hasCustomLabel") and m.get("label"): @@ -595,8 +877,20 @@ def keep_row_limit() -> None: else p.get("subheader") if text: out["subtitle"] = text - if p.get("y_axis_format"): + fmt = p.get("time_format") if isinstance(p.get("time_format"), str) else None + if p.get("force_timestamp_formatting") is True: + # Forced, the number shows as a date whatever its type, so number_format is inert. + out["date_format"] = fmt or "smart_date" + elif fmt not in (None, "", "smart_date"): + # Unforced, the format applies only when the metric itself is a date or a string, + # which the spec can't tell offline; date_format would force it on a number too. + losses.append(Loss(name, f"time_format {fmt!r} without Force date format not " + f"preserved; set date_format to keep it")) + if p.get("y_axis_format") not in (None, "", "SMART_NUMBER") and "date_format" not in out: out["number_format"] = p["y_axis_format"] + rules = _big_number_rules_to_spec(p, losses, name) + if rules: + out["conditional_formatting"] = rules elif spec_type == "big_number_trend": m = metric_one(p.get("metric")) x = p.get("x_axis") @@ -607,7 +901,7 @@ def keep_row_limit() -> None: out["time_column"] = x if p.get("time_grain_sqla"): out["time_grain"] = p["time_grain_sqla"] - if p.get("y_axis_format"): + if p.get("y_axis_format") not in (None, "", "SMART_NUMBER"): out["number_format"] = p["y_axis_format"] lag = _number(p.get("compare_lag")) if isinstance(lag, int) and lag >= 1: @@ -619,6 +913,10 @@ def keep_row_limit() -> None: colour = _rgb_to_spec(p.get("color_picker")) if colour: out["trend_color"] = colour + # "Start y-axis at 0" unticked (an unset key is the control's default, true). + if p.get("start_y_axis_at_zero") is False: + out["y_axis_truncate"] = True + _rolling_to_spec(p, out, losses, name) elif spec_type in ("timeseries_line", "timeseries_bar", "timeseries_area", "timeseries_scatter"): ms = [metric_one(m) for m in (p.get("metrics") or [])] ms = [m for m in ms if m] @@ -655,12 +953,37 @@ def keep_row_limit() -> None: # series: x_axis_sort from 6.0.0, x_axis_sort_series at 4.1.4 and 5.0.0). several = bool(p.get("groupby")) or len(ms) > 1 sort, stored_asc = p.get("x_axis_sort"), p.get("x_axis_sort_asc") - if several and sort in (None, "") and p.get("x_axis_sort_series") == "name": - sort, stored_asc = "name", p.get("x_axis_sort_series_ascending", True) + if several and sort in (None, "") and p.get("x_axis_sort_series") in _SERIES_SORTS: + sort = p["x_axis_sort_series"] + stored_asc = p.get("x_axis_sort_series_ascending", True) if sort == x or (several and sort == "name"): # In reading order (the horizontal axis runs bottom-up). ascending = bool(stored_asc) != (p.get("orientation") == "horizontal") out["category_sort"] = "asc" if ascending else "desc" + elif sort not in (None, ""): + # A ranking by a measure, largest first (on top of a horizontal bar): the + # series' sum ("total"), or a metric, which compile also makes the query's + # "Sort query by" (timeseries_limit_metric), drawn or not. + ranked = p.get("timeseries_limit_metric") + first = metric_label(ms[0]) + if several and sort == "sum": + out["sort_by"] = "total" + elif several and sort in _SERIES_SORTS: + losses.append(Loss(name, f"bars ranked by the series' {sort!r} value not " + "preserved (the first metric on re-apply)")) + elif ranked and _stored_label(ranked) == sort and not p.get("groupby"): + m = metric_one(ranked) + if m: + out["sort_by"] = m + elif sort != first and not p.get("groupby"): + m = next((m for m in ms if metric_label(m) == sort), None) + if m: + out["sort_by"] = m + largest_first = p.get("orientation") == "horizontal" + if stored_asc is not None and bool(stored_asc) != largest_first and ( + "sort_by" in out or sort == first): + losses.append(Loss(name, "bars ranked smallest first not preserved " + "(largest first on re-apply)")) if p.get("y_axis_format") not in (None, "SMART_NUMBER"): out["number_format"] = p["y_axis_format"] gb = _groupby_one(p, losses, name) @@ -720,6 +1043,7 @@ def keep_row_limit() -> None: out["conditional_formatting"] = rules hidden: list = [] per_label: dict[str, dict] = {spec_key: {} for spec_key in COLUMN_CONFIG_KEYS} + switches: dict[str, dict] = {spec_key: {} for spec_key in BAR_SWITCH_KEYS} for label, cfg in (p.get("column_config") or {}).items(): cfg = cfg if isinstance(cfg, dict) else {} if label not in labels: @@ -740,7 +1064,12 @@ def keep_row_limit() -> None: per_label["column_widths"][label] = width if isinstance(cfg.get("customColumnName"), str) and cfg["customColumnName"].strip(): per_label["column_headers"][label] = cfg["customColumnName"] - extra = sorted(k for k in cfg if k not in ("visible", *_COLUMN_CONFIG_SPEC)) + for spec_field, key in BAR_SWITCH_KEYS.items(): + if key in cfg: + # Superset tests `=== undefined`, so a stored null is a false. + switches[spec_field][label] = bool(cfg[key]) + extra = sorted(k for k in cfg if k not in ("visible", *_COLUMN_CONFIG_SPEC, + *BAR_SWITCH_KEYS.values())) if extra: losses.append(Loss(name, f"column_config {label!r} settings not preserved: {extra}")) if hidden: @@ -748,6 +1077,7 @@ def keep_row_limit() -> None: for spec_field, values in per_label.items(): if values: out[spec_field] = values + _bar_switches_to_spec(p, out, switches) page = _number(p.get("page_length")) if isinstance(page, int) and page >= 0: out["page_length"] = page @@ -755,8 +1085,6 @@ def keep_row_limit() -> None: out["show_totals"] = True if p.get("include_search") is True: out["search_box"] = True - if "show_cell_bars" in p: - out["cell_bars"] = bool(p["show_cell_bars"]) if p.get("table_timestamp_format") not in (None, "", "smart_date"): out["date_format"] = p["table_timestamp_format"] keep_row_limit() @@ -832,6 +1160,29 @@ def keep_row_limit() -> None: out["number_format"] = p["y_axis_format"] if p.get("show_legend") is False: out["show_legend"] = False + # Unset (5.0.0 and later have no default) reads as the tool's alpha_asc. + for key, field, orders in (("sort_x_axis", "x_order", _HEATMAP_X_ORDER), + ("sort_y_axis", "y_order", _HEATMAP_Y_ORDER)): + order = orders.get(p.get(key) or "alpha_asc") + if order is None: + losses.append(Loss(name, f"heatmap {key} {p[key]!r} not preserved (axis " + f"ascending on re-apply)")) + elif order != SUPERSET_DEFAULTS["heatmap"][field]: + out[field] = order + # Superset's own values (-1 = automatic spacing, 'auto' = no margin) map to nothing. + for key, spec_field in (("xscale_interval", "x_label_every"), + ("yscale_interval", "y_label_every")): + every = _number(p.get(key)) + if isinstance(every, int) and 1 <= every <= 50: + out[spec_field] = every + elif every not in (None, -1): + losses.append(Loss(name, f"heatmap {key} {p[key]!r} not preserved (automatic on re-apply)")) + margin = _number(p.get("left_margin")) + if isinstance(margin, int) and 0 <= margin <= 200: + out["left_margin"] = margin + elif p.get("left_margin") not in (None, "", "auto"): + losses.append(Loss(name, f"heatmap left_margin {p['left_margin']!r} not preserved " + "(none on re-apply)")) keep_row_limit() elif spec_type == "histogram": col = p.get("column") @@ -893,12 +1244,23 @@ def keep_row_limit() -> None: losses.append(Loss(name, f"mixed chart query {key.upper()} has no representable metrics; chart skipped")) return None series: dict = {"metrics": ms} - kind = p.get(kind_key) or "line" # the plugin's own default series type - if kind not in ("bar", "line"): - losses.append(Loss(name, f"query {key.upper()} series type {kind!r} not preserved (line on re-apply)")) - kind = "line" + stored = p.get(kind_key) or "line" # the plugin's own default series type + # transformSeries draws 'bar' as bars, and scatter, smooth and the steps their + # own way; any other value, the old echarts_timeseries_* names included, as a + # straight line (Timeseries/transformers.ts:237-243 at 4.1.4, :306-312 at 6.1.0). + kind = "bar" if stored == "bar" else "line" + if stored in ("scatter", "smooth", "start", "middle", "end"): + losses.append(Loss(name, f"query {key.upper()} series type {stored!r} not " + f"preserved (a straight line on re-apply)")) + disp = kind_key.replace("seriesType", "") # "" for query A, "B" for query B + # "Area chart" fills under a line; a bar draws no area, so there it does nothing. + if kind == "line" and p.get(f"area{disp}") is True: + kind = "area" if kind != "bar": series["kind"] = kind + opacity = _number(p.get(f"opacity{disp}")) + if kind == "area" and opacity is not None and 0 <= opacity <= 1 and opacity != 0.2: + series["opacity"] = opacity if p.get(axis_key) == 1: series["axis"] = "secondary" gb = [g for g in (p.get(f"groupby{sfx}") or []) if isinstance(g, str)] @@ -906,7 +1268,6 @@ def keep_row_limit() -> None: losses.append(Loss(name, f"query {key.upper()}: multiple groupby {gb}; kept first only")) if gb: series["groupby"] = gb[0] - disp = kind_key.replace("seriesType", "") # "" for query A, "B" for query B if p.get(f"markerEnabled{disp}"): series["markers"] = True if p.get(f"show_value{disp}") is True: @@ -926,16 +1287,26 @@ def keep_row_limit() -> None: if p.get("y_axis_format_secondary") not in (None, "SMART_NUMBER"): out["number_format_secondary"] = p["y_axis_format_secondary"] keep_row_limit() + elif spec_type == "waterfall": + waterfall_read = _waterfall_to_spec(p, out, losses, name, metric_one) + if waterfall_read is None: + return None + elif spec_type == "box_plot": + box_plot_read = _box_plot_to_spec(p, out, losses, name, metric_one) + if box_plot_read is None: + return None mapped_here = {"combineMetric", "conditional_formatting", "rowTotals", "colTotals", "colSubTotals"} if spec_type == "pivot_table" else ( {"order_by_cols", "timeseries_limit_metric", "series_limit_metric", "conditional_formatting", "column_config"} if spec_type == "table" else set()) if spec_type == "mixed": - mapped_here = {"metrics_b", "groupby_b", "adhoc_filters_b", "row_limit_b", "seriesTypeB", + mapped_here = {"metrics_b", "groupby_b", "adhoc_filters_b", "row_limit_b", "seriesType", + "seriesTypeB", "yAxisIndex", "yAxisIndexB", "y_axis_format_secondary", "markerEnabled", "markerEnabledB"} - mapped_here |= {f"{k}{s}" for k in ("show_value", "stack", "only_total") for s in ("", "B")} + mapped_here |= {f"{k}{s}" for k in ("show_value", "stack", "only_total", "area", "opacity") + for s in ("", "B")} mapped_here |= {f"{k}{s}" for k in ("limit", "timeseries_limit_metric", "order_desc") for s in ("", "_b")} if p.get("time_range") not in (None, "", "No filter"): @@ -947,17 +1318,31 @@ def keep_row_limit() -> None: if spec_type in LEGEND_TYPES: mapped_here = mapped_here | _legend_to_spec(p, out, spec_type) if spec_type == "big_number_trend": - mapped_here = mapped_here | {"compare_lag", "compare_suffix", "subtitle", "color_picker"} + # conditional_formatting: the trendline plugin never reads it (no control, and no + # colour rules in its transformProps at 4.1.4, 5.0.0 and 6.1.0), so rules left from + # a Big Number the chart was switched from colour nothing. + mapped_here = mapped_here | {"compare_lag", "compare_suffix", "subtitle", "color_picker", + "rolling_type", "rolling_periods", "min_periods", + "start_y_axis_at_zero", "conditional_formatting"} if spec_type == "big_number_total": - mapped_here = mapped_here | {"subtitle"} + mapped_here = mapped_here | {"subtitle", "time_format", "force_timestamp_formatting", + "conditional_formatting"} if spec_type == "table": - mapped_here = mapped_here | {"page_length", "show_totals", "include_search"} + mapped_here = mapped_here | {"page_length", "show_totals", "include_search", "color_pn", + "align_pn"} if spec_type == "pivot_table": mapped_here = mapped_here | {"rowSubTotals", "transposePivot"} if spec_type == "histogram": mapped_here = mapped_here | {"x_axis_title", "y_axis_title"} if spec_type == "pie": mapped_here = mapped_here | {"show_total"} + if spec_type == "heatmap": + mapped_here = mapped_here | {"sort_x_axis", "sort_y_axis", "xscale_interval", "yscale_interval", + "left_margin"} + if spec_type == "waterfall": + mapped_here = mapped_here | waterfall_read + if spec_type == "box_plot": + mapped_here = mapped_here | box_plot_read if spec_type in _AXIS_TYPES: _x_labels_to_spec(p, out, losses, name) mapped_here = mapped_here | _axis_to_spec(p, out, losses, name, spec_type) @@ -1217,6 +1602,29 @@ def _geo(meta: dict) -> dict: return geo +def _header_to_spec(meta: dict) -> dict | None: + """A HEADER node's meta as a spec header block; None when it has no text.""" + text = meta.get("text") + if not isinstance(text, str) or not text.strip(): + return None + header: dict = {"header": text} + # Superset draws a header with no stored size as small (Header.tsx). + size = _REVERSE_HEADER_SIZE.get(meta.get("headerSize") or "SMALL_HEADER", "small") + if size != "medium": + header["size"] = size + if meta.get("background") == BACKGROUND["white"]: + header["background"] = "white" + return header + + +def _markdown_height(meta: dict) -> int | float | None: + """A MARKDOWN node's height in spec units: exact, since text blocks take fifths.""" + if not meta.get("height"): + return None + h = int(meta["height"]) / ROW_UNITS_PER_SPEC_UNIT + return int(h) if h.is_integer() else round(h, 1) + + def _walk_rows(position: dict, children: list[str], kept_names: set[str], losses: list[Loss], geometry: dict[str, dict]) -> list: """Convert grid- or tab-level nodes into spec rows: ROWs (chart names + @@ -1248,9 +1656,8 @@ def handle(children_ids: list[str], depth: int = 0) -> None: block: dict = {"markdown": meta.get("code") or ""} if meta.get("width"): block["width"] = max(1, min(12, int(meta["width"]))) - if meta.get("height"): - h = int(meta["height"]) / ROW_UNITS_PER_SPEC_UNIT # exact: text blocks take fifths - block["height"] = int(h) if h.is_integer() else round(h, 1) + if _markdown_height(meta) is not None: + block["height"] = _markdown_height(meta) if block["markdown"]: row.append(block) else: @@ -1272,18 +1679,10 @@ def handle(children_ids: list[str], depth: int = 0) -> None: rows.append([nm]) geometry[nm] = _geo(meta) elif t == "HEADER" and depth == 0: - meta = node.get("meta") or {} - text = meta.get("text") - if not isinstance(text, str) or not text.strip(): + header = _header_to_spec(node.get("meta") or {}) + if header is None: losses.append(Loss("layout", "empty HEADER dropped")) continue - header: dict = {"header": text} - # Superset draws a header with no stored size as small (Header.tsx). - size = _REVERSE_HEADER_SIZE.get(meta.get("headerSize") or "SMALL_HEADER", "small") - if size != "medium": - header["size"] = size - if meta.get("background") == BACKGROUND["white"]: - header["background"] = "white" rows.append(header) elif t == "DIVIDER" and depth == 0: rows.append({"divider": True}) @@ -1365,58 +1764,201 @@ def _sketch_from(position: dict, children: list[str], kept_names: set[str], """A section of rows holding charts and COLUMNs of charts, as a sketch: the only spec layout that stacks charts beside a taller one (triage J). Each chart keeps its exact height as an explicit `height`, which wins over the sketch's line height, so - a line per stacked chart is enough. None when the section holds anything a sketch - can't (markdown, a header, a divider, a white row, a chart decompile can't keep), - and the caller walks it as rows instead.""" - from math import lcm + a line per stacked chart is enough; a markdown block keeps its own the same way. + Markdown and headers in the rows and columns become legend blocks, a header + between the rows a header across the sketch, each header one line. None when the + section holds anything a sketch can't (a divider, a white row, a chart outside a + row, a chart decompile can't keep, an empty block), and the caller walks it as + rows instead.""" + from .sketch import BlockRef, SketchError, parse_sketch + + def entry(node: dict): + """('chart', name) | ('markdown', block) | ('header', block), or None.""" + meta = node.get("meta") or {} + t = node.get("type") + if t == "CHART": + return ("chart", meta.get("sliceName")) if meta.get("sliceName") in kept_names else None + if t == "MARKDOWN": + if not (meta.get("code") or "").strip(): + return None + block: dict = {"markdown": meta["code"]} + if _markdown_height(meta) is not None: + block["height"] = _markdown_height(meta) + return ("markdown", block) + if t == "HEADER": + header = _header_to_spec(meta) + return ("header", header) if header else None + return None - rows: list[list[tuple[int, list[str]]]] = [] - for row_id in children: - node = position.get(row_id) or {} + bands: list = [] # ("header", block) | ("row", [(width, [entry, ...]), ...]) + for node_id in children: + node = position.get(node_id) or {} + if node.get("type") == "HEADER": + found = entry(node) + if found is None: + return None + bands.append(("header", found[1])) + continue if node.get("type") != "ROW" or (node.get("meta") or {}).get("background") == BACKGROUND["white"]: return None - items: list[tuple[int, list[str]]] = [] + items: list[tuple[int, list]] = [] for ch_id in node.get("children", []): ch = position.get(ch_id) or {} - if ch.get("type") == "CHART": + if ch.get("type") in ("CHART", "MARKDOWN"): stack = [ch] elif ch.get("type") == "COLUMN": stack = [position.get(x) or {} for x in ch.get("children", [])] - if not stack or any(x.get("type") != "CHART" for x in stack): + if not stack: return None else: return None - names = [(x.get("meta") or {}).get("sliceName") for x in stack] - if any(n not in kept_names for n in names): + entries = [entry(x) for x in stack] + if any(e is None for e in entries): return None width = int((ch.get("meta") or {}).get("width") or 0) if width < 1: return None - items.append((width, names)) + items.append((width, entries)) if not items or sum(w for w, _ in items) > 12: return None - rows.append(items) - names = [n for items in rows for _, stack in items for n in stack] - if not rows or len(names) != len(set(names)) or len(names) > len(_SKETCH_SYMBOLS): + bands.append(("row", items)) + names = [e[1] for kind, band in bands if kind == "row" + for _, stack in band for e in stack if e[0] == "chart"] + blocks = sum(1 if kind == "header" else sum(1 for _, stack in band for e in stack + if e[0] != "chart") + for kind, band in bands) + if not bands or len(names) != len(set(names)) or len(names) + blocks > len(_SKETCH_SYMBOLS): return None - symbol = {n: _SKETCH_SYMBOLS[i] for i, n in enumerate(names)} + symbols = iter(_SKETCH_SYMBOLS) + symbol = {n: next(symbols) for n in names} + legend: dict = {symbol[n]: n for n in names} + + def sym(e) -> str: + if e[0] == "chart": + return symbol[e[1]] + s = next(symbols) + legend[s] = e[1] + return s + lines: list[str] = [] - for items in rows: - depth = lcm(*(len(stack) for _, stack in items)) - for line in range(depth): - cells = "".join(symbol[stack[line * len(stack) // depth]] * width - for width, stack in items) - lines.append(cells.ljust(12, ".")) + for kind, band in bands: + if kind == "header": + lines.append(sym(("header", band)) * 12) + continue + stacks = [(width, [(e[0], sym(e)) for e in stack]) for width, stack in band] + span, counts = _band_lines([[k for k, _ in stack] for _, stack in stacks]) + drawn = [] + for (width, stack), per_item in zip(stacks, counts): + cells: list[str] = [] + for count, (_, s) in zip(per_item, stack): + cells += [s * width] * count + cells += ["." * width] * (span - len(cells)) # a stack that ends early + drawn.append(cells) + for line in range(span): + lines.append("".join(cells[line] for cells in drawn).ljust(12, ".")) + try: + # A band no drawing keeps whole (a header alone beside a stacked column) splits + # into bands, and the next one would open on the dots below the header. + parse_sketch(lines, {s: v if isinstance(v, str) + else BlockRef("header" if "header" in v else "markdown") + for s, v in legend.items()}, 2) + except SketchError: + return None for row_id in children: - for ch_id in (position.get(row_id) or {}).get("children", []): + node = position.get(row_id) or {} + for ch_id in node.get("children", []) if node.get("type") == "ROW" else []: ch = position.get(ch_id) or {} - for x in ([ch] if ch.get("type") == "CHART" else + for x in ([ch] if ch.get("type") != "COLUMN" else [position.get(c) or {} for c in ch.get("children", [])]): + if x.get("type") != "CHART": + continue meta = x.get("meta") or {} geo = _geo(meta) geo.pop("width", None) # the sketch holds widths geometry[meta.get("sliceName")] = geo - return {"sketch": lines, "legend": {symbol[n]: n for n in names}} + return {"sketch": lines, "legend": legend} + + +def _band_lines(stacks: list[list[str]]) -> tuple[int, list[list[int]]]: + """(lines, lines per item of each stack) to draw one band of stacks (each stack: its + items' kinds) so that the sketch parser reads it back as ONE band: it cuts a band + at every line boundary no item spans, so each boundary inside the band must fall + inside an item of some stack. A header takes one line. First the even drawing: the + stacks' items share the lines evenly (without headers, over the least common + multiple of the stack sizes); where that leaves a boundary no item spans, a + staggered one (_stagger).""" + from math import lcm + + sizes = [sum(1 for k in stack if k != "header") for stack in stacks] + heads = [len(stack) - n for stack, n in zip(stacks, sizes)] + if not any(heads): + span = lcm(*sizes) + else: + need = max(h + n for h, n in zip(heads, sizes)) + span = next((n_ for n_ in range(need, need + lcm(1, *(n for n in sizes if n)) + 1) + if all(n == 0 or (n_ - h) % n == 0 for h, n in zip(heads, sizes))), need) + even = [_stack_lines(stack, span) for stack in stacks] + if _one_band(even, span): + return span, even + return _stagger(stacks) + + +def _one_band(counts: list[list[int]], span: int) -> bool: + """Whether every line boundary inside the band falls inside some item.""" + inside: set[int] = set() + for per_item in counts: + top = 0 + for n in per_item: + inside.update(range(top + 1, top + n)) + top += n + return all(r in inside for r in range(1, span)) + + +def _stagger(stacks: list[list[str]]) -> tuple[int, list[list[int]]]: + """A drawing that offsets the stacks' item boundaries, line by line: an item ends + after its line (a header always does), each stack's last item runs to the end of the + band, and where every item would end together, the stack with the most lines to + spare runs its item on to span the boundary. The band ends once each stack has + drawn its last item. A boundary only headers or a lone stack reach is a cut no + drawing avoids; the rows it makes lay the items out as the one row did.""" + k = len(stacks) + item, drawn = [0] * k, [0] * k + counts: list[list[int]] = [[] for _ in stacks] + lines = 0 + while True: + lines += 1 + live = [s for s in range(k) if item[s] < len(stacks[s])] + for s in live: + drawn[s] += 1 + if all(item[s] == len(stacks[s]) - 1 for s in live): + for s in live: + counts[s].append(drawn[s]) + return lines, counts + ending = [s for s in live + if stacks[s][item[s]] == "header" or item[s] < len(stacks[s]) - 1] + if len(ending) == len(live) > 1: + spare = [s for s in ending if stacks[s][item[s]] != "header"] + if spare: + ending.remove(max(spare, key=lambda s: (item[s] - len(stacks[s]), -s))) + for s in ending: + counts[s].append(drawn[s]) + item[s] += 1 + drawn[s] = 0 + + +def _stack_lines(kinds: list[str], span: int) -> list[int]: + """Lines per item of one stack over `span` lines: a header one, the others the + rest, evenly (earlier items take any remainder); a stack of headers alone ends early.""" + n = sum(1 for k in kinds if k != "header") + base, extra = divmod(span - (len(kinds) - n), n) if n else (0, 0) + out, seen = [], 0 + for k in kinds: + if k == "header": + out.append(1) + else: + out.append(base + (1 if seen < extra else 0)) + seen += 1 + return out def _section(position: dict, children: list[str], kept_names: set[str], @@ -1433,7 +1975,8 @@ def _section(position: dict, children: list[str], kept_names: set[str], def _section_rows(section: dict) -> list: - """A section's rows, a sketch read as one row of its charts in reading order.""" + """A section's rows, a sketch read as one row of its charts and blocks in reading + order.""" if "sketch" in section: return [list(section["legend"][ch] for ch in dict.fromkeys( c for line in section["sketch"] for c in line if c in section["legend"]))] @@ -1592,10 +2135,16 @@ def body_and_footer() -> list: if geo.get("width"): c["width"] = max(1, min(12, int(geo["width"]))) if geo.get("height"): - h = max(1, round(int(geo["height"]) / ROW_UNITS_PER_SPEC_UNIT)) - c["height"] = h - if int(geo["height"]) != h * ROW_UNITS_PER_SPEC_UNIT: - losses.append(Loss(name, f"height {geo['height']} rounded to {h * ROW_UNITS_PER_SPEC_UNIT} row units")) + # Exact, in fifths: a chart's height is whole grid rows (8 px), so 23 rows read + # back as 4.6, as compile, absorb and a text block's height have it. Rounding to + # whole units made every fifth-unit height a change `plan` reported after apply. + rows = int(geo["height"]) + c["height"] = spec_units(max(rows, ROW_UNITS_PER_SPEC_UNIT)) + if rows < ROW_UNITS_PER_SPEC_UNIT: + # The spec's floor is one unit, Superset's own resize floor (GRID_MIN_ROW_UNITS + # = 5, dashboard/util/constants.ts:42 at 4.1.4, 5.0.0 and 6.1.0). + losses.append(Loss(name, f"height {geo['height']} row units raised to " + f"{ROW_UNITS_PER_SPEC_UNIT}, the spec's minimum")) if geo.get("display_name"): c["display_name"] = geo["display_name"] diff --git a/chartwright/design/__init__.py b/chartwright/design/__init__.py index 36ade40..4b0e968 100644 --- a/chartwright/design/__init__.py +++ b/chartwright/design/__init__.py @@ -392,3 +392,4 @@ def gate_block(advice: dict) -> str | None: from . import rules # noqa: E402,F401 from . import defaults # noqa: E402,F401 from . import standard_rules # noqa: E402,F401 +from . import readability # noqa: E402,F401 diff --git a/chartwright/design/brief.py b/chartwright/design/brief.py index e157c6c..6b8cf6f 100644 --- a/chartwright/design/brief.py +++ b/chartwright/design/brief.py @@ -19,6 +19,14 @@ def _guideline(name: str) -> str: return (resources.files("chartwright.design") / "guidelines" / name).read_text(encoding="utf-8") +def _kpi_units(kind: str, p) -> str: + """The height readability.kpi-text asks of a big number with a line under its value.""" + from .readability import KPI_MAX_HEIGHT, kpi_units + + units = kpi_units(kind, True, p) + return str(units) if units else f"more than {KPI_MAX_HEIGHT}" + + def render_brief(audience: str, overlay: Overlay | None = None) -> str: overlay = overlay if overlay is not None else load_overlay() p = params_for(audience, overlay) @@ -37,14 +45,18 @@ def render_brief(audience: str, overlay: Overlay | None = None) -> str: "## Budgets and sizes (1 height unit = 40 px; widths are twelfths of the page)", "", f"- Height budget per tab: {p.fold_units} units (~{p.fold_units * 40}px). Over budget -> tabs or prune.", - f"- KPI band: {p.kpi_row_min}-{p.kpi_row_max} big numbers, {p.kpi_height} units tall, first band on the page.", - f"- Axis charts (timeseries/bar/heatmap/histogram): >= {p.min_axis_height} units tall, 8 is the comfortable default.", + f"- KPI band: {p.kpi_row_min}-{p.kpi_row_max} big numbers, {p.kpi_height} units tall, first band on the page; " + f"Superset sizes their text by height, so a subtitle needs {_kpi_units('big_number_total', p)} " + f"units, a trendline KPI's comparison {_kpi_units('big_number_trend', p)}.", + f"- Axis charts (timeseries/bar/heatmap/histogram/waterfall/box_plot): >= {p.min_axis_height} units tall, 8 is the comfortable default.", f"- At most {p.max_row_charts} axis charts per row; below 3/12 width a chart is unreadable.", "- Pie/donut: >= 5/12 wide, >= 8 tall. Heatmap: >= 5/12 wide (7/12 when many columns), >= 6 tall.", f"- Vertical bars: <= {p.vbar_max_categories} categories, then flip horizontal; an ordered axis " f"(hours, ranks) stays vertical with category_sort and x_label_every. Pies: <= {p.pie_max_slices} slices.", f"- Timeseries: <= {p.series_max} grouped series (series_limit keeps the top N). Tables: height should " - f"show >= {p.table_visible_ratio:.0%} of row_limit (~0.8 units/row), or one page with page_length.", + f"show {'every row' if p.table_visible_ratio >= 1 else f'>= {p.table_visible_ratio:.0%}'} of " + f"row_limit (~0.75 units/row + 2.5), or one page with page_length. Pivots: a row per distinct " + f"row-dimension value (~0.65 units/row + ~4 to 6 for headers and totals), never per row_limit.", "- Charts sharing a row share a height; Superset sizes the row to its tallest child.", ] if p.recommended_heights: @@ -88,8 +100,9 @@ def render_brief(audience: str, overlay: Overlay | None = None) -> str: "", "## Design defaults: `advise --fix` fills these, so leave them unset unless the user asks", "", - "- x_label_format, compare_suffix, number_format of COUNT metrics, and on tables cell_bars,", - " page_length and search_box; show_legend on a single named series; show_value on few bars.", + "- x_label_format, compare_suffix, number_format of COUNT metrics, date_format on a big", + " number of a date column, and on tables cell_bars, page_length and search_box;", + " show_legend on a single named series; show_value on few bars; a heatmap's left_margin.", " Set one yourself only when the user asks for it, and say so in your reply.", "- Run `chartwright advise --fix` (MCP: fix_spec) before you build. It writes each value", " into the spec file and records it in design.filled; its `fixed` entries say kind \"fill\" and why.", @@ -97,7 +110,7 @@ def render_brief(audience: str, overlay: Overlay | None = None) -> str: " A filled value you change or delete is yours from then on (kind \"release\"); never edit design.filled,", " except to rename a chart's entry when you rename the chart, so its fills stay the brain's.", "- Never filled, set them only on request: category_sort, y_axis_truncate, compare_lag,", - " series_limit, show_totals, and currency formats.", + " rolling_type, series_limit, show_totals, and currency formats.", ] lines += ["", _guideline("chart-choice.md").strip(), "", _guideline("composition.md").strip()] diff --git a/chartwright/design/content.py b/chartwright/design/content.py index da09796..f9b8f9b 100644 --- a/chartwright/design/content.py +++ b/chartwright/design/content.py @@ -70,7 +70,7 @@ NUMBER_FORMAT_TYPES = frozenset({ "big_number_total", "big_number_trend", "timeseries_line", "timeseries_bar", "timeseries_area", "timeseries_scatter", "bar", "pie", "pivot_table", "heatmap", "funnel", - "treemap"}) + "treemap", "waterfall", "box_plot"}) _MARK = re.compile(r"/\* cw:(std|end) ([A-Za-z0-9][A-Za-z0-9_-]*)(?: ([0-9a-f]{12}))? \*/") @@ -387,12 +387,14 @@ def _locked_by(std, slot: str, index: int) -> str | None: def chart_number_format(std, chart) -> tuple[str, int] | None: """(format, index of the innermost layer it comes from) for a chart, or None: the chart must have one number_format, every metric it shows must have the same format - in the standard, and it must not plot shares (contribution, a 100 % stack) or a pivot - aggregation that changes the unit.""" + in the standard, and it must not plot shares (contribution, a 100 % stack), a pivot + aggregation that changes the unit, or a big number as a date (date_format).""" if chart.type not in NUMBER_FORMAT_TYPES: return None if getattr(chart, "contribution", None) or getattr(chart, "stack", None) == "expand": return None + if getattr(chart, "date_format", None): + return None # a big number shown as a date takes no number format if chart.type == "pivot_table" and chart.aggregate_function not in _UNIT_AGGREGATES: return None merged: dict[str, tuple[str, int]] = {} diff --git a/chartwright/design/defaults.py b/chartwright/design/defaults.py index dc86af2..ed25600 100644 --- a/chartwright/design/defaults.py +++ b/chartwright/design/defaults.py @@ -38,7 +38,7 @@ from typing import Any, Callable from ..spec import (DEFAULT_TIME_GRAIN, grid_rows_visible, metric_label, parse_metric, - table_header_units) + set_value, table_header_units) from .model import KPI_TYPES, TIMESERIES_TYPES, Finding, RuleContext, rule from .rules import _span_days @@ -135,7 +135,7 @@ def _findings(ctx: RuleContext, fill: Fill): def _fill(rule_id: str, field: str, types, doc: str, *, superset, superset_text: str, - override: str): + override: str, since: str = "5"): def deco(decide): f = Fill(rule_id, field, frozenset(types), decide, superset, superset_text, override) FILLS[rule_id] = f @@ -143,7 +143,7 @@ def deco(decide): def fn(ctx: RuleContext): yield from _findings(ctx, f) - rule(rule_id, "info", doc, fixable=True, since="5")(fn) + rule(rule_id, "info", doc, fixable=True, since=since)(fn) return decide return deco @@ -220,7 +220,8 @@ def _x_label_format(ctx: RuleContext, c): @_fill("default.compare-suffix", "compare_suffix", {"big_number_trend"}, - "a trendline KPI's change says what it compares against ('vs previous month')", + "a trendline KPI's change says what it compares against ('vs previous month'; " + "'vs prior 12 months' between trailing windows)", superset="", superset_text="Superset's bare percentage", override="write compare_suffix yourself, e.g. 'vs last month'") def _compare_suffix(ctx: RuleContext, c): @@ -231,17 +232,49 @@ def _compare_suffix(ctx: RuleContext, c): if unit is None: return None, f"grain {label} has no plain name" lag = c.compare_lag + if lag > 1 and c.rolling_type in ("sum", "mean", "std") and c.rolling_periods == lag: + # Both values are windows of `lag` steps that meet: this one and the one before. + return (f"vs prior {lag} {unit}s", + f"compare_lag {lag} over a {lag}-step rolling {c.rolling_type} at grain {label}") text = f"vs previous {unit}" if lag == 1 else f"vs {lag} {unit}s earlier" return text, f"compare_lag {lag} at grain {label}" +# A big number of a date, as a freshness tile reads it: "Sat 3 Oct 2026". +DATE_TILE_FORMAT = "%a %-d %b %Y" + + +@_fill("default.date-tile", "date_format", {"big_number_total"}, + "a big number of a date column's MIN or MAX reads as a whole date ('Sat 3 Oct " + "2026'); needs the column's type (--profile)", + superset="smart_date", superset_text="Superset's adaptive date ('Tue 31')", + override="write date_format yourself, e.g. '%Y-%m-%d'", since="9") +def _date_tile(ctx: RuleContext, c): + parsed = parse_metric(c.metric) or {} + column = parsed.get("column") + if parsed.get("aggregate") not in ("MIN", "MAX") or not column or column == "*": + return None, "the metric is not MIN or MAX of a column" + if ctx.written(c, "number_format"): + return None, "number_format is written, so the author reads it as a number" + ds = ctx.dataset_for(c) + temporal = ds.is_temporal(column) if ds is not None else None + if temporal is None: + return None, f"{column!r} has no known type (advise --profile reads it)" + if not temporal: + return None, f"{column!r} is not a date column" + # Superset's default (smart_date, and a total has no time grain) shows only the day, + # e.g. 'Tue 31', for the latest date (seen on 4.1.4, 5.0.0 and 6.1.0). + return DATE_TILE_FORMAT, (f"{parsed['aggregate']}({column}) is a date, which Superset " + f"shows as only its day ('Tue 31')") + + # -- number formats ------------------------------------------------------------- _COUNTS = ("COUNT", "COUNT_DISTINCT") # Pivot aggregations that keep a count a whole number. _WHOLE_AGGREGATES = {"Sum", "Count", "Count Unique Values", "Minimum", "Maximum", "First", "Last"} COUNT_FORMAT_TYPES = (KPI_TYPES | TIMESERIES_TYPES - | {"bar", "pie", "pivot_table", "heatmap", "funnel", "treemap"}) + | {"bar", "pie", "pivot_table", "heatmap", "funnel", "treemap", "waterfall"}) @_fill("default.count-format", "number_format", COUNT_FORMAT_TYPES, @@ -249,6 +282,8 @@ def _compare_suffix(ctx: RuleContext, c): superset="SMART_NUMBER", superset_text="Superset's SMART_NUMBER ('12.3k')", override="write number_format yourself, e.g. '.3s'") def _count_format(ctx: RuleContext, c): + if getattr(c, "date_format", None): + return None, "date_format shows the number as a date" shown = [c.metric] if hasattr(c, "metric") else list(c.metrics) other = [m for m in shown if (parse_metric(m) or {}).get("aggregate") not in _COUNTS] if other: @@ -274,10 +309,15 @@ def _count_format(ctx: RuleContext, c): "a raw table with id, code, year or zip columns draws no cell bars (a bar behind " "an identifier reads as an amount)", superset=True, superset_text="Superset's bars behind every number", - override="write cell_bars: true to keep the bars") + override="write cell_bars: true to keep the bars, or list the columns that keep " + "them, e.g. cell_bars: [\"amount\"]") def _cell_bars(ctx: RuleContext, c): if not c.columns: return None, "an aggregate table draws bars on its metrics only, never on a dimension" + asks = [f for f in ("color_by_sign", "absolute_bars") if set_value(c, f)] + if asks: + # Both draw on the bars, so false would leave them nothing to draw on. + return None, f"{asks[0]} draws on the cell bars, so which columns keep them is yours" hits = [col for col in c.columns if ID_LIKE.search(col)] ds = ctx.dataset_for(c) if ds is not None and ds.column_types: @@ -300,10 +340,12 @@ def _page_length(ctx: RuleContext, c): # The one grid model size.table-window reads (spec.py): a page fills the whole rows # that fit beside the page-size bar and the pager, so a fill can never make that # rule ask for more height. - fits = math.floor(grid_rows_visible(h, table_header_units(controls=c.search_box))) + fits = math.floor(grid_rows_visible(h, table_header_units(controls=c.search_box, + totals=c.show_totals))) if c.row_limit <= fits: return None, f"all {c.row_limit} rows fit at height {h:g}" - page = math.floor(grid_rows_visible(h, table_header_units(controls=True, pager=True))) + page = math.floor(grid_rows_visible(h, table_header_units(controls=True, pager=True, + totals=c.show_totals))) if page < ctx.params.page_min_rows: return None, f"height {h:g} fits {page} rows beside a pager, too few to page; raise the height" return page, (f"row_limit {c.row_limit} at height {h:g}: {fits} rows fit on one page, " @@ -311,8 +353,8 @@ def _page_length(ctx: RuleContext, c): @_fill("default.search-box", "search_box", {"table"}, - "a raw table of more than ~20 rows gets a search box, when its rows still fit " - "beside it (the 20 is judgement)", + "a raw table of more than ~20 rows that outgrow its panel, and so page, gets a " + "search box; one whose rows all show gets none (the 20 is judgement)", superset=False, superset_text="no search box", override="write search_box: false") def _search_box(ctx: RuleContext, c): @@ -323,16 +365,21 @@ def _search_box(ctx: RuleContext, c): n = ctx.params.search_min_rows if c.row_limit <= n: return None, f"row_limit {c.row_limit} is {n} rows or fewer" - if not c.page_length: - # A paged table already draws the bar the search box sits in. On one page, the - # bar takes room from the rows, and must not push any behind the scrollbar. - h = ctx.height(c.name) - fits = math.floor(grid_rows_visible(h, table_header_units(controls=True))) - if c.row_limit > fits: - return None, (f"{c.row_limit} rows on one page at height {h:g}: a search bar " - f"would leave room for {fits}; page the table or raise the height") - return True, (f"up to {c.row_limit} raw rows: searching beats scrolling " - f"(the {n}-row threshold is judgement)") + h = ctx.height(c.name) + # The rows the panel shows with nothing above them. A table that shows every row + # takes no DataTables chrome (size.table-chrome): a search bar over rows the reader + # already sees only pushes the last ones behind the scrollbar. + shown = math.floor(grid_rows_visible(h, table_header_units())) + if c.row_limit <= shown: + return None, f"all {c.row_limit} rows show at height {h:g}" + if not c.page_length or c.page_length >= c.row_limit: + # The box sits in the bar a paged table already draws (its page-size picker), so + # it costs no row there. A table on one page that outgrows its panel is either + # still to be paged (default.page-length) or too short to page. + return None, (f"{c.row_limit} rows on one page outgrow height {h:g}; the box comes " + f"with paging") + return True, (f"up to {c.row_limit} raw rows over pages of {c.page_length}: searching " + f"beats paging (the {n}-row threshold is judgement)") # -- legends and labels ------------------------------------------------------------- @@ -367,20 +414,42 @@ def _single_series_legend(ctx: RuleContext, c): HBAR_LABEL_UNITS = 0.4125 -@_fill("default.value-labels", "show_value", {"bar"}, - "few bars carry their values: <= 12 bars, on a panel >= 6/12 wide (vertical) or " - "tall enough to space the labels (horizontal)", +@_fill("default.value-labels", "show_value", {"bar", "waterfall"}, + "few bars carry their values: <= 12 bars (row_limit x metrics for grouped bars), on a " + "panel >= 6/12 wide (vertical, and a waterfall's steps) or tall enough to space the " + "labels (horizontal)", superset=False, superset_text="no values on the bars", override="write show_value: false") def _value_labels(ctx: RuleContext, c): - if len(c.metrics) != 1 or c.groupby: - return None, "more than one series, so labels would crowd" + most, min_w = ctx.params.value_label_max_bars, ctx.params.value_label_min_width + if c.type == "waterfall": + # A bridge is read by its deltas: each step's value is the point of the chart. + if c.groupby: + return None, "a breakdown per x value, so the number of bars is unknown" + if c.steps is not None: + n = len(c.steps) + 1 + (c.opening is not None) + elif ctx.written(c, "row_limit"): + n = c.row_limit + 1 + else: + return None, "neither steps nor row_limit is set, so the number of bars is unknown" + if n > most: + return None, f"up to {n} bars with the total, more than {most}" + w = ctx.width(c.name) + if w < min_w: + return None, f"{w}/12 wide, narrower than {min_w}/12" + return True, f"{n} bars with the total at {w}/12 wide: each step's change reads without the axis" + # Grouped bars (several metrics side by side) label every bar, so they count as + # row_limit x metrics; a groupby's series count is unknown, and a stack's labels + # land on each segment. + if c.groupby: + return None, "a series per group value, so the number of bars is unknown" + if c.stack and len(c.metrics) > 1: + return None, "stacked series, so each segment would carry a label" if c.contribution: return None, "contribution plots shares" if not ctx.written(c, "row_limit"): return None, "row_limit is not set, so the number of bars is unknown" - most, min_w = ctx.params.value_label_max_bars, ctx.params.value_label_min_width - n = c.row_limit + n = c.row_limit * len(c.metrics) if n > most: return None, f"up to {n} bars, more than {most}" if c.orientation == "horizontal": @@ -394,3 +463,25 @@ def _value_labels(ctx: RuleContext, c): if w < min_w: return None, f"{w}/12 wide, narrower than {min_w}/12" return True, f"at most {n} bars at {w}/12 wide: each value reads without the axis" + + +# -- heatmaps ----------------------------------------------------------------------- + +# A heatmap's y labels sit inside its grid (containLabel), whose left edge is the card's +# edge while left_margin is Superset's 'auto' (Heatmap/transformProps.ts 6.1.0 :354-357). +# Seen on 6.1.0: the longest label is drawn wider than the room the grid gave it and +# loses its first letters there ('rucks and Buses', a 95 px label; 'ustralian Gift +# Network, Co', 150 px). 8 px cleared the first and 16 px both; 4.1.4 and 5.0.0 drew +# them whole. Its own spec field is the only way: on 6.1.0 Heatmap.tsx:25 renders +# without vizType, so a theme's per-chart-type overrides never reach it. +HEATMAP_LABEL_ROOM = 16 + + +@_fill("default.heatmap-label-room", "left_margin", {"heatmap"}, + "a heatmap keeps 16 px left of its y labels, where Superset 6.1.0 cuts off the " + "longest one's first letters", + superset=None, superset_text="no margin (Superset's 'auto')", + override="write left_margin yourself (0 for none)", since="12") +def _heatmap_label_room(ctx: RuleContext, c): + return HEATMAP_LABEL_ROOM, ("Superset 6.1.0 cuts the first letters off the longest y " + "label at the card's edge; 16 px keeps labels up to ~300 px whole") diff --git a/chartwright/design/guidelines/chart-choice.md b/chartwright/design/guidelines/chart-choice.md index 92cb934..5cced60 100644 --- a/chartwright/design/guidelines/chart-choice.md +++ b/chartwright/design/guidelines/chart-choice.md @@ -9,27 +9,50 @@ | Several measures per item | `table` or `pivot_table`, not grouped bars. Pivots: <= 3 total dimensions; column-dim values x metrics <= ~15 rendered columns | | Two dimensions, one measure | `heatmap` (keep the grid under ~400 cells) | | Distribution of one column | `histogram`, 20-30 bins; trim long tails with a WHERE filter and say so in the title | +| A distribution per group | `box_plot`: median, quartiles, whiskers and outliers of many observations each (daily sales by month) | | Staged conversion | `funnel`, 3-8 ordered stages | | Hierarchical share | `treemap`, at most 2 levels | +| A bridge between two totals | `waterfall`: the opening, 3-12 steps that add up, the closing; `steps` and `closing` keep its order | -- A `row_limit` on a pie redefines the whole: the shown slices read as 100%, - so a truncated pie lies about share. Prefer a horizontal bar for top-N; a - truncated pie's title must say "top N". +- A `row_limit` on a pie redefines the whole: its slices read as 100%, a lie + about share. Prefer a horizontal bar for top-N; else title the pie "top N". - Table and pivot height is a DATA-DEPENDENT property, not a one-time layout - choice: size from expected rows (~0.75 units per row + 3 for title/header, - +1 for pivot column headers) and re-check whenever a row dimension is - expected to gain members. Rows past the fold hide behind an inner - scrollbar; smoke warns at apply time when they do. + choice: size from expected rows (a table ~0.75 units per row + 2.5, more + with a search box, pager or totals row; a pivot ~0.65 per distinct + row-dimension value + 4 to 6 for its header rows and totals row, never from + its row_limit, which counts records: rows x columns) and re-check whenever a + row dimension is expected to gain members. Rows past the fold hide behind + an inner scrollbar; smoke warns at apply time when they do. - Sort order is per family: bars sort by their first metric (right for rankings, wrong for ordinals like weekday/month) unless `category_sort` - orders them by label; heatmap and pivot categories sort alphabetically - (a pivot can sort by value: `row_order`). For ordinal dimensions, chart an - order-encoded label column (labels prefixed with a sort index: '1-Mon') - if the dataset has one, and give a bar `category_sort: "asc"`. An ordered - axis such as hours stays vertical: add `x_label_every` so every label shows. -- A paged table (`page_length`) needs room for one page and its pager, not - for every row; a timeseries with many groups keeps the top few with - `series_limit`. + orders them by label. Stacked or grouped bars need `sort_by: "total"` to + rank (several series otherwise come in name order); `sort_by` a metric ranks + by one the chart needn't draw, so two charts share an order. Heatmap and + pivot categories sort alphabetically (by value: a pivot's `row_order`, a + heatmap's `x_order`/`y_order`). For ordinal dimensions, chart an + order-encoded label column ('1-Mon') if the dataset has one, and give a bar + `category_sort: "asc"`. An ordered axis such as hours stays vertical: add + `x_label_every`; on a heatmap it is a step (6 on hours reads 0, 6, 12, 18). + A heatmap's `y_order: "a_to_z"` puts the first label on top (a cohort triangle). +- Title every axis: `y_axis_title` the measure and unit ("GW"), `x_axis_title` + the dimension ("Month"). Bars carry `show_value` (`only_total` on a stack). +- A rolling KPI (`rolling_type`): name its window ("Revenue, trailing 12 months"), + give a range holding it plus `compare_lag`, and set `y_axis_truncate`. +- A table whose rows all show takes no `page_length` or `search_box`; a + timeseries with many groups keeps the top few with `series_limit`. A mixed + chart's lines share one width: give a reference series a pale `label_colors` + colour; a fixed level is an annotation (`style`, `width`). +- A waterfall explains how one total became another (last year to this year, + plan to actual) through steps that ADD UP: a SUM or COUNT of the change, + never an average. Put the opening, each step and the closing in the dataset + as rows of one step column; name the ends in `opening` and `closing`, the + steps in `steps` (6.1.0+). An opening total needs a theme that keeps zero on + the axis (see `opening` in the schema). Set the three colours and `show_value`. +- Show the distribution, not the average, when the spread is the story + (delivery times, daily sales): a `box_plot` beats a line of P50 and P90 + across categories or a few periods; keep the line for long trends. Give each + box many observations (`distribute_across` a time column at P1D, `groupby` + the month) and pin one colour to its groups in `dashboard.label_colors`. - One dominant category flattening its siblings is the data talking: note it or filter it, don't hide it. - Every chart with a time axis should tolerate the dashboard time filter; diff --git a/chartwright/design/guidelines/composition.md b/chartwright/design/guidelines/composition.md index 8bbeb29..ae66d6f 100644 --- a/chartwright/design/guidelines/composition.md +++ b/chartwright/design/guidelines/composition.md @@ -1,28 +1,29 @@ # Composition -- Inverted pyramid, top to bottom: KPI band first (the answer), trends second - (the why), breakdowns and tables last (the detail). Readers scan in a Z: - the top-left cell is the most valuable slot on the page. +- Inverted pyramid: KPI band first (the answer), trends next (the why), breakdowns + and tables last. Readers scan in a Z: the top-left cell is the prime slot. - Group by question, not by chart type: a trend and its breakdown belong side by side; two unrelated charts sharing a row invite false comparison. -- Matched granularity per row: charts in one row should share their time - window and grain, or say in the title why not. -- One message per chart. A chart needing a paragraph to explain wants to be - two charts, or a table. +- A row's charts share their time window and grain, or the title says why not. +- One message per chart: a chart needing a paragraph is two charts, or a table. - Titles state the answer where possible ("Orders fell 12% WoW"), the question otherwise ("Orders by week"); never just a column name. Filtered charts name their scope in the title. -- Tabs when sections answer different questions; scrolling when one question - deepens. Never a tab with a single lonely chart. -- Whitespace is markdown's job, sparingly: a one-line section header beats an - empty band (2 units is plenty for a header). Consistent number formats per - measure across KPI cards and axes (`number_format` on KPIs, timeseries and - bars; `number_formats` per column on tables; pivots use Superset's smart - default). +- Tabs split different questions; scrolling deepens one. Never a one-chart tab; + a one- or two-chart sub-tab belongs on its parent page, under a header row. +- Whitespace is markdown's job, sparingly: a one-line header (1.6-2.4 units) beats + an empty band. In a sketch, legend blocks draw headers and notes: `"T": {"header": + "Revenue"}` across the page titles the band below, above a chart its column; + `{"markdown": "...", "height": 1.6}` is a slim caption. +- One number format per measure across KPIs and axes (`number_format`; + `number_formats` per table column; pivots keep Superset's smart default). - Color restraint: Superset's default palette, RAG only where a threshold has a real business meaning; never encode the same dimension with two palettes. -- RAG polarity is a convention, not a choice: red = adverse, green = good - (IBCS). Bands on one metric must be disjoint and tell one story. -- Heatmaps ship with a sequential scale normalized over the whole map: right - for magnitudes, wrong for signed deltas -- don't heatmap a metric that - crosses zero. +- RAG polarity is a convention: red = adverse, green = good (IBCS). Bands on one + metric are disjoint and tell one story; a status-coloured KPI colours only its + exceptions and states its thresholds in its subtitle or description. +- A heatmap's default sequential scale, normalized over the whole map, suits + magnitudes, not signed deltas: don't heatmap a metric that crosses zero. +- Table bars only where size is the point (`cell_bars: ["Revenue"]`), never on + ids or years. Superset tints every bar by sign, green on 6.x even for revenue: + keep that for a change, `color_by_sign: ["Change"]`, listed in cell_bars too. diff --git a/chartwright/design/markdown_fit.py b/chartwright/design/markdown_fit.py new file mode 100644 index 0000000..7f16e46 --- /dev/null +++ b/chartwright/design/markdown_fit.py @@ -0,0 +1,1231 @@ +"""How tall Superset draws a dashboard markdown block, estimated offline from its +text and the dashboard's CSS: `size.markdown-fit` and `layout.markdown-height` +(rules.py) both read it. + +What Superset does with a markdown block, at 4.1.4, 5.0.0 and 6.1.0 alike: +- renders the text with react-markdown and GitHub-flavoured markdown (tables, + strikethrough), raw HTML sanitized (packages/superset-ui-core/src/components/ + SafeMarkdown.tsx at 4.1.4 and 5.0.0, components/SafeMarkdown/SafeMarkdown.tsx at + 6.1.0). The elements sit straight inside the holder, with no wrapper of their own; +- puts it in a holder with 16 px of padding on every side (DashboardBuilder.tsx + :326-332 at 4.1.4, :325-331 at 5.0.0, :311-317 at 6.1.0) that scrolls what doesn't + fit (overflow-y: auto; gridComponents/Markdown.jsx:106-109 at 4.1.4 and 5.0.0, + gridComponents/Markdown/Markdown.tsx:126-130 at 6.1.0). Text past the bottom edge + is cut off. macOS shows no scrollbar until the reader scrolls inside the block, so + the text just stops mid-line, as in a screenshot; Windows draws a scrollbar; +- never draws it shorter than 5 grid rows, 40 px: the markdown component passes + GRID_MIN_ROW_UNITS = 5 (util/constants.ts:42) to ResizableContainer as its + minHeight (resizable/ResizableContainer.jsx:285-286 at 4.1.4, + ResizableContainer.tsx:287 at 6.1.0); +- lays the grid out in 12 columns with 16 px gutters (util/constants.ts:38-40), as + wide as the window less 64 px. Beside the vertical filter bar, which opens by + default on a dashboard with native filters (DashboardBuilder/state.ts:39-41 at + 4.1.4; OPEN_FILTER_BAR_WIDTH = 260 in dashboard/constants.ts), it is 260 px + narrower at 4.1.4 and 5.0.0 and 228 px at 6.1.0. + +The dashboard's CSS reaches the markdown: every `.dashboard-markdown ` rule +beats Superset's own type, on every release, except an h6's font size and the weight +of h4 to h6, which the markdown component sets inside its own class (Markdown.jsx +:92-104 at 4.1.4 and 5.0.0, Markdown/Markdown.tsx:112-124 at 6.1.0). So the estimate +reads the CSS the spec carries and applies it in cascade order, with Superset's own +styles (RELEASES) underneath: rules on `.dashboard-markdown` and its elements, with +descendant, child and sibling combinators (`h4 + p`), :first-child and :last-child, +and the properties that size text: font-size, line-height, font-weight, letter- and +word-spacing, margins, padding, borders and a table's border-spacing. Anything else +that could size markdown (a font family, an attribute selector, a pseudo-element, an +@media block, the block's own padding) is listed in `Stylesheet.unread`, and the +finding names it. + +Superset's own styles and the character widths below were measured on 2026-10-05 in +headless Chromium, each release with its default theme (tools/record_markdown_fit.py; +docs/DESIGN-BRAIN.md, "Markdown blocks"). Headings' line heights are a factor, 1.4 +(dashboard/styles.ts:21-50), the body's antd's 1.5715, so both follow a font size the +CSS sets. The text wraps word by word as the browser wraps it, with Inter's own +advance widths, a line is as tall as the inline text on it needs, and block margins +collapse as CSS collapses them. tests/test_markdown_fit.py holds what each release +drew for 17 texts at five widths in two windows, beside the filter bar and not, and +for a dashboard's own markdown under its own CSS, and checks both bounds below +against every one. + +The rule assumes a window 1440 px wide (VIEWPORT), a common 13-inch laptop's +screen. A wider window wraps fewer lines, so a height that fits there fits wider too. + +Two answers, because a rule must be sure before it speaks and its fix must work on +every release: +- low: never more than any release drew. Its characters are 3% narrower than the + table's and it takes the more compact release in the widest grid, so a block it + calls cut off is cut off on every release; +- high: never less than any release drew, as Windows draws it. It takes the taller + release, its characters are 1% wider than the table's, and it allows for two + classic scrollbars of 17 px: the page's own, which narrows the grid, and one in the + block. A block that overflows while it loads (before its web font arrives, say) + draws a scrollbar, and the narrower text can need the extra line that keeps it + there: 5.0.0 kept one on a two-line text whose first line came within 1% of the + edge. Every block raised to the high bound showed every line with no inner + scrollbar, on every release, with Windows-style scrollbars on. +""" + +from __future__ import annotations + +import html +import math +import re +import unicodedata +from dataclasses import dataclass, field +from functools import lru_cache + +# Inter's advance widths in em for ASCII 32..126 at weight 400, the widest each +# character measured at any release (canvas measureText): 4.1.4 ships Inter 3, 5.0.0 +# and 6.1.0 Inter 4, and their letters differ by under 2%. +_EM = tuple(float(f"0{v}") for v in ( + ".281 .288 .466 .633 .642 .982 .644 .300 .365 .365 .501 .662 .288 .460 .288 .360 " + ".631 .464 .610 .636 .646 .608 .624 .571 .619 .624 .288 .302 .662 .662 .662 .511 " + ".966 .690 .654 .730 .722 .601 .590 .746 .743 .269 .571 .672 .565 .903 .753 .765 " + ".639 .765 .644 .642 .646 .744 .690 .985 .682 .679 .629 .365 .360 .365 .471 .456 " + ".497 .564 .621 .571 .621 .583 .370 .613 .591 .242 .242 .549 .242 .876 .591 .600 " + ".612 .612 .376 .528 .364 .591 .562 .818 .546 .562 .552 .426 .333 .426 .662").split()) +# The same for Latin-1 and common punctuation and symbols, measured the same way. +_EM_MORE = {ch: em for em, chars in ( + (0.223, "′"), (0.241, "‚"), (0.242, "ìíîï"), (0.261, "‘’"), (0.268, "¸"), + (0.269, "ÌÍÎÏ"), (0.27, "¦"), (0.281, "\u00a0"), (0.288, "¡·"), (0.313, "¹"), + (0.384, "‹›"), (0.421, "„"), (0.44, "“”"), (0.442, "²″"), (0.446, "³"), (0.454, "ª"), + (0.456, "°"), (0.478, "¯"), (0.482, "º"), (0.497, "´"), (0.5, "–"), (0.511, "¿"), + (0.549, "≤≥≈≠√"), (0.55, "¥"), (0.556, "‡"), (0.562, "ýÿγ"), (0.563, "•"), + (0.564, "àáâãäå"), (0.568, "§"), (0.571, "¢ç✗"), (0.583, "«»èéêëð"), (0.584, "δ"), + (0.591, "¨ñùúûü"), (0.596, "β"), (0.6, "òóôõöø"), (0.601, "ÈÉÊË"), (0.603, "¶"), + (0.604, "●○■□"), (0.611, "™"), (0.612, "þ∆"), (0.615, "◇"), (0.616, "ß"), (0.622, "µ"), + (0.631, "£†π"), (0.636, "Þ"), (0.662, "¬±×÷−"), (0.664, "α"), (0.667, "€"), + (0.67, "®"), (0.677, "✘"), (0.679, "Ý"), (0.69, "ÀÁÂÃÄÅ"), (0.713, "∑∞"), (0.723, "◆"), + (0.725, "¤"), (0.73, "Ç"), (0.735, "Ð"), (0.744, "ÙÚÛÜ"), (0.753, "Ñ"), (0.764, "✓"), + (0.765, "ÒÓÔÕÖØ"), (0.77, "Ω"), (0.846, "✔"), (0.864, "…"), (0.884, "¼"), (0.915, "©"), + (0.925, "æ"), (0.926, "½"), (0.949, "¾"), (0.955, "↑↓"), (0.99, "▲▼►◄"), (0.994, "Æ"), + (1.0, "—‰→←↔★☆"), (0.0, "\u00ad\u200b\u200c\u200d\ufeff"), +) for ch in chars} +_EM_MONO = 0.6 # every character of a monospace font (Fira Code, Courier, DejaVu Mono) +# A character the tables don't hold: the low bound takes it narrow, the high one wide, +# East Asian wide and fullwidth characters (drawn by a fallback font) widest of all. +_EM_UNKNOWN = {"low": (0.3, 0.9), "high": (0.8, 1.0)} +LOW_SCALE = 0.97 # the low bound's character widths, against the table's +HIGH_SCALE = 1.01 # the high bound's +VIEWPORT = 1440 # the window the rule assumes, in px + +PAD = 16.0 # the holder's padding, each side +SCROLLBAR = 17.0 # a classic scrollbar's width on Windows +GUTTER = 16.0 # GRID_GUTTER_SIZE +PAGE_MARGIN = 64.0 # the window's width less the grid's +MIN_BOX = 40.0 # GRID_MIN_ROW_UNITS grid rows of GRID_BASE_UNIT (8) px +SNAP = 0.4 # px a block may run over and not scroll (scrollHeight is whole px) +UNIT = 40.0 # px per spec height unit +ROOT_FONT = 16.0 # the html element's font size, what rem counts in + + +def _weight_factor(weight: float) -> float: + """How much wider Inter is at a weight than at 400, per sentence: 500 is 0.8% + wider, 600 (and 700, which Superset doesn't load, so the browser draws 600) 1.8%.""" + return 1.02 if weight >= 600 else 1.01 if weight >= 500 else 1.0 + + +# -- Superset's own markdown styles --------------------------------------------------- + + +@dataclass(frozen=True) +class Release: + """One release's markdown styles, as CSS on the elements, with the specificity + Superset's own rules lose by: any `.dashboard-markdown ` rule beats them, + except the PROTECTED ones, given with the specificity they have in Superset.""" + + name: str + newest: tuple[int, int, int] # the newest release it stands for + bar: float # px the open vertical filter bar takes from the grid + flag_for_horizontal_bar: bool # draws the horizontal bar only with a feature flag + css: str + protected: tuple[tuple[str, str, tuple[int, int, int]], ...] + root: tuple[float, float] # the font size and line height (a factor) inherited + + def grid(self, viewport: float, filter_bar: bool) -> float: + return viewport - PAGE_MARGIN - (self.bar if filter_bar else 0.0) + + +# Measured with every property set from `.dashboard-markdown ` (the +# "probe" stylesheet in the fixture): each took, h6's font size apart. Line heights +# are factors (headings 1.4 in `body h1`..`h6`, dashboard/styles.ts:21-50; the body's +# 1.5715 from antd), so they follow a font size the CSS sets. +_HEADINGS = """ +.dashboard-markdown h1, .dashboard-markdown h2, .dashboard-markdown h3, +.dashboard-markdown h4, .dashboard-markdown h5, .dashboard-markdown h6 { line-height: 1.4; } +.dashboard-markdown h1 { font-size: 28px; margin: 12px 0 12px; } +.dashboard-markdown h3, .dashboard-markdown h4, .dashboard-markdown h5 { font-size: 16px; } +.dashboard-markdown h3, .dashboard-markdown h4, .dashboard-markdown h5, +.dashboard-markdown h6 { margin: 8px 0 4px; } +.dashboard-markdown p { margin: 0 0 8px; } +.dashboard-markdown ul, .dashboard-markdown ol { padding-left: 40px; } +.dashboard-markdown li ul, .dashboard-markdown li ol { margin: 0; } +.dashboard-markdown code, .dashboard-markdown pre { font-family: monospace; } +""" +_PROTECTED = ( + # Markdown.jsx:92-104 at 4.1.4 and 5.0.0, Markdown/Markdown.tsx:112-124 at 6.1.0: + # inside the component's styled class, so (0,2,1). + (".dashboard-markdown h6", "font-size: 12px", (0, 2, 1)), + (".dashboard-markdown h4, .dashboard-markdown h5, .dashboard-markdown h6", + "font-weight: 400", (0, 2, 1)), +) +RELEASES = ( + Release( + "4.1.4 and 5.0.0", (5, 0, 0), bar=260.0, flag_for_horizontal_bar=True, + css=_HEADINGS + """ +.dashboard-markdown h1, .dashboard-markdown h2, .dashboard-markdown h3 { font-weight: 600; } +.dashboard-markdown h2 { font-size: 21px; margin: 12px 0 8px; } +.dashboard-markdown ul, .dashboard-markdown ol { margin: 0 0 9.5px; } +.dashboard-markdown strong, .dashboard-markdown b, .dashboard-markdown th { font-weight: 600; } +.dashboard-markdown table { border-collapse: collapse; } +.dashboard-markdown blockquote { font-size: 17.5px; margin: 0 0 19px; padding: 9.5px 19px; + border-left: 5px solid; } +.dashboard-markdown pre { font-size: 13px; line-height: 1.4; margin: 0 0 9.5px; padding: 9px; + border: 1px solid; } +.dashboard-markdown code { font-size: 90%; padding: 2px 4px; } +.dashboard-markdown pre code { font-size: inherit; padding: 0; } +.dashboard-markdown hr { margin: 15px 0; border-top: 1px solid; } +""", + # Bootstrap's `blockquote p:last-child` (0,1,2): no gap under a quote's text. + protected=_PROTECTED + ( + (".dashboard-markdown blockquote p:last-child, " + ".dashboard-markdown blockquote ul:last-child, " + ".dashboard-markdown blockquote ol:last-child", "margin-bottom: 0", (0, 1, 2)),), + root=(14.0, 1.5715), + ), + Release( + "6.1.0", (6, 1, 0), bar=228.0, flag_for_horizontal_bar=False, + css=_HEADINGS + """ +.dashboard-markdown h1, .dashboard-markdown h2, .dashboard-markdown h3 { font-weight: 500; } +.dashboard-markdown h2 { font-size: 20px; margin: 12px 0 8px; } +.dashboard-markdown ul, .dashboard-markdown ol { margin: 14px 0; } +.dashboard-markdown strong, .dashboard-markdown b { font-weight: 600; } +.dashboard-markdown th { font-weight: 500; } +.dashboard-markdown table { border-spacing: 2px; } +.dashboard-markdown td, .dashboard-markdown th { padding: 1px; } +.dashboard-markdown blockquote { margin: 14px 40px; } +.dashboard-markdown pre { margin: 14px 0; } +.dashboard-markdown hr { margin: 7px 0; border: 1px solid; } +""", + protected=_PROTECTED, root=(14.0, 22 / 14), + ), +) + + +# -- reading CSS -------------------------------------------------------------------- + +_COMMENT = re.compile(r"/\*.*?\*/", re.S) +_SPLIT_TOP = re.compile(r",(?![^(\[]*[)\]])") +_COMPOUND = re.compile(r"^(\*|[a-zA-Z][\w-]*)?((?:\.[\w-]+)*)((?::(?:first-child|last-child|only-child))*)$") +_COMBINATOR = re.compile(r"\s*([>+~])\s*|\s+") +# Classes on the markdown block itself and the layout around it: a selector naming +# any other class targets something else, and never reaches the markdown's text. +_CHAIN = ("dashboard-markdown", "resizable-container", "dashboard-component", + "dashboard-component-chart-holder") +_AROUND = frozenset(_CHAIN) | { + "dashboard", "dashboard-content", "grid-container", "grid-content", "grid-row", + "dashboard-grid", "dashboard-component-tabs", "dashboard-component-tabs-content", + "dashboard-component-row", "dashboard-component-column", "dragdroppable", + "dragdroppable-row", "dragdroppable-column", "background--white", + "background--transparent", "ant-tabs", "ant-tabs-content", "ant-tabs-content-holder", + "ant-tabs-content-top", "ant-tabs-tabpane", "ant-tabs-tabpane-active", + "dashboard-markdown--editing"} +# What the estimate reads, and what else would change a markdown block's size. +_READ = re.compile(r"^(?:font-size|line-height|font-weight|letter-spacing|word-spacing|" + r"margin(?:-(?:top|right|bottom|left))?|padding(?:-(?:top|right|bottom|left))?|" + r"border(?:-(?:top|right|bottom|left))?(?:-(?:width|style))?|border-spacing|" + r"border-collapse|font-family)$") +_SIZING = re.compile(r"^(?:font|font-(?:size|family|weight|stretch|variant-numeric)|line-height|" + r"letter-spacing|word-spacing|margin.*|padding.*|border|border-spacing|" + r"border-collapse|border-(?:top|right|bottom|left)(?:-(?:width|style))?|" + r"border-(?:width|style)|text-transform|text-indent|white-space|word-break|" + r"overflow-wrap|word-wrap|hyphens|display|width|min-width|max-width|height|" + r"min-height|max-height|float|columns|column-[\w-]+|content|zoom|" + r"writing-mode|box-sizing)$") +_SIDES = ("top", "right", "bottom", "left") + + +@dataclass(frozen=True) +class _Selector: + compounds: tuple[tuple[str | None, frozenset, tuple[str, ...]], ...] + combinators: tuple[str, ...] # between compound k and k+1 + specificity: tuple[int, int, int] + + +@dataclass(frozen=True) +class _Rule: + selector: _Selector + decls: tuple[tuple[str, str, bool], ...] # longhand property, value, !important + specificity: tuple[int, int, int] + origin: int # 0 Superset's, 1 the dashboard's + order: int + + +@dataclass(frozen=True) +class Stylesheet: + """The parts of a dashboard's CSS the estimate reads, and the rest that could + size markdown text: (selector, property) for each.""" + + rules: tuple[_Rule, ...] = () + unread: tuple[tuple[str, str], ...] = () + + +def _parse_selector(text: str) -> _Selector | None: + """A selector the estimate can match against markdown elements, or None.""" + text = text.strip() + if not text or re.search(r"[\[\]#()\"']|::", text): + return None + parts = _COMBINATOR.split(text) + # re.split with a group alternates compound, combinator-or-None, compound... + compounds, combinators = [], [] + for k, part in enumerate(parts): + if k % 2: + combinators.append((part or " ").strip() or " ") + continue + m = _COMPOUND.match(part or "") + if not part or not m: + return None + tag = m.group(1).lower() if m.group(1) and m.group(1) != "*" else None + classes = frozenset(c for c in m.group(2).split(".") if c) + pseudo = tuple(p for p in m.group(3).split(":") if p) + compounds.append((tag, classes, pseudo)) + spec = (0, sum(len(c[1]) + len(c[2]) for c in compounds), sum(1 for c in compounds if c[0])) + return _Selector(tuple(compounds), tuple(combinators), spec) + + +def _declarations(body: str) -> list[tuple[str, str, bool]]: + out = [] + for decl in re.split(r";(?![^(]*\))", body): + if ":" not in decl: + continue + prop, value = decl.split(":", 1) + value = value.strip() + important = bool(re.search(r"!\s*important\s*$", value, re.I)) + value = re.sub(r"!\s*important\s*$", "", value, flags=re.I).strip() + out.append((prop.strip().lower(), value, important)) + return out + + +def _longhands(prop: str, value: str) -> list[tuple[str, str]] | None: + """A property as its longhands, or None when the value can't be read.""" + if "var(" in value or "calc(" in value or "env(" in value: + return None + parts = value.split() + if prop in ("margin", "padding", "border-width", "border-style"): + if not 1 <= len(parts) <= 4: + return None + top = parts[0] + right = parts[1] if len(parts) > 1 else top + bottom = parts[2] if len(parts) > 2 else top + left = parts[3] if len(parts) > 3 else right + name = prop if prop in ("margin", "padding") else "border" + suffix = "" if prop in ("margin", "padding") else "-" + prop.split("-")[1] + return [(f"{name}-{s}{suffix}", v) for s, v in zip(_SIDES, (top, right, bottom, left))] + if prop.startswith("border") and prop not in ("border-spacing", "border-collapse"): + bits = prop.split("-")[1:] + sides = [bits[0]] if bits and bits[0] in _SIDES else list(_SIDES) + if bits and bits[-1] in ("width", "style"): + return [(prop, parts[0])] if len(parts) == 1 else None + width, style = "medium", "none" + for p in parts: + if p in _STYLES: + style = p + elif p in ("thin", "medium", "thick") or re.fullmatch(r"-?[\d.]+(?:px|em|rem)?", p): + width = p + return [(f"border-{s}-{k}", v) for s in sides for k, v in (("width", width), + ("style", style))] + return [(prop, value)] + + +def _rules_of(css: str): + """(selector list, declarations, the @media or other at-rule it sits in) for each + style rule.""" + css = _COMMENT.sub("", css) + i, n = 0, len(css) + while i < n: + j = css.find("{", i) + if j < 0: + return + prelude = css[i:j].rsplit(";", 1)[-1].strip() + depth, k = 1, j + 1 + while k < n and depth: + depth += {"{": 1, "}": -1}.get(css[k], 0) + k += 1 + body = css[j + 1:k - 1] + if prelude.startswith("@"): + if re.match(r"@(?:media|supports|layer|container|document)\b", prelude, re.I): + for sel, decls, _ in _rules_of(body): + yield sel, decls, " ".join(prelude.split()) + else: + yield prelude, body, None + i = k + + +_STYLES = ("none", "hidden", "solid", "dashed", "dotted", "double", "groove", "ridge", + "inset", "outset") +_LENGTH = r"(?:0|-?(?:\d+\.?\d*|\.\d+)(?:px|em|rem)|thin|medium|thick|auto)" +_VALUES = { + "font-size": rf"inherit|{_LENGTH}|\d+\.?\d*%", + "line-height": rf"inherit|normal|\d+\.?\d*|\.\d+|{_LENGTH}|\d+\.?\d*%", + "font-weight": r"inherit|normal|bold|bolder|lighter|\d{3}", + "letter-spacing": rf"inherit|normal|{_LENGTH}", + "word-spacing": rf"inherit|normal|{_LENGTH}", + "border-spacing": rf"{_LENGTH}(?: {_LENGTH})?", + "border-collapse": r"inherit|collapse|separate", +} + + +def _readable(prop: str, value: str) -> bool: + """Whether the estimate can resolve this value: lengths in px, em or rem (a + percentage only for a font size or line height), and the keywords it knows.""" + v = value.strip().lower() + if prop.endswith("-style"): + return v in _STYLES + if prop.startswith(("margin", "padding", "border")) and prop not in _VALUES: + return bool(re.fullmatch(_LENGTH, v)) + return prop == "font-family" or bool(re.fullmatch(_VALUES[prop], v)) + + +def _family(value: str, origin: int) -> list[tuple[str, str]] | None: + """A font family the estimate knows: Superset's own, `inherit`, Inter (its body + font) or a monospace font, whose characters are all 0.6 em.""" + first = value.split(",")[0].strip().strip("'\"").lower() + if not origin or first in ("inherit", "inter") or re.search(r"mono|courier|code|consol", first): + return [("font-family", value)] + return None + + +def _reaches_markdown(selector: str) -> bool: + """Whether a selector the estimate can't match might still style markdown text: + it names the block, or names only elements and layout around it.""" + if "dashboard-markdown" in selector or "MARKDOWN-" in selector: + return True + # Superset's test hooks (data-test, data-test-chart-name) mark its other components. + if "[data-test" in selector: + return False + classes = set(re.findall(r"\.([\w-]+)", re.sub(r"\[[^\]]*\]", "", selector))) + return not (classes - _AROUND) and "#" not in selector + + +@lru_cache(maxsize=256) +def read_css(css: str | None, origin: int = 1, start: int = 0) -> Stylesheet: + """The rules of a stylesheet the estimate applies, in source order, and the + selectors and properties it leaves unread that could size markdown text.""" + rules, unread = [], [] + order = start + for selectors, body, nested in _rules_of(css or ""): + decls = _declarations(body) + sizing = [(p, v, imp) for p, v, imp in decls if _SIZING.match(p)] + if not sizing: + continue + for text in _SPLIT_TOP.split(selectors): + text = " ".join(text.split()) + sel = None if nested else _parse_selector(text) + if sel is None or (origin and sel.specificity[1] == 0): + # A selector with no class can't be ordered against Superset's own, and + # whether an at-rule applies depends on the window. + if _reaches_markdown(text): + shown = f"{nested} {{ {text} }}" if nested else text + unread += [(shown, p) for p, _, _ in sizing] + continue + # The block's own boxes: their padding is Superset's, set more specifically + # than a stylesheet can be sure to beat, so a change to it isn't read; their + # margins sit outside the text. + box = bool(sel.compounds[-1][1] & set(_CHAIN)) + longs = [] + for prop, value, imp in sizing: + parts = None + if box and prop.startswith("margin"): + continue + if prop == "font-family": + parts = _family(value, origin) + elif _READ.match(prop) and not (box and prop.startswith("padding")): + parts = _longhands(prop, value) + if parts is not None and not all(_readable(p, v) for p, v in parts): + parts = None + if parts is None: + if _reaches_markdown(text): + unread.append((text, prop)) + continue + longs += [(p, v, imp) for p, v in parts] + if longs: + order += 1 + rules.append(_Rule(sel, tuple(longs), sel.specificity, origin, order)) + return Stylesheet(tuple(rules), tuple(dict.fromkeys(unread))) + + +@lru_cache(maxsize=8) +def _base(release: Release) -> tuple[_Rule, ...]: + """Superset's own rules, all below any dashboard rule with a class in it (their + real selectors are elements, `body h1`), in their own order among themselves, + and the protected ones at the specificity they have in Superset.""" + rules = [_Rule(r.selector, r.decls, (0, 0, r.specificity[1] * 100 + r.specificity[2]), 0, + r.order) for r in read_css(release.css, origin=0).rules] + for k, (selector, decls, spec) in enumerate(release.protected): + for r in read_css(f"{selector} {{ {decls} }}", origin=0, start=10_000 + k * 10).rules: + rules.append(_Rule(r.selector, r.decls, spec, 0, r.order)) + return tuple(rules) + + +# -- parsing: markdown to blocks ---------------------------------------------------- + + +@dataclass +class Block: + kind: str # p | h | list | table | quote | code | hr + level: int = 0 # a heading's level + texts: list[str] = field(default_factory=list) # p: its lines between hard breaks + items: list[tuple[int, str]] = field(default_factory=list) # list: (depth, text) + ordered: list[bool] = field(default_factory=list) # list: each item's list is numbered + loose: bool = False # list items separated by blank lines + rows: list[list[str]] = field(default_factory=list) # table rows, header first + lines: int = 0 # code: its lines + + +_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") +_ATX = re.compile(r"^ {0,3}(#{1,6})(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$") +_SETEXT = re.compile(r"^ {0,3}(?:=+|-+)[ \t]*$") +_HR = re.compile(r"^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$") +_ITEM = re.compile(r"^([ \t]*)([-*+]|\d{1,9}[.)])(?:[ \t]+(.*)|[ \t]*$)") +_QUOTE = re.compile(r"^ {0,3}> ?(.*)$") +_DELIM = re.compile(r"^[ \t]*\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$") +_HTML_H = re.compile(r"^\s*]*>(.*?)\s*$", re.I) +_HARD = re.compile(r"(?: {2,}|\\)$") +_BR = re.compile(r"", re.I) +_MEDIA = re.compile(r"!\[[^\]]*\]\([^)]*\)|<(?:img|iframe|video|svg|object|embed)\b[^>]*>", re.I) + + +def _starts_block(line: str) -> bool: + return bool(_FENCE.match(line) or _ATX.match(line) or _HR.match(line) + or _ITEM.match(line) or _QUOTE.match(line)) + + +def _cells(line: str) -> list[str]: + s = line.strip() + s = s[1:] if s.startswith("|") else s + s = s[:-1] if s.endswith("|") and not s.endswith("\\|") else s + return [c.strip() for c in re.split(r"(? tuple[list[Block], bool]: + """The blocks a markdown text draws, and whether it holds an image or embedded + media, whose height the text can't tell. CommonMark's common shapes: ATX and + setext headings, paragraphs with soft and hard breaks, lists (nested, tight or + loose), GFM tables, block quotes, fenced code and rules. HTML reads as its text, + with
a line break.""" + lines = markdown.replace("\r\n", "\n").replace("\t", " ").split("\n") + blocks: list[Block] = [] + i, n = 0, len(lines) + while i < n: + line = lines[i] + if not line.strip(): + i += 1 + elif m := _FENCE.match(line): + j = i + 1 + while j < n and not lines[j].strip().startswith(m.group(1)): + j += 1 + blocks.append(Block("code", lines=max(1, j - i - 1))) + i = j + 1 + elif m := _ATX.match(line) or _HTML_H.match(line): + level = len(m.group(1)) if m.re is _ATX else int(m.group(1)) + blocks.append(Block("h", level=level, texts=[m.group(2) or ""])) + i += 1 + elif _HR.match(line): + blocks.append(Block("hr")) + i += 1 + elif _QUOTE.match(line): + body = [] + while i < n and (m := _QUOTE.match(lines[i])): + body.append(m.group(1)) + i += 1 + inner, _ = parse("\n".join(body)) + texts = [t for b in inner for t in (b.texts or [it for _, it in b.items])] + blocks.append(Block("quote", texts=texts or [""])) + elif "|" in line and i + 1 < n and "-" in lines[i + 1] and _DELIM.match(lines[i + 1]): + rows = [_cells(line)] + i += 2 + while i < n and lines[i].strip() and "|" in lines[i]: + rows.append(_cells(lines[i])) + i += 1 + blocks.append(Block("table", rows=rows)) + elif _ITEM.match(line): + i = _parse_list(lines, i, blocks) + else: + # A paragraph runs to a blank line or another block; an underline of = or - + # makes it a heading (setext). + para = [line] + i += 1 + while (i < n and lines[i].strip() and not _starts_block(lines[i]) + and not _SETEXT.match(lines[i])): + para.append(lines[i]) + i += 1 + if i < n and _SETEXT.match(lines[i]): + level = 1 if lines[i].strip()[0] == "=" else 2 + blocks.append(Block("h", level=level, texts=[" ".join(l.strip() for l in para)])) + i += 1 + else: + blocks.append(Block("p", texts=_paragraph_lines(para))) + return blocks, bool(_MEDIA.search(markdown)) + + +def _paragraph_lines(raw: list[str]) -> list[str]: + """A paragraph's lines between hard breaks (two trailing spaces, a backslash or +
); a soft break joins its lines with a space.""" + out, cur = [], [] + for k, line in enumerate(raw): + cur.append(line.strip().rstrip("\\").strip()) + if _HARD.search(line) and k < len(raw) - 1: + out.append(" ".join(cur)) + cur = [] + out.append(" ".join(cur)) + return [seg for text in out for seg in _BR.split(text)] + + +def _marker(marker: str) -> str: + """What makes items one list: the same bullet, or numbers with the same delimiter.""" + return marker[-1] if marker[0].isdigit() else marker + + +def _parse_list(lines: list[str], i: int, blocks: list[Block]) -> int: + """One list from line i, nested items and all; returns the line after it. Another + bullet or delimiter at the list's own level starts another list.""" + block = Block("list") + indents: list[int] = [] # each open level's marker indent + kind = None # the top level's marker + gap = False # a blank line since the last line of an item + while i < len(lines): + line = lines[i] + if not line.strip(): + gap = True + elif m := _ITEM.match(line): + indent = len(m.group(1)) + if kind is not None and indent <= indents[0] and _marker(m.group(2)) != kind: + break + kind = kind or _marker(m.group(2)) + while indents and indent < indents[-1]: + indents.pop() + if not indents or indent > indents[-1] + 1: + indents.append(indent) + if gap and block.items and len(indents) == 1: + block.loose = True + block.items.append((len(indents) - 1, m.group(3) or "")) + block.ordered.append(m.group(2)[0].isdigit()) + gap = False + elif (gap and not line.startswith(" ")) or (not gap and _starts_block(line)): + break # the list has ended + else: + depth, text = block.items[-1] + block.items[-1] = (depth, f"{text} {line.strip()}".strip()) + block.loose = block.loose or gap # a second paragraph in an item + gap = False + i += 1 + blocks.append(block) + return i + + +# -- the element tree --------------------------------------------------------------- + + +@dataclass(eq=False) +class Node: + """An element as react-markdown draws it, with what selectors match on.""" + + tag: str + classes: frozenset = frozenset() + children: list[Node] = field(default_factory=list) + texts: list[str] = field(default_factory=list) # its lines of inline text + code_lines: int = 0 # pre: its lines + parent: Node | None = None + prev: Node | None = None # the element before it in its parent + first: bool | None = None + last: bool | None = None + + def add(self, *kids: Node) -> Node: + self.children.extend(kids) + return self + + +def _li(text: str, loose: bool) -> Node: + return Node("li").add(Node("p", texts=_BR.split(text)) if loose + else Node("#text", texts=_BR.split(text))) + + +def tree(markdown: str) -> Node: + """The markdown block's elements under its holder, the holder under the + resizable container under `.dashboard-markdown`, as Superset draws them.""" + holder = Node("div", frozenset({"dashboard-component", "dashboard-component-chart-holder"})) + for b in parse(markdown)[0]: + if b.kind == "p": + holder.add(Node("p", texts=b.texts)) + elif b.kind == "h": + holder.add(Node(f"h{b.level}", texts=_BR.split(b.texts[0]))) + elif b.kind == "hr": + holder.add(Node("hr")) + elif b.kind == "code": + holder.add(Node("pre", code_lines=b.lines).add(Node("code"))) + elif b.kind == "quote": + holder.add(Node("blockquote").add(*(Node("p", texts=_BR.split(t)) for t in b.texts))) + elif b.kind == "table": + head = Node("tr").add(*(Node("th", texts=[c]) for c in b.rows[0])) + body = [Node("tr").add(*(Node("td", texts=[c]) for c in r)) for r in b.rows[1:]] + table = Node("table").add(Node("thead").add(head)) + if body: + table.add(Node("tbody").add(*body)) + holder.add(table) + else: + stack: list[Node] = [] + for (depth, text), ordered in zip(b.items, b.ordered or [False] * len(b.items)): + while len(stack) > depth + 1: + stack.pop() + if len(stack) <= depth: + lst = Node("ol" if ordered else "ul") + (stack[-1].children[-1] if stack else holder).add(lst) + stack.append(lst) + stack[-1].add(_li(text, b.loose and depth == 0)) + resizable = Node("div", frozenset({"resizable-container"})).add(holder) + root = Node("div", frozenset({"dashboard-markdown"})).add(resizable) + _link(root) + return holder + + +def _link(node: Node) -> None: + elements = [c for c in node.children if c.tag != "#text"] + for k, child in enumerate(node.children): + child.parent = node + if child.tag != "#text": + at = elements.index(child) + child.prev = elements[at - 1] if at else None + child.first, child.last = at == 0, at == len(elements) - 1 + _link(child) + + +# -- the cascade -------------------------------------------------------------------- + + +def _match_compound(compound, node: Node) -> bool: + tag, classes, pseudo = compound + if (tag and tag != node.tag) or not classes <= node.classes: + return False + known = {"first-child": node.first, "last-child": node.last, + "only-child": node.first and node.last if None not in (node.first, node.last) + else None} + return all(known[p] is True for p in pseudo) + + +def _around(compound) -> bool: + """A compound that may match an element above `.dashboard-markdown`, which the + estimate doesn't see: a plain element, or layout classes.""" + tag, classes, pseudo = compound + return not pseudo and tag in (None, "div", "body", "html", "main", "section") and \ + classes <= (_AROUND - set(_CHAIN)) + + +def _matches(sel: _Selector, k: int, node: Node | None) -> bool: + if node is None: + return all(_around(c) for c in sel.compounds[:k + 1]) + if not _match_compound(sel.compounds[k], node): + return False + if k == 0: + return True + comb = sel.combinators[k - 1] + if comb == ">": + return _matches(sel, k - 1, node.parent) + if comb == " ": + up = node.parent + while up is not None: + if _matches(sel, k - 1, up): + return True + up = up.parent + return _matches(sel, k - 1, None) + sib = node.prev + while sib is not None: + if _matches(sel, k - 1, sib): + return True + sib = sib.prev if comb == "~" else None + return False + + +@dataclass(frozen=True) +class Style: + font: float + line: float # px + line_factor: float | None # how the line height inherits: a factor, or None (px) + weight: float + letter: float = 0.0 + word: float = 0.0 + mono: bool = False + spacing: float = 0.0 # a table's vertical border-spacing (inherited) + collapse: bool = False # border-collapse (inherited) + margin: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) + padding: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) + border: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) + + @property + def inset(self) -> float: + """Horizontal room it takes from its parent's width.""" + return sum(s[1] + s[3] for s in (self.margin, self.padding, self.border)) + + @property + def edge_top(self) -> float: + return self.padding[0] + self.border[0] + + @property + def edge_bottom(self) -> float: + return self.padding[2] + self.border[2] + + +_LEN = re.compile(r"^(-?(?:\d+\.?\d*|\.\d+))(px|em|rem|%)?$") + + +def _length(value: str, font: float, parent_font: float | None = None) -> float | None: + """A length in px; em counts in `font`, or in `parent_font` for a font size.""" + v = value.strip().lower() + if v in ("auto", "0"): + return 0.0 + if v in ("thin", "medium", "thick"): + return {"thin": 1.0, "medium": 3.0, "thick": 5.0}[v] + m = _LEN.match(v) + if not m: + return None + n, unit = float(m.group(1)), m.group(2) + if unit == "px": + return n + if unit == "em": + return n * (parent_font if parent_font is not None else font) + if unit == "rem": + return n * ROOT_FONT + if unit == "%" and parent_font is not None: + return n * parent_font / 100 + return None if n else 0.0 + + +def _compute(node: Node, parent: Style, rules: tuple[_Rule, ...]) -> Style: + won: dict[str, tuple] = {} + for r in rules: + if not _matches(r.selector, len(r.selector.compounds) - 1, node): + continue + for prop, value, imp in r.decls: + key = (imp, r.specificity, r.origin, r.order) + if prop not in won or key >= won[prop][0]: + won[prop] = (key, value) + get = {p: v for p, (_, v) in won.items()} + font = parent.font + if "font-size" in get: + v = get["font-size"].lower() + font = parent.font if v == "inherit" else (_length(v, font, parent.font) or font) + factor, line = parent.line_factor, parent.line if parent.line_factor is None else None + if "line-height" in get: + v = get["line-height"].lower() + if v == "normal": + factor, line = 1.21, None # Inter's ascent and descent + elif re.fullmatch(r"\d+\.?\d*|\.\d+", v): + factor, line = float(v), None + elif v.endswith("%") and _LEN.match(v): + factor, line = None, float(v[:-1]) * font / 100 + elif v != "inherit" and (px := _length(v, font)) is not None: + factor, line = None, px + line = factor * font if factor is not None else line + weight = parent.weight + if "font-weight" in get: + v = get["font-weight"].lower() + weight = {"normal": 400, "bold": 700, "lighter": 400, + "bolder": 700 if parent.weight < 600 else 900}.get( + v, float(v) if v.replace(".", "").isdigit() else weight) + + def spacing(prop: str, current: float) -> float: + v = get.get(prop) + if v is None or v.lower() == "inherit": + return current + return 0.0 if v.lower() == "normal" else (_length(v, font) or 0.0) + + def sides(prop: str) -> tuple[float, ...]: + out = [] + for s in _SIDES: + v = _length(get.get(f"{prop}-{s}", "0"), font) or 0.0 + out.append(v if prop == "margin" else max(0.0, v)) + return tuple(out) + + border = tuple(0.0 if get.get(f"border-{s}-style", "none").lower() in ("none", "hidden") + else (_length(get.get(f"border-{s}-width", "medium"), font) or 0.0) + for s in _SIDES) + fam = get.get("font-family", "").split(",")[0].strip().strip("'\"").lower() + mono = parent.mono if not fam or fam == "inherit" else bool( + re.search(r"mono|courier|code|consol", fam)) + gap = parent.spacing + if "border-spacing" in get: + parts = get["border-spacing"].split() + gap = _length(parts[-1], font) or 0.0 + collapse = get.get("border-collapse", "collapse" if parent.collapse else "separate") + return Style(font, line, factor, weight, spacing("letter-spacing", parent.letter), + spacing("word-spacing", parent.word), mono, gap, + collapse.lower() == "collapse", sides("margin"), sides("padding"), border) + + +class _Styles: + """Every element's computed style for one release and one dashboard stylesheet.""" + + def __init__(self, release: Release, sheet: Stylesheet): + self.rules = _base(release) + sheet.rules + font, factor = release.root + self.root = Style(font, font * factor, factor, 400.0) + self.cache: dict[tuple, Style] = {} + + def of(self, node: Node) -> Style: + key = (node, ()) # the node itself, not its id: a freed node's id comes back + if key not in self.cache: + parent = self.of(node.parent) if node.parent is not None else self.root + self.cache[key] = parent if node.tag == "#text" else _compute(node, parent, self.rules) + return self.cache[key] + + def inline(self, node: Node, tags: tuple[str, ...]) -> Style: + """The style of text inside these inline elements, innermost last.""" + key = (node, tags) + if key not in self.cache: + if not tags: + self.cache[key] = self.of(node) + else: + at = node + for tag in tags: + at = Node(tag, parent=at) + parent = self.inline(node, tags[:-1]) + self.cache[key] = _compute(at, parent, self.rules) + return self.cache[key] + + +# -- text width --------------------------------------------------------------------- + +_LINK = re.compile(r"!?\[([^\]]*)\]\([^)]*\)|!?\[([^\]]*)\]\[[^\]]*\]") +_AUTOLINK = re.compile(r"<((?:https?|mailto):[^>\s]+)>") +_TAG = re.compile(r"<(/?)([A-Za-z][\w-]*)[^>]*?(/?)>") +_INLINE_TAGS = {"a", "abbr", "b", "cite", "code", "del", "em", "i", "ins", "kbd", "mark", + "q", "s", "samp", "small", "span", "strong", "sub", "sup", "u", "var"} +_ESCAPED = re.compile(r"\\([!-/:-@\[-`{-~])") + + +def _runs(text: str) -> list[tuple[str, tuple[str, ...]]]: + """Inline markdown and HTML as runs of the text a reader sees, each with the + inline elements it sits in, outermost first.""" + keep: list[str] = [] + + def hold(s: str) -> str: + keep.append(s) + return f"\ue000{len(keep) - 1}\ue001" + + text = _ESCAPED.sub(lambda m: hold(html.escape(m.group(1))), text) + text = re.sub(r"(`+)(.+?)\1", lambda m: f"{hold(html.escape(m.group(2).strip()))}", + text) + text = _MEDIA.sub("", text) + text = _LINK.sub(lambda m: f"{m.group(1) or m.group(2) or ''}", text) + text = _AUTOLINK.sub(r"\1", text) + for _ in range(2): + text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*|(?{m.group(1) or m.group(2)}", text) + text = re.sub(r"~~(?=\S)(.+?)(?<=\S)~~", r"\1", text) + text = re.sub(r"\*(?=[^\s*])(.+?)(?<=[^\s*])\*|(?{m.group(1) or m.group(2)}", text) + out: list[tuple[str, tuple[str, ...]]] = [] + stack: list[str] = [] + at = 0 + for m in _TAG.finditer(text + ""): + chunk = text[at:m.start()] if m.end() <= len(text) else text[at:] + if chunk: + chunk = re.sub(r"\ue000(\d+)\ue001", lambda h: keep[int(h.group(1))], chunk) + out.append((html.unescape(chunk), tuple(stack))) + if m.end() > len(text): + break + at = m.end() + closing, tag, selfclose = m.group(1), m.group(2).lower(), m.group(3) + if tag not in _INLINE_TAGS or selfclose: + continue + if not closing: + stack.append(tag) + elif tag in stack: + del stack[len(stack) - 1 - stack[::-1].index(tag):] + return out + + +def plain(text: str) -> str: + """Inline markdown and HTML reduced to the text a reader sees.""" + return "".join(t for t, _ in _runs(text)) + + +def _char_em(ch: str, bound: str = "high") -> float: + o = ord(ch) + if 32 <= o < 127: + return _EM[o - 32] + if ch in _EM_MORE: + return _EM_MORE[ch] + narrow, wide = _EM_UNKNOWN[bound] + return wide if unicodedata.east_asian_width(ch) in ("W", "F") else narrow + + +def em_width(text: str) -> float: + """A text's width on one line, in em of Inter at 400.""" + return sum(_char_em(ch) for ch in text) + + +# A break opportunity follows a space, an en or em dash, or a hyphen before a letter: +# Chromium breaks "great-circle" and "Jul–Sep" there, and long hyphenated URLs too. +_BREAK_AFTER = "–—" + + +def _wrap(chars: list[tuple], width: float) -> list[set]: + """The lines (character, px, style) take in `width` px, wrapped greedily as the + browser wraps them, each as the set of inline styles on it. A word wider than the + line takes a line of its own (the holder clips it sideways); an empty text still + takes its line.""" + words: list[tuple[float, list[tuple[float, set]]]] = [] # (space before, pieces) + space, pieces, piece, on = 0.0, [], 0.0, set() + for k, (ch, w, st) in enumerate(chars): + if ch in " \t\n": + if pieces or piece: + words.append((space, pieces + [(piece, on)])) + pieces, piece, space, on = [], 0.0, 0.0, set() + space = space or w + continue + piece += w + on.add(st) + nxt = chars[k + 1][0] if k + 1 < len(chars) else "" + if ch in _BREAK_AFTER or (ch == "-" and nxt.isalpha()): + pieces.append((piece, on)) + piece, on = 0.0, set() + if pieces or piece: + words.append((space, pieces + [(piece, on)])) + lines, x = [set()], 0.0 + for space, parts in words: + for k, (w, styles) in enumerate(parts): + add = w + (space if k == 0 and x > 0 else 0.0) + if x > 0 and x + add > width: + lines.append(set()) + x = w + else: + x += add + lines[-1] |= styles + return lines + + +# Inter's ascent and descent in em: where an inline box sits on the line. +_ASCENT, _DESCENT = 0.96875, 0.2421875 + + +def _line_box(block: Style, styles: set, bound: str) -> float: + """A line's height: its block's line height, or more where inline text of + another size or line height on it reaches past it (CSS 2.1 10.8). Chromium draws + such a line up to a px taller than the arithmetic says (a smaller in a + paragraph); the high bound allows a px for it.""" + def span(s: Style) -> tuple[float, float]: + lead = (s.line - (_ASCENT + _DESCENT) * s.font) / 2 + return -(_ASCENT * s.font + lead), _DESCENT * s.font + lead + + others = [s for s in styles if (s.font, s.line) != (block.font, block.line)] + if not others: + return block.line + spans = [span(block)] + [span(s) for s in others] + height = max(b for _, b in spans) - min(t for t, _ in spans) + return height + (1.0 if bound == "high" else 0.0) + + +def _measure(runs, style_of, scale: float, bound: str) -> list[tuple]: + """Each character of the runs with its width in px and its style, an inline + element's padding, border and margin on its first and last character.""" + out: list[tuple] = [] + prev: tuple[str, ...] = () + for k, (text, tags) in enumerate(runs): + st = style_of(tags) + px = st.font * _weight_factor(st.weight) * scale + nxt = runs[k + 1][1] if k + 1 < len(runs) else () + start = len(out) + for ch in text: + em = _EM_MONO if st.mono and ch not in " \t\n" else _char_em(ch, bound) + out.append((ch, em * px + st.letter + (st.word if ch == " " else 0.0), st)) + if len(out) > start: + opened = [t for d, t in enumerate(tags) if d >= len(prev) or prev[d] != t] + closed = [t for d, t in enumerate(tags) if d >= len(nxt) or nxt[d] != t] + for depth in range(len(tags)): + edge = style_of(tags[:depth + 1]) + left = edge.margin[3] + edge.padding[3] + edge.border[3] + right = edge.margin[1] + edge.padding[1] + edge.border[1] + if tags[depth] in opened and left: + ch, w, s = out[start] + out[start] = (ch, w + left, s) + if tags[depth] in closed and right: + ch, w, s = out[-1] + out[-1] = (ch, w + right, s) + prev = tags + return out + + +def wrapped_lines(text: str, width: float, font_px: float = 14.0, scale: float = 1.0) -> int: + """Lines a plain text takes in `width` px at `font_px` in Inter at 400.""" + return len(_wrap([(ch, _char_em(ch) * font_px * scale, None) for ch in text], width)) + + +# -- layout: elements to px --------------------------------------------------------- + + +def _collapse(margins: list[float]) -> float: + """Adjoining margins collapse: the largest positive one plus the most negative.""" + return max([m for m in margins if m > 0], default=0.0) + min([m for m in margins if m < 0], + default=0.0) + + +class _Layout: + def __init__(self, styles: _Styles, scale: float, bound: str): + self.styles, self.scale, self.bound = styles, scale, bound + + def lines(self, node: Node, text: str, width: float) -> list[set]: + runs = _runs(text) or [("", ())] + chars = _measure(runs, lambda tags: self.styles.inline(node, tags), self.scale, + self.bound) + return _wrap(chars, width) + + def text_height(self, node: Node, texts: list[str], width: float) -> float: + st = self.styles.of(node) + return sum(_line_box(st, line, self.bound) + for t in texts for line in self.lines(node, t, width)) + + def table(self, node: Node, width: float) -> float: + """Rows of one line while the columns fit side by side. Past that, each column + gets a share of the width in proportion to its widest cell, roughly as the + browser's automatic table layout does, and a row is as tall as its tallest cell.""" + st = self.styles.of(node) + gap = 0.0 if st.collapse else st.spacing + rows = [tr for group in node.children for tr in group.children] + cells = [[(c, self.styles.of(c)) for c in tr.children] for tr in rows] + cols = max(len(r) for r in cells) + inset = max((cs.inset for r in cells for _, cs in r), default=0.0) + room = width - gap * (cols + 1) - inset * cols + + def one_line(c: Node, cs: Style) -> float: + runs = _runs(c.texts[0]) or [("", ())] + return sum(w for _, w, _ in _measure(runs, lambda t: self.styles.inline(c, t), + self.scale, self.bound)) + + widest = [max((one_line(*r[k]) if k < len(r) else 0.0) for r in cells) + for k in range(cols)] + total = sum(widest) or 1.0 + height = gap + for r in cells: + tallest = 0.0 + for k, (c, cs) in enumerate(r): + share = room if total <= room else room * widest[k] / total + tallest = max(tallest, self.text_height(c, c.texts, share) + cs.edge_top + + cs.edge_bottom) + height += tallest + gap + return height + + def box(self, node: Node, width: float) -> tuple[list[float], float, list[float]]: + """(the margins at its top edge, its height, the margins at its bottom edge): + the margins that collapse with whatever sits above or below it.""" + st = self.styles.of(node) + inner = width - (0.0 if node.tag == "#text" else st.inset) + if node.tag == "table": + height = self.table(node, inner) + elif node.tag == "pre": + height = node.code_lines * st.line + elif node.texts or node.tag == "#text": + height = self.text_height(node, node.texts or [""], inner) + else: + height, first, last = self.stack(node.children, inner, st.edge_top == 0, + st.edge_bottom == 0) + return ([st.margin[0], *first], height + st.edge_top + st.edge_bottom, + [st.margin[2], *last]) + if node.tag == "#text": + return [], height, [] + return [st.margin[0]], height + st.edge_top + st.edge_bottom, [st.margin[2]] + + def stack(self, children: list[Node], width: float, open_top: bool, + open_bottom: bool) -> tuple[float, list[float], list[float]]: + """Children stacked with their margins collapsed: (the height to the last one's + bottom edge, the margins passed up through an open top, through an open + bottom). A closed edge (padding or a border) keeps them inside.""" + y, through_top, pending = 0.0, [], None + for child in children: + top, height, bottom = self.box(child, width) + if pending is None: + if open_top: + through_top = top + else: + y += _collapse(top) + else: + y += _collapse(pending + top) + y += height + pending = bottom + if pending is None: + return 0.0, [], [] + if open_bottom: + return y, through_top, pending + return y + _collapse(pending), through_top, [] + + +def layout(holder: Node, width: float, styles: _Styles, scale: float = 1.0, + bound: str = "high") -> tuple[float, float]: + """(how far down the last line reaches, the height that shows everything with no + inner scrollbar), in px from the holder's top edge, for text `width` px wide. + The holder is a scroll container: its padding keeps the first block's margin + above and the last one's below inside it.""" + lay = _Layout(styles, scale, bound) + text, _, below = lay.stack(holder.children, width, False, True) + border = styles.of(holder).border + top = PAD + border[0] + text + return top, top + _collapse(below) + PAD + border[2] + + +# -- the estimate ------------------------------------------------------------------- + + +def text_width(cols: int, grid: float) -> float: + """The width a block `cols` twelfths wide wraps its text in, on a grid `grid` px.""" + column = (grid - 11 * GUTTER) / 12 + return cols * column + (cols - 1) * GUTTER - 2 * PAD + + +def box_px(height: float) -> float: + """The px Superset gives a block of this spec height: never under 40.""" + return max(MIN_BOX, height * UNIT) + + +@dataclass(frozen=True) +class Fit: + text_low: float # px: the last line reaches at least this far down + need_low: float # px: the block needs at least this much + need_high: float # px: this much shows it whole on every release + media: bool # an image or embed adds height the text can't tell + + @property + def units(self) -> float: + """need_high in spec units, up to a whole 8 px grid row (a fifth of a unit). + Chromium's scroll height is whole px, so a block under half a px over doesn't + scroll: an 11.5 px footer line takes 48.1 px and fits 48.""" + return math.ceil(round((self.need_high - SNAP) / 8, 6)) / 5 + + def short(self, box: float) -> bool: + """Whether the block needs more than a box this tall: it scrolls.""" + return self.need_low > box + SNAP + + +def releases_from(floor: tuple[int, int, int] | None) -> tuple[Release, ...]: + """The releases a spec can go to, given the oldest one it needs.""" + return tuple(r for r in RELEASES if floor is None or r.newest >= floor) or RELEASES[-1:] + + +def estimate(markdown: str, cols: int, filter_bar: bool | None = False, + css: str | Stylesheet | None = None, viewport: float = VIEWPORT, + releases: tuple[Release, ...] = RELEASES) -> Fit: + """How a markdown text fits a block `cols` twelfths wide in a window `viewport` + px wide, beside the open vertical filter bar or not, under the dashboard's CSS. + None says the bar may be there or not: the low bound then wraps without it and + the high bound beside it.""" + sheet = css if isinstance(css, Stylesheet) else read_css(css) + holder = tree(markdown) + media = parse(markdown)[1] + lows, highs = [], [] + wide = max(r.grid(viewport, bool(filter_bar)) for r in releases) + for r in releases: + styles = _Styles(r, sheet) + lows.append(layout(holder, text_width(cols, wide), styles, LOW_SCALE, "low")) + # Windows narrows the grid by the page's own scrollbar too. + grid = r.grid(viewport, filter_bar is not False) - SCROLLBAR + highs.append(layout(holder, text_width(cols, grid) - SCROLLBAR, styles, HIGH_SCALE, + "high")[1]) + return Fit(min(a for a, _ in lows), min(b for _, b in lows), max(highs), media) diff --git a/chartwright/design/model.py b/chartwright/design/model.py index ad50749..3edf9d3 100644 --- a/chartwright/design/model.py +++ b/chartwright/design/model.py @@ -26,13 +26,54 @@ # "7" = fills keep a null record when the author edits one (an edit later deleted stays # deleted), and default.stale-record: a fixable finding for a renamed or removed chart's # design.filled entry, which now validates. Both change what `--fix` writes. -DESIGN_BRAIN_VERSION = "7" +# "8" = one height question for tables and pivots (sec.15.42): table_visible_ratio 1 and +# every-row targets in size.table-window, size.pivot-window reading pivot rows instead of +# row_limit, size.grid-fit counting every pivot layout. No new rule, but a table can +# newly warn (and block a strict gate) and `--fix` writes different heights. +# "9" = data.rolling-window-span, a new warn-severity rule for a rolling trendline KPI +# whose time range can't hold its window and comparison; default.compare-suffix writes +# 'vs prior 12 months' between trailing windows; chart.ordinal-order leaves a heatmap axis +# ordered by value alone; default.date-tile fills date_format on a big number of a date +# column (data-aware), and a big number shown as a date gets no number format from +# narrative.big-number-format, default.count-format or a standard. +# Also "9", the waterfall: chart.waterfall-additive and chart.waterfall-steps (warn) can +# newly block a strict gate, chart.waterfall-order, chart.waterfall-colors and +# chart.waterfall-axis-titles are new findings, and default.value-labels and +# default.count-format now fill a waterfall's show_value and number_format. The box +# plot: chart.box-plot-observations and chart.box-plot-groups (warn), and +# chart.ordinal-order now reads a box plot's groups. +# "10" = no DataTables chrome on a table whose rows all show: default.search-box fills only +# a paged table whose rows outgrow its panel (and --fix removes a box it filled on one +# whose rows all show), and size.table-chrome, a new info rule, names an author's page +# size or search box there. +# "11" = size.markdown-fit, a new warn-severity rule for markdown text cut off by its +# block in a 1440 px window, under the dashboard's own CSS, and layout.markdown-height +# fixes a one-line block to the height its line takes (1.6 to 2.4 units) instead of 2, +# from the same estimate. +# "12" = default.heatmap-label-room, a fill (left_margin on every heatmap, which 6.1.0 +# otherwise draws with its longest y label cut off); it changes what `--fix` writes. +# With it, a bar's sort_by: data.top-n-sort warns on a row_limit under sort_by "total" +# (the query is still ordered by the first metric), and chart.ordinal-order leaves a +# bar ranked by sort_by alone. +# "14" = big-number colour rules: chart.format-bands reads them (and no longer counts an +# '=' band as everything above it), chart.color-contrast (warn) and +# narrative.kpi-thresholds (info). A new warn rule can newly block a strict gate. +# "15" = chart.format-bands reads each colour rule's own range: the new >=, <=, != and +# between_inclusive operators take their bounds in, and an '=' rule stays one value. +# Findings change, no rule is added. +# "16" = the readability.* family (sec.17, "Type sizes"): table-text and chart-text warn +# when the dashboard's CSS or theme sets text below its floor, so a strict gate can newly +# block, and kpi-text raises a big number's height until its value and subtitle reach +# theirs, which changes what `--fix` writes. +# "17" = chart.axis-titles and layout.thin-subtabs (info), and default.value-labels now +# labels grouped bars too, which changes what `--fix` writes. +DESIGN_BRAIN_VERSION = "17" KPI_TYPES = {"big_number_total", "big_number_trend"} TIMESERIES_TYPES = {"timeseries_line", "timeseries_bar", "timeseries_area", "timeseries_scatter"} -AXIS_TYPES = TIMESERIES_TYPES | {"bar", "heatmap", "histogram", "mixed"} +AXIS_TYPES = TIMESERIES_TYPES | {"bar", "heatmap", "histogram", "mixed", "waterfall", "box_plot"} # Charts that are neither KPI nor axis-bearing; a contract test asserts the -# three sets exactly cover CHART_TYPES, so a 15th chart type fails CI until +# three sets exactly cover CHART_TYPES, so a new chart type fails CI until # someone consciously classifies it (and reviews which rules apply). STANDALONE_TYPES = {"pie", "table", "pivot_table", "funnel", "treemap"} @@ -414,15 +455,24 @@ def _bands_from_rows(spec: DashboardSpec, rows) -> list[Band]: def _bands_from_sketch(spec: DashboardSpec, holder) -> list[Band]: + from ..sketch import SketchBlock, SketchColumn + + def height(sc) -> float: + # A header counts nothing, as a header row does in rows mode. + if isinstance(sc, SketchBlock): + return holder.sketch_block_height(sc) if sc.kind == "markdown" else 0.0 + return spec.resolved_height(sc.name) + bands = [] for srow in holder.parsed_sketch(): + if srow.header_band is not None: + continue # a section title between bands holds no charts, as in rows items = [] for child in srow.children: - if hasattr(child, "children"): # SketchColumn: a stack of charts - names = [sc.name for sc in child.children] - total = sum(spec.resolved_height(n) for n in names) - items.append(BandItem(child.width, total, names)) - else: - items.append(BandItem(child.width, spec.resolved_height(child.name), [child.name])) + stack = child.children if isinstance(child, SketchColumn) else [child] + names = [sc.name for sc in stack if not isinstance(sc, SketchBlock)] + # A slot of blocks only (a note, a header) reads as markdown does in rows. + items.append(BandItem(child.width, sum(height(sc) for sc in stack), names, + is_markdown=not names)) bands.append(Band(items)) return bands diff --git a/chartwright/design/presets.py b/chartwright/design/presets.py index 7151b31..1000e3d 100644 --- a/chartwright/design/presets.py +++ b/chartwright/design/presets.py @@ -25,11 +25,12 @@ class Params: kpi_row_max: int kpi_height: int min_axis_height: int - # Min visible rows / row_limit. 0.5 everywhere: below half, the MAJORITY of - # the rows you deliberately asked for sit behind the grid's inner - # scrollbar -- the exact defect size.grid-fit and apply-time smoke exist to - # catch, so the offline rule must not bless it. Still a knob (deployments - # override per audience in design.yaml), just no longer a lenient default. + # Min visible rows / row_limit before size.table-window warns on an unpaged table. + # 1 everywhere: smoke warns after apply when any row the data fills row_limit + # with hides behind the grid's inner scrollbar, so at 0.5 advise passed a + # 25-row table at 13 units that smoke then said needed 21. Still a knob + # (design.yaml, per audience) for tables that rarely reach their row_limit; the + # rule's target height is every row whatever it is set to. table_visible_ratio: float vbar_max_categories: int pie_max_slices: int @@ -48,6 +49,17 @@ class Params: # '%d %b' names a single date only within 365 days: a 366-day span without a # 29 February starts and ends on the same day and month. day_label_max_span_days: int = 365 + # Type floors of the readability.* rules, in CSS px at a 1440 px laptop window + # (DESIGN-BRAIN.md, "Type sizes": IBM Carbon's and Material's type scales and the + # critical print size of reading research). Text read row by row (table and pivot + # cells): Carbon's data-table body and Material's body-medium are 14 px. + min_cell_text_px: float = 14 + # Labels read at a glance (table headers, axis labels, legends, a KPI's subtitle): + # the smallest size either system sets for text a reader must read, 12 px. + min_label_text_px: float = 12 + min_title_text_px: float = 14 # chart titles: Carbon's and Material's headings + min_kpi_value_px: float = 24 # a big number's value: WCAG's large text (18 pt) + min_row_px: float = 24 # a table or pivot row: Carbon's densest (extra small) # chart type -> height (spec units); calibrate/overlay feed this, # size-rule autofixes target it. recommended_heights: dict[str, float] = field(default_factory=dict) @@ -59,15 +71,15 @@ class Params: AUDIENCES: dict[str, Params] = { "executive": Params("executive", fold_units=22, max_row_charts=3, kpi_row_min=2, kpi_row_max=5, kpi_height=5, min_axis_height=8, - table_visible_ratio=0.5, vbar_max_categories=6, + table_visible_ratio=1.0, vbar_max_categories=6, pie_max_slices=5, series_max=5, max_filter_selects=5), "analytical": Params("analytical", fold_units=66, max_row_charts=4, kpi_row_min=2, kpi_row_max=6, kpi_height=4, min_axis_height=6, - table_visible_ratio=0.5, vbar_max_categories=8, + table_visible_ratio=1.0, vbar_max_categories=8, pie_max_slices=7, series_max=10, max_filter_selects=6), "operational": Params("operational", fold_units=22, max_row_charts=4, kpi_row_min=2, kpi_row_max=8, kpi_height=3, min_axis_height=5, - table_visible_ratio=0.5, vbar_max_categories=8, + table_visible_ratio=1.0, vbar_max_categories=8, pie_max_slices=7, series_max=8, max_filter_selects=7), } AUDIENCE_NAMES = tuple(AUDIENCES) diff --git a/chartwright/design/readability.py b/chartwright/design/readability.py new file mode 100644 index 0000000..4b34ae5 --- /dev/null +++ b/chartwright/design/readability.py @@ -0,0 +1,994 @@ +"""Readability: the type sizes a dashboard draws, against floors for reading it on a +laptop (docs/DESIGN-BRAIN.md sec.17, "Type sizes"). + +Three rules, each reading what sets a size before judging it: +- readability.table-text: table and pivot cells, their headers and their rows; +- readability.chart-text: axis labels, legends and chart titles; +- readability.kpi-text: a big number's value, subtitle and comparison, which Superset + sizes by the card's height. + +A size the dashboard sets below its floor (its CSS, or its theme's tokens and ECharts +overrides when a theme is named and `advise --profile` read it) is a warn; a size that +is Superset's own default and below the floor is an info. The floors are audience +parameters (presets.py, min_cell_text_px and the rest), so design.yaml and a standard +tune or lock them. + +What sets each size at 4.1.4, 5.0.0 and 6.1.0 (measured in headless Chromium at a +1440 px window on 2026-10-05, sec.17): +- Table cells and headers: 12 px. `.table-condensed { font-size: @font-size-s }` + (superset-frontend/src/assets/stylesheets/superset.less:178-180; @font-size-s is 12px, + less/variables.less:155) at 4.1.4 and 5.0.0; the table plugin's own + `table.table-condensed { font-size: theme.fontSizeSM }` (plugin-chart-table/src/ + Styles.tsx:74-77) at 6.0.0 and 6.1.0, 12 px from antd's default seed. Neither sets a + size on the cells, which inherit it. The table renders as `table table-striped + table-condensed` (TableChart.tsx:1073 at 4.1.4, :1080 at 5.0.0, :1501 at 6.1.0) inside + `.superset-chart-table` (components/Chart/ChartRenderer.jsx:277-281 at 4.1.4, + ChartRenderer.tsx:457-460 at 6.1.0). +- Pivot cells: 12 px, from `table.pvtTable { font-size }` in the pivot plugin's styled + wrapper (react-pivottable/Styles.js:24-32 at 4.1.4 and 5.0.0, Styles.ts:24-33 at + 6.1.0); its header cells set their own (Styles.js:44-50, Styles.ts:46-51). The + pivot's chart class is `.pivot_table_v_2`, lodash's snakeCase of pivot_table_v2. +- Axis labels and legends: ECharts draws them on a canvas, so no CSS reaches them. + Superset sets no size on either, so zrender's default applies: 12 px + (zrender lib/core/platform.js, DEFAULT_FONT_SIZE). On 6.1.0 a theme's + echartsOptionsOverrides and echartsOptionsOverridesByChartType are merged over every + chart's options (plugin-chart-echarts/src/components/Echart.tsx:254-275), and a + global textStyle.fontSize reaches every label that sets none (echarts 5.6.0 + lib/label/labelStyle.js:312, 386-388). 6.0.0's Echart.tsx reads neither key. +- Chart titles: 16 px, theme.typography.sizes.l on the slice header + (dashboard/components/SliceHeader/index.tsx:62 at 4.1.4, :67 at 5.0.0) and + theme.fontSizeLG (:80 at 6.1.0); `.header-title` inherits it. +- Big numbers: BigNumberViz sizes each line to a share of the card's height + (PROPORTION and computeMaxFontSize, BigNumberViz.tsx:36-57 and :275-321 at 4.1.4), and + chartwright pins header_font_size 0.4 and subheader_font_size 0.15 + (compiler._pin_big_number_fonts). The size is an inline style, so only CSS marked + `!important` changes it. + +Theme tokens (6.0 and later): Superset lets antd derive every map token from the seed, +then lays the theme's own tokens over them (packages/superset-core/src/theme/ +Theme.tsx:109-137 at 6.1.0). antd 5 derives fontSizeSM and fontSizeLG from fontSize +(theme/themes/shared/genFontSizes.js, genFontMapToken.js) and the compact algorithm +derives them from fontSizeSM instead (theme/themes/compact/index.js), so a compact +theme draws table text at 10 px. + +The CSS reader is minimal by design: chartwright/design/markdown_fit.py on the +markdown-fit branch has a cascade reader for markdown, which this one doesn't share, +since that branch isn't merged. It reads top-level rules, and only font-size, +line-height, padding and height, on table and pivot cells, rows and tables, on +`.header-title`, and on the big number's lines. A declaration applies where it beats +Superset's own rule for that property on that release (a tie goes to the dashboard's +CSS, measured on all three); a rule it can't place (a pseudo-class, an attribute, a +chart id, an @media block, var()) is named in the finding, never guessed at. +""" + +from __future__ import annotations + +import math +import re +from dataclasses import dataclass + +from ..compiler import VIZ_TYPE +from ..versions import parse_version +from .content import strip_comments +from .model import AXIS_TYPES, KPI_TYPES, Finding, RuleContext, rule + +# -- how each release draws text --------------------------------------------------- + + +@dataclass(frozen=True) +class Release: + name: str + modern: bool # antd theme tokens: 6.0 and later + root_px: float # 1rem + table_line: float # the line-height factor a table chart's cell draws at + table_pad: float # a table chart cell's padding, above and below + + +# 4.1.4 and 5.0.0: `html, body { font-size: @font-size-base }`, 14px (less/index.less +# :26-30, variables.less:151); Bootstrap's `.table > tbody > tr > td` takes Superset's +# @line-height-base 1.4 (variables.less:162) and `.table-condensed` cells +# @table-condensed-cell-padding 5px (less/cosmo/variables.less:130). +LEGACY = Release("4.1.4 and 5.0.0", False, 14.0, 1.4, 5.0) +# 6.0 and later: the browser's 16 px root, `padding: 0.3rem` (Styles.tsx:79-84), and +# the line height cells inherit from the body: antd's (fontSize + 8) / fontSize. +MODERN = Release("6.0 and later", True, 16.0, 22 / 14, 4.8) + +DEFAULT_TEXT = 12.0 # table and pivot cells, axis labels, legends +DEFAULT_TITLE = 16.0 # chart titles +PIVOT_LINE = 1.4 # table.pvtTable { line-height: 1.4 } (Styles.js:32, Styles.ts:33) +PIVOT_PAD = 4.0 # theme.gridUnit / sizeUnit (Styles.js:106, Styles.ts:113) +BORDER = 1.0 # each row's 1 px rule +VIEWPORT = 1440.0 # the window the floors assume: a laptop +PAGE_MARGIN = 64.0 # the window's width less the grid's +GUTTER = 16.0 # GRID_GUTTER_SIZE between columns +CARD_PAD = 16.0 # a chart holder's padding, each side +FILTER_BAR = 260.0 # OPEN_FILTER_BAR_WIDTH (dashboard/constants.ts:40 at 4.1.4) + +# Superset's own rule for (chart type, element, property): its specificity on +# (LEGACY, MODERN), None where it has none there. A dashboard rule applies where it is +# at least as specific (a tie goes to the dashboard's CSS: measured on all three). +# A styled wrapper's emotion class adds (0,1,0) to the plugin's selectors. +_SUPERSET = { + # `.table-condensed` (superset.less:178); `.css-x table.table-condensed` + ("table", "table", "font-size"): ((0, 1, 0), (0, 2, 1)), + # Bootstrap's `.table > tbody > tr > td` and `.table-condensed > tbody > tr > td`; + # `.css-x table.table-condensed td` (Styles.tsx:79-84) + ("table", "td", "line-height"): ((0, 1, 3), None), + ("table", "th", "line-height"): ((0, 1, 3), None), + ("table", "td", "padding"): ((0, 1, 3), (0, 2, 2)), + ("table", "th", "padding"): ((0, 1, 3), (0, 2, 2)), + # `.css-x table.pvtTable` and its cells (Styles.js:24-50 and :104-106) + ("pivot_table", "table", "font-size"): ((0, 2, 1), (0, 2, 1)), + ("pivot_table", "table", "line-height"): ((0, 2, 1), (0, 2, 1)), + ("pivot_table", "th", "font-size"): ((0, 2, 4), (0, 2, 4)), + ("pivot_table", "td", "padding"): ((0, 2, 4), (0, 2, 4)), + ("pivot_table", "th", "padding"): ((0, 2, 4), (0, 2, 4)), +} + +GRID_KINDS = ("table", "pivot_table") +KIND_WORDS = {"table": "tables", "pivot_table": "pivots"} +KIND_CLASS = {"table": ".superset-chart-table", "pivot_table": ".pivot_table_v_2"} + + +def releases(ctx: RuleContext) -> tuple[Release, ...]: + """The releases this spec can go to: the one resolution checked, 6.0 and later for a + spec naming a theme (versions.py refuses one before 6.0.0), or all of them.""" + known = parse_version(ctx.resolution.superset_version) if ctx.resolution else None + if known: + return (MODERN,) if known >= (6, 0, 0) else (LEGACY,) + if ctx.spec.dashboard.theme: + return (MODERN,) + return (LEGACY, MODERN) + + +def _echarts_overrides(ctx: RuleContext) -> bool: + """A theme's ECharts overrides take effect: on 6.1 and later, read as 6.1.0 when the + release is unknown.""" + known = parse_version(ctx.resolution.superset_version) if ctx.resolution else None + return known is None or known >= (6, 1, 0) + + +# -- theme tokens ------------------------------------------------------------------ + + +def _num(value) -> float | None: + """A token's number: Superset's own config writes some as strings ("8").""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + m = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*(?:px)?\s*", value) + return float(m.group(1)) if m else None + return None + + +def _antd_sizes(base: float) -> list[float]: + """antd 5's font scale from a base size (theme/themes/shared/genFontSizes.js).""" + sizes = [] + for index in range(10): + size = base * math.exp((index - 1) / 5) + whole = math.floor(size) if index > 1 else math.ceil(size) + sizes.append(float(whole // 2 * 2)) + sizes[1] = float(base) + return sizes + + +@dataclass(frozen=True) +class Tokens: + sm: float = DEFAULT_TEXT # fontSizeSM: table and pivot text + lg: float = DEFAULT_TITLE # fontSizeLG: chart titles + body: float = 14.0 # fontSize + line: float = 22 / 14 # lineHeight, the factor cells inherit + theme: str | None = None # the theme's name when one was read + written: tuple[str, ...] = () # the size tokens it writes (algorithm included) + + def size(self, px: float) -> "Size": + """A size these tokens give: the theme's when it writes any size token (one it + writes, or one antd derives from it), else Superset's default.""" + if not self.written: + return Size(px, "superset") + return Size(px, "theme", f"theme {self.theme}: {', '.join(self.written)}") + + +def tokens(theme_json, name: str | None = None) -> Tokens: + """The size tokens a theme's JSON leaves a 6.x dashboard with.""" + if not isinstance(theme_json, dict): + return Tokens() + token = theme_json.get("token") if isinstance(theme_json.get("token"), dict) else {} + algorithm = theme_json.get("algorithm") + algorithms = algorithm if isinstance(algorithm, list) else [algorithm] + compact = "compact" in algorithms + seed = _num(token.get("fontSize")) or 14.0 + base = _antd_sizes(seed)[0] if compact else seed + sizes = _antd_sizes(base) + written = tuple(k for k in ("fontSize", "fontSizeSM", "fontSizeLG", "lineHeight") + if _num(token.get(k)) is not None) + if compact: + written += ("algorithm: compact",) + body = _num(token.get("fontSize")) or sizes[1] + return Tokens(sm=_num(token.get("fontSizeSM")) or sizes[0], + lg=_num(token.get("fontSizeLG")) or sizes[2], + body=body, line=_num(token.get("lineHeight")) or (sizes[1] + 8) / sizes[1], + theme=name, written=written) + + +def _theme(ctx: RuleContext) -> tuple[Tokens, dict | None]: + """The spec's theme as resolution read it: its tokens and its JSON, or the defaults + when the spec names none or this run didn't read it.""" + data = getattr(ctx.resolution, "theme_json", None) if ctx.resolution else None + if not ctx.spec.dashboard.theme or not isinstance(data, dict): + return Tokens(), None + return tokens(data, ctx.spec.dashboard.theme), data + + +def _theme_unread(ctx: RuleContext) -> str: + theme = ctx.spec.dashboard.theme + if theme and _theme(ctx)[1] is None: + return (f"; the theme {theme!r} was not read in this run (`advise --profile` reads " + f"its tokens and ECharts overrides)") + return "" + + +# -- the dashboard's CSS ------------------------------------------------------------ + + +@dataclass(frozen=True) +class Decl: + selector: str + element: str # td | th | row | table | title | kpi-value | kpi-label + kinds: frozenset | None # the chart types it reaches; None: any + prop: str # font-size | line-height | padding-top | padding-bottom | height + value: str + important: bool + spec: tuple[int, int, int] + order: int + area: str | None = None # head | body: a selector naming thead or tbody + + +@dataclass(frozen=True) +class Sheet: + decls: tuple[Decl, ...] = () + # (selector, property, why, element) for each rule the reader can't place, which + # could size the text these rules judge. + unread: tuple[tuple[str, str, str, str], ...] = () + + +# Classes only one chart type's markup carries: a selector naming one reaches only it. +_SCOPE = { + "superset-chart-table": {"table"}, "table": {"table"}, "table-condensed": {"table"}, + "table-striped": {"table"}, "pivot_table_v_2": {"pivot_table"}, + "pvtTable": {"pivot_table"}, "big_number_total": {"big_number_total"}, + "big_number": {"big_number_trend"}, "echarts_timeseries_line": {"timeseries_line"}, + "echarts_timeseries_bar": {"timeseries_bar", "bar"}, "echarts_area": {"timeseries_area"}, + "echarts_timeseries_scatter": {"timeseries_scatter"}, "pie": {"pie"}, + "heatmap_v_2": {"heatmap"}, "histogram_v_2": {"histogram"}, "funnel": {"funnel"}, + "treemap_v_2": {"treemap"}, "mixed_timeseries": {"mixed"}, +} +_TABLE_CLASSES = {"table", "table-condensed", "table-striped", "pvtTable"} +# The layout around every chart, as rendered at 4.1.4 and 6.1.0 (sec.17): naming one +# narrows nothing. Any other class might match nothing, so a rule naming one is unread. +_AROUND = frozenset({ + "dashboard", "dashboard-content", "grid-container", "dashboard-grid", "grid-content", + "grid-row", "background--transparent", "background--white", "dragdroppable", + "dragdroppable-row", "dragdroppable-column", "with-popover-menu", "resizable-container", + "dashboard-component", "dashboard-component-chart-holder", "dashboard-component-tabs", + "dashboard-component-tabs-content", "dashboard-component-row", + "dashboard-component-column", "chart-slice", "dashboard-chart", "chart-container", + "slice_container", "text-container", "superset-legacy-chart-big-number", "no-trendline", + "ant-tabs", "ant-tabs-top", "ant-tabs-content", "ant-tabs-content-top", + "ant-tabs-content-holder", "ant-tabs-tabpane", "ant-tabs-tabpane-active", "ant-layout", + "ant-layout-content"}) +_ROLE_CLASSES = {"header-title": "title", "header-line": "kpi-value", + "subheader-line": "kpi-label", "subtitle-line": "kpi-label"} +_PROPS = {"font", "font-size", "line-height", "padding", "padding-top", "padding-bottom", + "height", "min-height"} +_TOKEN = re.compile(r""" + (?P\s*[>+~]\s*|\s+) + | (?P\*|[A-Za-z][\w-]*) + | \.(?P[\w-]+) + | \#(?P[\w-]+) + | (?P\[[^\]]*\]) + | (?P::[\w-]+(?:\([^)]*\))?) + | (?P:[\w-]+(?:\((?:[^()]|\([^()]*\))*\))?) +""", re.X) + + +def _rules(css: str): + """(selector list, declarations, the at-rule it sits in or None) per style rule.""" + i, n = 0, len(css) + while i < n: + j = css.find("{", i) + if j < 0: + return + prelude = css[i:j].rsplit(";", 1)[-1].strip().lstrip("}").strip() + depth, k = 1, j + 1 + while k < n and depth: + depth += {"{": 1, "}": -1}.get(css[k], 0) + k += 1 + body = css[j + 1:k - 1] + if prelude.startswith("@"): + if re.match(r"@(?:media|supports|layer|container|document)\b", prelude, re.I): + for sel, decls, _ in _rules(body): + yield sel, decls, " ".join(prelude.split()) + else: + yield prelude, body, None + i = k + + +def _compounds(selector: str) -> list[dict] | None: + """A selector as its compounds, or None when it doesn't parse.""" + out = [{"tag": None, "cls": [], "id": [], "attr": [], "pc": [], "pe": []}] + pos = 0 + while pos < len(selector): + m = _TOKEN.match(selector, pos) + if not m or m.end() == pos: + return None + kind = m.lastgroup + if kind == "comb": + out.append({"tag": None, "cls": [], "id": [], "attr": [], "pc": [], "pe": []}) + elif kind == "tag": + out[-1]["tag"] = None if m.group("tag") == "*" else m.group("tag").lower() + else: + out[-1][kind].append(m.group(kind)) + pos = m.end() + return out + + +def _specificity(compounds: list[dict]) -> tuple[int, int, int]: + return (sum(len(c["id"]) for c in compounds), + sum(len(c["cls"]) + len(c["attr"]) + len(c["pc"]) for c in compounds), + sum(bool(c["tag"]) + len(c["pe"]) for c in compounds)) + + +def _place(compounds: list[dict]) -> tuple[str | None, frozenset | None, str | None]: + """(element, chart types reached, why it can't be read) for a selector.""" + every = [x for c in compounds for x in c["cls"]] + if "dashboard-markdown" in every or any(i.startswith("MARKDOWN") for c in compounds + for i in c["id"]): + return None, None, None # markdown text: not what these rules judge + subject = compounds[-1] + if subject["pe"]: + return None, None, None # ::before and the like style generated text + tag, cls = subject["tag"], set(subject["cls"]) + if tag in ("td", "th"): + element = tag + elif tag in ("tr", "tbody", "thead", "tfoot"): + element = "row" + elif (tag == "table" or (cls & _TABLE_CLASSES)) and tag in (None, "table"): + element = "table" + elif cls & set(_ROLE_CLASSES): + element = _ROLE_CLASSES[sorted(cls & set(_ROLE_CLASSES))[0]] + else: + return None, None, None + kinds = None + for c in every: + if c in _SCOPE: + kinds = set(_SCOPE[c]) if kinds is None else kinds & _SCOPE[c] + if element == "table" and kinds is None: + kinds = {"table"} if cls & {"table", "table-condensed", "table-striped"} else ( + {"pivot_table"} if "pvtTable" in cls else set(GRID_KINDS)) + if any(c["pc"] for c in compounds): + return element, None, "it applies to some rows, cells or states (a pseudo-class)" + if (any(c["id"] or c["attr"] for c in compounds) + or any(re.fullmatch(r"dashboard-chart-id-\d+", x) for x in every)): + return element, None, "it applies to some charts (an id or attribute)" + if element in ("td", "th", "row") and cls: + return element, None, "it applies to some cells (a class on the cell)" + known = set(_SCOPE) | _AROUND | _TABLE_CLASSES | set(_ROLE_CLASSES) + for x in every: + if x not in known: + m = re.fullmatch(r"(\w+)_v(\d+)", x) + hint = (f"; Superset renders the class `.{m.group(1)}_v_{m.group(2)}`" if m + else "") + return element, None, f"it names `.{x}`, a class this check doesn't know{hint}" + return element, (frozenset(kinds) if kinds is not None else None), None + + +def _declarations(body: str): + for part in re.split(r";(?![^(]*\))", body): + prop, sep, value = part.partition(":") + if not sep: + continue + value = " ".join(value.split()) + important = bool(re.search(r"!\s*important\s*$", value, re.I)) + value = re.sub(r"\s*!\s*important\s*$", "", value, flags=re.I).strip() + yield prop.strip().lower(), value, important + + +def _longhands(prop: str, value: str) -> list[tuple[str, str]] | None: + """The properties read, from one declaration; None when its value can't be split.""" + if prop in ("font-size", "line-height", "padding-top", "padding-bottom"): + return [(prop, value)] + if prop in ("height", "min-height"): + return [("height", value)] + if prop == "padding": + parts = value.split() + if not 1 <= len(parts) <= 4: + return None + return [("padding-top", parts[0]), + ("padding-bottom", parts[2] if len(parts) > 2 else parts[0])] + # font: [style variant weight] size[/line-height] family; it resets line-height. + m = re.search(r"(?:^|\s)(\d*\.?\d+(?:px|rem|em|%))(?:\s*/\s*(\S+))?\s+\S", value) + return [("font-size", m.group(1)), ("line-height", m.group(2) or "normal")] if m else None + + +def read_css(css: str | None) -> Sheet: + """The declarations of a dashboard's CSS these rules apply, and the rules they can't + place that could size the same text.""" + decls, unread = [], [] + order = 0 + for selectors, body, at in _rules(strip_comments(css or "")): + wanted = [(p, v, imp) for p, v, imp in _declarations(body) if p in _PROPS] + if not wanted: + continue + for text in re.split(r",(?![^(\[]*[)\]])", selectors): + text = " ".join(text.split()) + compounds = _compounds(text) + if compounds is None: + continue + element, kinds, why = _place(compounds) + if element is None: + continue + if at and not why: + why = f"it sits in {at}, which applies at some window sizes" + for prop, value, important in wanted: + if why: + unread.append((text, prop, why, element)) + continue + longs = _longhands(prop, value) + if longs is None or any(re.search(r"\b(?:var|calc|clamp|min|max)\(", v) + for _, v in longs): + unread.append((text, prop, "its value can't be resolved offline", + element)) + continue + order += 1 + tags = {c["tag"] for c in compounds} + area = "head" if "thead" in tags else "body" if "tbody" in tags else None + for p, v in longs: + decls.append(Decl(text, element, kinds, p, v, important, + _specificity(compounds), order, area)) + return Sheet(tuple(decls), tuple(dict.fromkeys(unread))) + + +def _sheet(ctx: RuleContext) -> Sheet: + cached = getattr(ctx, "_readability_sheet", None) + if cached is None: + cached = ctx._readability_sheet = read_css(ctx.spec.dashboard.css) + return cached + + +def _length(value: str, release: Release, font: float) -> float | None: + """px of a length: px, rem, or em and % of `font`.""" + m = re.fullmatch(r"(-?\d*\.?\d+)(px|rem|em|%)?", value.strip().lower()) + if not m: + return None + n, unit = float(m.group(1)), m.group(2) + if unit == "px" or (unit is None and n == 0): + return n + return {"rem": n * release.root_px, "em": n * font, "%": n * font / 100}.get(unit) + + +def _line(value: str, release: Release, font: float) -> float | None: + """px of a line height: a factor, a length, or normal (Inter's 1.2).""" + v = value.strip().lower() + if v == "normal": + return 1.2 * font + if re.fullmatch(r"\d*\.?\d+", v): + return float(v) * font + return _length(v, release, font) + + +def _winner(sheet: Sheet, kind: str, element: str, prop: str, + area: str | None = None) -> Decl | None: + """The dashboard's declaration that wins for this property on this element of this + chart type: !important first, then specificity, then the later one. `area` keeps + only what reaches the header (head) or the body rows (body).""" + found = [d for d in sheet.decls if d.element == element and d.prop == prop + and (d.kinds is None or kind in d.kinds) + and (area is None or d.area in (None, area))] + return max(found, key=lambda d: (d.important, d.spec, d.order), default=None) + + +def _beats(d: Decl, kind: str, element: str, prop: str, release: Release) -> bool: + base = "padding" if prop.startswith("padding") else prop + theirs = _SUPERSET.get((kind, element, base)) + rival = theirs[1 if release.modern else 0] if theirs else None + return d.important or rival is None or d.spec >= rival + + +def _rival(kind: str, element: str, prop: str, release: Release) -> bool: + theirs = _SUPERSET.get((kind, element, prop)) + return bool(theirs and theirs[1 if release.modern else 0]) + + +@dataclass +class Size: + px: float + source: str # superset | theme | css + by: str = "" # the selector or token that set it + lost: tuple[str, ...] = () # dashboard selectors Superset's own rule beat + + +def _font(sheet: Sheet, kind: str, element: str, release: Release, tok: Tokens, + area: str | None = None) -> Size: + """A table or pivot cell's font size: its own rule, else what it inherits from its + row, else its table's (the dashboard's where it beats Superset's), else the default.""" + lost: list[str] = [] + base = tok.size(tok.sm) if release.modern else Size(DEFAULT_TEXT, "superset") + size = Size(base.px, base.source, base.by) + d = _winner(sheet, kind, "table", "font-size") + if d is not None: + if _beats(d, kind, "table", "font-size", release): + px = _length(d.value, release, tok.body if release.modern else 14.0) + if px is not None: + size = Size(px, "css", d.selector) + else: + lost.append(d.selector) + d = _winner(sheet, kind, "row", "font-size", area) + if d is not None: + px = _length(d.value, release, size.px) + if px is not None: + size = Size(px, "css", d.selector) + if _rival(kind, element, "font-size", release): + # The cell's own rule (a pivot's header cells): nothing inherited reaches it. + size = Size(base.px, base.source, base.by) + d = _winner(sheet, kind, element, "font-size", area) + if d is not None: + if _beats(d, kind, element, "font-size", release): + px = _length(d.value, release, size.px) + if px is not None: + size = Size(px, "css", d.selector) + else: + lost.append(d.selector) + size.lost = tuple(lost) + return size + + +def _cell_row(sheet: Sheet, kind: str, element: str, release: Release, tok: Tokens, + font: float, area: str | None = None) -> Size: + """How tall one cell makes its row: its line box and padding, at least any height + set on it or its row.""" + line = (tok.line if release.modern else release.table_line) if kind == "table" else PIVOT_LINE + line_px, by = line * font, "" + for scope in ("table", "row", element): + if scope == element and _rival(kind, element, "line-height", release): + line_px, by = line * font, "" # the cell's own rule, unless the CSS beats it + d = _winner(sheet, kind, scope, "line-height", area) + if d is not None and _beats(d, kind, scope, "line-height", release): + px = _line(d.value, release, font) + if px is not None: + line_px, by = px, d.selector + pad = release.table_pad if kind == "table" else PIVOT_PAD + pads = [] + for prop in ("padding-top", "padding-bottom"): + d = _winner(sheet, kind, element, prop, area) + px = (_length(d.value, release, font) if d is not None + and _beats(d, kind, element, prop, release) else None) + if px is not None: + by = by or d.selector + pads.append(pad if px is None else px) + least = 0.0 + for scope in ("row", element): + d = _winner(sheet, kind, scope, "height", area) + px = _length(d.value, release, font) if d is not None else None + if px is not None and px > least: + least, by = px, d.selector + px = max(least, line_px + sum(pads) + BORDER) + return Size(round(px, 1), "css" if by else "superset", by) + + +def _row(sheet: Sheet, kind: str, release: Release, tok: Tokens) -> Size: + """A body row's height: its tallest cell's. A pivot's row labels are header cells + (th.pvtRowLabel) in the same row as its values. A row the dashboard's own font size + shrinks is the dashboard's doing, as one its spacing shrinks is.""" + rows = [] + for element in ("td", "th") if kind == "pivot_table" else ("td",): + font = _font(sheet, kind, element, release, tok, "body") + row = _cell_row(sheet, kind, element, release, tok, font.px, "body") + if row.source == "superset" and font.source != "superset": + row = Size(row.px, font.source, font.by) + rows.append(row) + return max(rows, key=lambda s: s.px) + + +# -- readability.table-text --------------------------------------------------------- + + +def _charts_of(ctx: RuleContext, kinds) -> str: + parts = [] + for kind in GRID_KINDS: + names = [c.name for c in ctx.spec.charts if c.type == kind] + if kind in kinds and names: + parts.append(f"{KIND_WORDS[kind]} ({', '.join(names)})") + return " and ".join(parts) + + +def _per_release(rows: list[tuple[str, Release, Size]], rels) -> str: + """'12 px' when it is that size on every release the spec can go to, else each + release's size: '12 px on 6.0 and later'.""" + by_size: dict[str, list[str]] = {} + for _, r, s in rows: + names = by_size.setdefault(f"{s.px:g} px", []) + if r.name not in names: + names.append(r.name) + if len(by_size) == 1 and len(next(iter(by_size.values()))) == len(rels): + return next(iter(by_size)) + return "; ".join(f"{size} on {' and '.join(names)}" for size, names in by_size.items()) + + +def _setters(rows) -> str: + by = sorted({s.by for _, _, s in rows if s.by}) + return f" (set by `{'`, `'.join(by)}`)" if by else "" + + +def _losers(rows, rels) -> str: + lost = sorted({x for _, _, s in rows for x in s.lost}) + if not lost: + return "" + where = sorted({r.name for _, r, s in rows if s.lost}) + there = "" if len(where) == len(rels) else f" on {' and '.join(where)}" + return (f". `{'`, `'.join(lost)}` loses to Superset's own, more specific rule{there}: " + f"set the size on the cells") + + +def _unread(ctx: RuleContext, elements: set[str]) -> str: + """The rules the reader couldn't place on these elements, named.""" + rows = [(s, p, why) for s, p, why, e in _sheet(ctx).unread if e in elements] + if not rows: + return "" + shown = "; ".join(f"`{s}` {p}: {why}" for s, p, why in rows[:3]) + more = f" and {len(rows) - 3} more" if len(rows) > 3 else "" + return f". Not read: {shown}{more}" + + +@rule("readability.table-text", "info", + "table and pivot cells read at >= 14 px on a laptop, their headers at >= 12 px and " + "rows at >= 24 px (Superset draws cells at 12 px: dashboard.css " + "`.superset-chart-table td, .pivot_table_v_2 td { font-size: 14px; }`)", + since="16", severities=("info", "warn")) +def table_text(ctx: RuleContext): + kinds = [k for k in GRID_KINDS if any(c.type == k for c in ctx.spec.charts)] + if not kinds: + return + sheet, rels = _sheet(ctx), releases(ctx) + tok, _ = _theme(ctx) + p = ctx.params + cells = {(k, r.name): _font(sheet, k, "td", r, tok, "body") for k in kinds for r in rels} + heads = {(k, r.name): _font(sheet, k, "th", r, tok, "head") for k in kinds for r in rels} + rows = {(k, r.name): _row(sheet, k, r, tok) for k in kinds for r in rels} + roles = ( + ("cells", "td", cells, p.min_cell_text_px, "text read row by row"), + ("headers", "th", heads, p.min_label_text_px, "labels"), + ("rows", "row", rows, p.min_row_px, "a row"), + ) + for where, element, sizes, floor, what in roles: + low = [(k, r, sizes[(k, r.name)]) for k in kinds for r in rels + if sizes[(k, r.name)].px < floor] + if not low: + continue + own = [x for x in low if x[2].source != "superset"] + severity = "warn" if own else "info" + shown = own or low + whose = _charts_of(ctx, {k for k, _, _ in shown}) + size = _per_release(shown, rels) + if element == "row": + detail = (f"{whose} draw body rows {size} tall{_setters(shown)}, below the " + f"{floor:g} px floor for {what}: give the cells room (line-height, " + f"padding) or drop the height set on them") + else: + source = "" if own else ", Superset's default" + detail = (f"{whose} draw their {where} at {size}{source}{_setters(shown)}: below " + f"the {floor:g} px floor for {what} on a laptop{_losers(shown, rels)}. " + + _remedy(element, {k for k, _, _ in shown}, floor, rels)) + detail += _unread(ctx, {element, "row", "table"} if element != "row" else {"td", "row"}) + detail += _theme_unread(ctx) + yield Finding("readability.table-text", severity, None, f"table {where}", detail) + + +def _remedy(element: str, kinds: set[str], floor: float, rels) -> str: + sel = ", ".join(f"{KIND_CLASS[k]} {element}" for k in GRID_KINDS if k in kinds) + if element == "th" and "pivot_table" in kinds: + # A pivot's header cells carry Superset's own (0,2,4) rule: match it. + sel = ", ".join(([".superset-chart-table th"] if "table" in kinds else []) + + [".pivot_table_v_2 table.pvtTable thead tr th", + ".pivot_table_v_2 table.pvtTable tbody tr th"]) + out = f"Set it in dashboard.css: `{sel} {{ font-size: {floor:g}px; }}`" + if any(r.modern for r in rels): + out += ("; on 6.0 and later the theme token fontSizeSM sets it too, with every " + "other small text on the dashboard") + if element == "td": + out += (". Larger cells draw taller rows (at 14 px: 30.6 px on 4.1.4 and 5.0.0, " + "32.6 px on 6.1.0, against the 29.6 px the table-height rules count, and " + "long text wraps in narrow columns), so check table heights on the " + "rendered dashboard") + return out + + +# -- readability.chart-text ----------------------------------------------------------- + + + +def _components(layer: dict, key: str, path: tuple[str, ...]) -> list[float]: + value = layer.get(key) + items = value if isinstance(value, list) else [value] + out = [] + for item in items: + for step in path: + item = item.get(step) if isinstance(item, dict) else None + if _num(item) is not None: + out.append(_num(item)) + return out + + +def echarts_size(theme_json: dict | None, viz: str, role: str) -> tuple[float, str] | None: + """(px, where it's set) for an axis label or legend a theme's ECharts overrides set on + one chart type: the component's own size beats a global textStyle, and the chart + type's overrides beat the global ones (Echart.tsx:267-273).""" + if not isinstance(theme_json, dict): + return None + by = theme_json.get("echartsOptionsOverridesByChartType") + layers = [(f"echartsOptionsOverridesByChartType.{viz}", + by.get(viz) if isinstance(by, dict) else None), + ("echartsOptionsOverrides", theme_json.get("echartsOptionsOverrides"))] + layers = [(n, l) for n, l in layers if isinstance(l, dict)] + found = [] + keys = (("xAxis", ("axisLabel", "fontSize")), ("yAxis", ("axisLabel", "fontSize"))) \ + if role == "axis" else (("legend", ("textStyle", "fontSize")),) + for key, path in keys: + size = None + for name, layer in layers: + got = _components(layer, key, path) + if got: + size = (min(got), f"{name}.{key}.{'.'.join(path)}") + break + if size is None: + for name, layer in layers: + text = layer.get("textStyle") + if isinstance(text, dict) and _num(text.get("fontSize")) is not None: + size = (_num(text["fontSize"]), f"{name}.textStyle.fontSize") + break + if size is not None: + found.append(size) + return min(found) if found else None + + +@rule("readability.chart-text", "info", + "axis labels and legends read at >= 12 px and chart titles at >= 14 px (a theme's " + "tokens or ECharts overrides, or dashboard CSS, can set them smaller)", + since="16", severities=("info", "warn")) +def chart_text(ctx: RuleContext): + p = ctx.params + rels = releases(ctx) + tok, data = _theme(ctx) + overrides = data if _echarts_overrides(ctx) else None + roles = ( + ("axis labels", "axis", [c for c in ctx.spec.charts if c.type in AXIS_TYPES], + p.min_label_text_px), + # A heatmap's show_legend is its colour scale, not a legend of names. + ("legends", "legend", [c for c in ctx.spec.charts if getattr(c, "show_legend", False) + and c.type != "heatmap"], p.min_label_text_px), + ) + for where, role, charts, floor in roles: + low = [] + for c in charts: + for r in rels: + # A theme overrides by viz_type, as the compiler writes it. + got = echarts_size(overrides, VIZ_TYPE[c.type], role) if r.modern else None + size = Size(got[0], "theme", got[1]) if got else Size(DEFAULT_TEXT, "superset") + if size.px < floor: + low.append((c.name, r, size)) + if not low: + continue + own = [x for x in low if x[2].source != "superset"] + names = sorted({n for n, _, _ in (own or low)}) + if own: + by = sorted({s.by for _, _, s in own}) + detail = (f"the theme {tok.theme!r} sets {where} on {', '.join(names)} to " + f"{_per_release(own, rels)} (`{'`, `'.join(by)}`): below the " + f"{floor:g} px floor for labels on a laptop; raise it to {floor:g}") + else: + detail = (f"{', '.join(names)} draw {where} at {DEFAULT_TEXT:g} px, ECharts' " + f"default, below the {floor:g} px floor set for labels. Nothing in a " + f"spec or dashboard CSS reaches them (ECharts draws on a canvas); on " + f"6.1 and later a theme's echartsOptionsOverrides.textStyle.fontSize " + f"sets them") + yield Finding("readability.chart-text", "warn" if own else "info", None, where, + detail + _theme_unread(ctx)) + yield from _titles(ctx, rels, tok) + + +def _titles(ctx: RuleContext, rels, tok: Tokens): + floor = ctx.params.min_title_text_px + if not ctx.spec.charts: + return + low = [] + for r in rels: + size = tok.size(tok.lg) if r.modern else Size(DEFAULT_TITLE, "superset") + d = _winner(_sheet(ctx), "any", "title", "font-size") + if d is not None: + px = _length(d.value, r, size.px) + if px is not None: + size = Size(px, "css", d.selector) + if size.px < floor: + low.append(("titles", r, size)) + if not low: + return + own = [x for x in low if x[2].source != "superset"] + shown = own or low + by = sorted({s.by for _, _, s in shown if s.by}) + detail = (f"chart titles draw at {_per_release(shown, rels)}" + + (f" (set by `{'`, `'.join(by)}`)" if by else ", Superset's default") + + f": below the {floor:g} px floor for a chart's heading on a laptop. Set " + f"`.header-title {{ font-size: {floor:g}px; }}` in dashboard.css" + + ("; on 6.0 and later the theme token fontSizeLG sets it" + if any(r.modern for r in rels) else "")) + yield Finding("readability.chart-text", "warn" if own else "info", None, "chart titles", + detail + _unread(ctx, {"title"}) + _theme_unread(ctx)) + + +# -- readability.kpi-text ----------------------------------------------------------- + +# What each release drew for a big number at each whole height, the smallest of the +# three (px; sec.17): the value, and the subtitle or comparison under it. A trendline +# KPI gives its line 30% of the card, so its text is smaller. Measured with chartwright's +# pinned proportions at 1440 px, on cards wide enough that height alone set the size. +KPI_TOTAL = {2: (6, 2), 3: (18, 6), 4: (31, 11), 5: (43, 17), 6: (57, 22), 7: (70, 26), + 8: (84, 31), 9: (97, 36), 10: (109, 41)} +KPI_TREND = {2: (5, 2), 3: (12, 4), 4: (22, 8), 5: (31, 11), 6: (39, 14), 7: (50, 19), + 8: (59, 22), 9: (68, 26), 10: (76, 29)} +KPI_MAX_HEIGHT = 6 # size.kpi-height's ceiling: a fix past it would fight that rule +EM_PER_CHAR = 0.5 # a narrow guess at Inter's average width (it measured 0.50-0.56) + + +def kpi_px(kind: str, height: float) -> tuple[float, float]: + """(value, subtitle) px at a height in units, between and past the measured ones.""" + table = KPI_TREND if kind == "big_number_trend" else KPI_TOTAL + if height < 2: + return 0.0, 0.0 + lo = min(math.floor(height), 9) + a, b = table[lo], table[lo + 1] + t = height - lo + return a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t + + +def kpi_units(kind: str, lines: bool, params) -> int | None: + """The least whole height at which a big number's value, and its lines when it has + any, reach the floors, by height alone (None: past KPI_MAX_HEIGHT).""" + for units in range(2, KPI_MAX_HEIGHT + 1): + value, label = kpi_px(kind, units) + if value >= params.min_kpi_value_px and (not lines or label >= params.min_label_text_px): + return units + return None + + +def _card_text_px(ctx: RuleContext, width: int) -> float: + """The px a card's text may span: computeMaxFontSize's 90% of the card less its + padding, with the vertical filter bar open when the dashboard has native filters.""" + bar = FILTER_BAR if (ctx.spec.filters and ctx.spec.dashboard.filter_bar_orientation + != "horizontal") else 0.0 + pitch = (VIEWPORT - PAGE_MARGIN - bar + GUTTER) / 12 + return 0.9 * (pitch * width - GUTTER - 2 * CARD_PAD) + + +def _kpi_lines(ctx: RuleContext, c) -> list[tuple[str, str]]: + """(what, text) of each line under the value the card draws.""" + lines = [] + if c.type == "big_number_trend" and c.compare_lag is not None: + lines.append(("comparison", f"+00.0% {c.compare_suffix or ''}".strip())) + if getattr(c, "subtitle", None) and (c.type == "big_number_total" + or any(r.modern for r in releases(ctx))): + lines.append(("subtitle", c.subtitle)) + return lines + + +def _kpi_css(ctx: RuleContext, c, element: str) -> Decl | None: + """An !important size from the dashboard's CSS: the only kind that beats the inline + style BigNumberViz writes.""" + d = _winner(_sheet(ctx), c.type, element, "font-size") + return d if d is not None and d.important else None + + +@dataclass +class _Kpi: + chart: object + height: float + lines: list # (what, text) under the value + short: list # what draws below its floor at this height, worded + css: list # (what, px, selector): sizes the CSS pins below a floor + need: int | None = None # the whole height that brings it to the floors + capped: str = "" # why no height can: the card is too narrow + + +def _kpi(ctx: RuleContext, c) -> _Kpi: + p = ctx.params + h = ctx.height(c.name) + lines = _kpi_lines(ctx, c) + css, pinned = [], {} + for element, what in (("kpi-value", "value"), ("kpi-label", "subtitle")): + d = _kpi_css(ctx, c, element) + px = _length(d.value, LEGACY, 14.0) if d is not None else None + if px is not None: + pinned[element] = px + floor = p.min_kpi_value_px if element == "kpi-value" else p.min_label_text_px + if px < floor and (element == "kpi-value" or lines): + css.append((what, px, d.selector)) + caps = [math.floor(_card_text_px(ctx, ctx.width(c.name)) / (EM_PER_CHAR * len(t))) + for _, t in lines if t] + cap = min(caps) if caps else None + + def short(units: float) -> list[str]: + """What the card's height draws below its floor; a size the CSS pins is the + CSS finding's, whatever the height.""" + value, label = kpi_px(c.type, units) + if cap is not None: + label = min(label, cap) + out = ([f"value at {value:.0f} px"] + if "kpi-value" not in pinned and value < p.min_kpi_value_px else []) + if lines and "kpi-label" not in pinned and label < p.min_label_text_px: + out.append(f"{' and '.join(w for w, _ in lines)} at {label:.0f} px") + return out + + k = _Kpi(c, h, lines, short(h), css) + if not k.short: + return k + if (cap is not None and "kpi-label" not in pinned and cap < p.min_label_text_px): + k.capped = (f"; at {ctx.width(c.name)}/12 the card is too narrow for " + f"{p.min_label_text_px:g} px lines at any height: widen it or shorten " + f"the {' and '.join(w for w, _ in lines)}") + return k + for units in range(math.floor(h) + 1, KPI_MAX_HEIGHT + 1): + if not short(units): + k.need = units + break + return k + + +@rule("readability.kpi-text", "info", + "Superset sizes a big number's text by the card's height: the value needs >= 24 px " + "and a subtitle or comparison >= 12 px (5 units with a subtitle, 6 for a trendline " + "KPI's comparison)", fixable=True, since="16", severities=("info", "warn")) +def kpi_text(ctx: RuleContext): + p = ctx.params + kpis = {c.name: _kpi(ctx, c) for c in ctx.spec.charts if c.type in KPI_TYPES} + if not kpis: + return + # One band, one height: KPIs side by side rise together, to what the neediest needs. + goal: dict[str, int] = {} + for sec in ctx.sections: + for band in sec.bands: + names = [n for n in band.chart_names if n in kpis] + needs = [kpis[n].need for n in names if kpis[n].need] + if needs: + goal.update((n, max(needs)) for n in names) + floors = f"{p.min_kpi_value_px:g} px for the value, {p.min_label_text_px:g} px for its lines" + for name, k in kpis.items(): + where = ctx.where(name) + for what, px, selector in k.css: + floor = p.min_kpi_value_px if what == "value" else p.min_label_text_px + yield Finding("readability.kpi-text", "warn", name, where, + f"dashboard.css sets this big number's {what} to {px:g} px " + f"(`{selector}`), below the {floor:g} px floor; raise it, or drop " + f"the rule and let the card's height size it") + target = goal.get(name) + if k.short: + drawn = (f"Superset sizes a big number's text by its card: at {k.height:g} units " + f"it draws its {' and its '.join(k.short)}, below the floors ({floors})") + if k.capped: + yield Finding("readability.kpi-text", "info", name, where, drawn + k.capped) + elif k.need is None: + yield Finding("readability.kpi-text", "info", name, where, + f"{drawn}, and no height up to {KPI_MAX_HEIGHT} units (where " + f"size.kpi-height stops) reaches them") + else: + yield Finding("readability.kpi-text", "info", name, where, + f"{drawn}; {target} units reach them" + + ("" if target == k.need else + " and keep its KPI band at one height"), + fix={"chart": name, "set": {"height": target}}, + height_driven=True) + elif target is not None and target > k.height: + yield Finding("readability.kpi-text", "info", name, where, + f"its KPI band rises to {target} units so its neighbours' text " + f"reaches the floors ({floors}); one band, one height", + fix={"chart": name, "set": {"height": target}}, height_driven=True) diff --git a/chartwright/design/rules.py b/chartwright/design/rules.py index a9618a4..b58c371 100644 --- a/chartwright/design/rules.py +++ b/chartwright/design/rules.py @@ -18,17 +18,33 @@ from datetime import date from ..spec import ( + DEFAULT_HEIGHT, DEFAULT_ROW_LIMIT, DEFAULT_TIME_GRAIN, + FORMAT_RANGE_OPERATORS, + FORMAT_TEXT_HEX, SUPERSET_COLOR_SCHEMES, + WATERFALL_DEFAULT_HEX, HeaderBlock, + PIVOT_METRIC, _ColorSchemeMixin, - grid_header, + grid_fit, grid_rows_visible, grid_units_for_rows, + pivot_axes, + pivot_frame, + pivot_rows_from_counts, row_items, + table_page, + hex_to_rgb, + parse_metric, + set_value, + table_header_units, without_superset_defaults, ) +from ..visible import contrast +from ..versions import gated_fields_used, parse_version +from .markdown_fit import VIEWPORT, Fit, box_px, estimate, read_css, releases_from from .model import AXIS_TYPES, KPI_TYPES, TIMESERIES_TYPES, Finding, RuleContext, rule # -- size: minimum readable geometry ------------------------------------------ @@ -142,12 +158,64 @@ def heatmap_geometry(ctx: RuleContext): ) -def _table_page(c) -> int | None: - """Rows on one page of a paged table (page_length > 0), never more than row_limit; - None when every row is on one page.""" - if not getattr(c, "page_length", None): +_PROBE_CAP = 60 # distinct values a grid probe counts before it saturates + + +def _row_dims(c) -> list[str]: + """The dimensions a table's or pivot's body rows are keyed on: a pivot's row + attributes less the metric names (spec.pivot_axes), an aggregate table's groupby.""" + if c.type == "pivot_table": + return [a for a in pivot_axes(c)[0] if a != PIVOT_METRIC] + return list(c.groupby or []) if not c.columns else [] + + +def _counted(c, dims: list[str]) -> str: + """What a grid's row count counts, for a finding: the dimensions, and the metrics + when a pivot lays them out as rows.""" + what = " x ".join(repr(d) for d in dims) + if c.type == "pivot_table" and len(c.metrics) > 1 and PIVOT_METRIC in pivot_axes(c)[0]: + what = f"{what} x {len(c.metrics)} metrics" if what else f"{len(c.metrics)} metrics" + return what + + +def _probed_rows(ctx: RuleContext, c) -> tuple[int, int, bool] | None: + """(leaf rows, subtotal rows, exact) a table or pivot draws, from one bounded + distinct count per row dimension (advise --profile; the probes are cached). None + offline, when a probe fails, or for a chart a per-column count can't size (a raw + table, a grid with no row dimension). Exact with one dimension that didn't saturate + the probe; otherwise a lower bound. size.grid-fit reports it; size.table-window and + size.pivot-window leave a chart it sizes to that rule, so one advise run never + gives a chart two height targets.""" + dims = _row_dims(c) if c.type in ("table", "pivot_table") else [] + if ctx.prober is None or not dims or (ds := ctx.dataset_for(c)) is None: return None - return min(c.page_length, c.row_limit) if c.row_limit else c.page_length + # row_limit caps the records, so the distinct row keys too (a pivot's records are + # rows x columns, never fewer than its rows). + cap = c.row_limit or DEFAULT_ROW_LIMIT[c.type] + probe_cap = min(cap, _PROBE_CAP) + counts, saturated = {}, False + for d in dims: + n = ctx.prober.count_up_to(ds, d, probe_cap) + if n is None: + return None + saturated |= n > probe_cap and probe_cap < cap + counts[d] = min(n, cap) + if c.type == "pivot_table": + leaf, subtotals, exact = pivot_rows_from_counts(c, counts) + else: + leaf, subtotals, exact = min(max(counts.values()), cap), 0, len(dims) == 1 + return leaf, subtotals, exact and not saturated + + +def _pivot_overhead(c) -> str: + """What a pivot draws besides its body rows, in words (spec.pivot_frame).""" + header_rows, hscroll, totals = pivot_frame(c) + parts = ["the card frame", f"{header_rows} header row" + ("s" if header_rows != 1 else "")] + if totals: + parts.append("its totals row") + if hscroll: + parts.append("room for a horizontal scrollbar") + return ", ".join(parts[:-1]) + " and " + parts[-1] def _brain_page(ctx: RuleContext, c) -> bool: @@ -160,37 +228,42 @@ def _brain_page(ctx: RuleContext, c) -> bool: return ctx.brain_owns(c, "page_length") -@rule("size.table-window", "warn", "a table's height should show a meaningful share of its row_limit", - fixable=True) +@rule("size.table-window", "warn", "a table's height should show every row its row_limit allows, " + "or one whole page", fixable=True) def table_window(ctx: RuleContext): for c in ctx.spec.charts: if c.type != "table" or (c.row_limit is None and not c.page_length): continue + if _probed_rows(ctx, c) is not None: + continue # size.grid-fit sizes it by its real row count h = ctx.height(c.name) - # Same grid model as size.grid-fit, the fills and apply-time smoke (spec.py): - # offline this can only reason about row_limit (the ceiling), where grid-fit - # probes the real count, but all of them measure a table the same way. The - # header counts the search bar, the page-size bar and the pager the chart draws. - header, row = grid_header(c, c.row_limit) + # The question smoke asks after apply, of the most rows the table can render: + # row_limit (the ceiling, which a top-N or raw table reaches), or one page. Same + # grid_fit as size.grid-fit, the fills and smoke (spec.py); the header counts + # the search bar, the page-size bar, the pager and the Summary row it draws. + held, header, row = grid_fit(c, c.row_limit) visible = grid_rows_visible(h, header, row) # Whole rows, rounded down: rounding 6.67 up once told a 6-row page it showed # ~7 rows while asking for more height (the pager was what didn't fit). fits = f"table at {h:g} units fits {math.floor(visible)} full rows" - page = _table_page(c) + page = table_page(c) if page is not None: # A paged table shows one page at a time, plus its page controls: the whole # page should fit, but rows beyond it are a click away, not a scroll. want = page shown = f"{fits} beside its page controls, short of its {page}-row page" else: + # table_visible_ratio is 1 by default: the target is every row either way, + # so advise never passes a height smoke then flags when the data fills it. want = ctx.params.table_visible_ratio * c.row_limit - shown = f"{fits} of its {c.row_limit}" + shown = (f"{fits} of its {c.row_limit}; the rest hide behind an inner scrollbar " + f"once the data fills row_limit") if visible < want: brain = page is not None and _brain_page(ctx, c) - target = grid_units_for_rows(want, header, row) + target = grid_units_for_rows(held, header, row) yield Finding( "size.table-window", "warn", c.name, ctx.where(c.name), - f"{shown} (a scroll dungeon); " + f"{shown}; " + ("the page is a design default: advise --fix refits it to the panel" if brain else f"raise height to ~{math.ceil(target)} or " @@ -201,6 +274,45 @@ def table_window(ctx: RuleContext): ) +@rule("size.table-chrome", "info", + "a table whose rows all show in its panel draws no page-size picker, pager or search " + "box", since="10") +def table_chrome(ctx: RuleContext): + """DataTables chrome over rows the reader already sees. Any page_length above 0 draws + the page-size picker, on one page too (hasPagination, plugin-chart-table + DataTable/DataTable.tsx:123 at 4.1.4 and 5.0.0, :173 at 6.1.0), and the pager joins + it from a second page (:407 at 4.1.4 and 5.0.0, :616 at 6.1.0); search_box + (include_search) draws the search bar (:383 at 4.1.4 and 5.0.0, :593 at 6.1.0). The + brain's own fills never add them to such a table (default.page-length, + default.search-box), so this names the author's.""" + for c in ctx.spec.charts: + if c.type != "table" or c.row_limit is None: + continue + h = ctx.height(c.name) + # The rows the panel shows with nothing above or below them (spec.py's grid model). + shown = math.floor(grid_rows_visible(h, table_header_units())) + if c.row_limit > shown: + continue # the rows outgrow the panel: paging and search earn their room + chrome, fields = [], [] + if c.page_length and not ctx.brain_owns(c, "page_length"): + pages = math.ceil(c.row_limit / c.page_length) + chrome.append("a page-size picker" + (f" and a pager over {pages} pages" + if pages > 1 else "")) + fields.append(f"page_length {c.page_length}") + if c.search_box and not ctx.brain_owns(c, "search_box"): + chrome.append("a search box") + fields.append("search_box") + if not chrome: + continue + yield Finding( + "size.table-chrome", "info", c.name, ctx.where(c.name), + f"all {c.row_limit} rows show at height {h:g}, yet {' and '.join(fields)} " + f"draw{'s' if len(fields) == 1 else ''} {' and '.join(chrome)} over them; " + f"remove {'it' if len(fields) == 1 else 'them'} so the table reads whole " + f"(page_length 0 shows every row on one page with no picker)", + ) + + @rule("size.hbar-window", "warn", "horizontal bars need ~0.5 units of height per bar", fixable=True) def hbar_window(ctx: RuleContext): for c in ctx.spec.charts: @@ -219,72 +331,89 @@ def hbar_window(ctx: RuleContext): ) -@rule("size.pivot-window", "warn", "a pivot's height should show a meaningful share of its row_limit", - fixable=True, since="2") +@rule("size.pivot-window", "warn", "a pivot's height must hold its header rows, totals row and the " + "body rows its spec fixes (advise --profile counts the rest)", fixable=True, since="2") def pivot_window(ctx: RuleContext): + """A pivot draws a body row per distinct row key, never per record: row_limit caps + the records (rows x columns), so it says nothing about the rows on screen. Offline + the spec fixes only a floor: a row per metric laid out as rows, at least one value + per row dimension, and no body row at all without row attributes (spec.pivot_rows). + The count beyond it is size.grid-fit's, from probes, and smoke's, from the data.""" for c in ctx.spec.charts: - if c.type != "pivot_table" or c.row_limit is None: - continue + if c.type != "pivot_table" or _probed_rows(ctx, c) is not None: + continue # a probed pivot is size.grid-fit's: it knows the real row count + dims = _row_dims(c) + leaf, subtotals, _ = pivot_rows_from_counts(c, dict.fromkeys(dims, 1)) + held, header, row = grid_fit(c, leaf + subtotals) + needed = grid_units_for_rows(held, header, row) h = ctx.height(c.name) - # Header rows per column dimension, the horizontal-scrollbar allowance and the - # pinned totals row: the shared grid model (spec.py), the one smoke reads. - header, row = grid_header(c) - visible = grid_rows_visible(h, header, row) - want = ctx.params.table_visible_ratio * c.row_limit - if visible < want: - target = grid_units_for_rows(want, header, row) + if needed <= h: + continue + frame = f"{_pivot_overhead(c)} ({round(header, 2):g} units)" + if not dims: + # Exact: without a row dimension the spec fixes every row the pivot draws. yield Finding( "size.pivot-window", "warn", c.name, ctx.where(c.name), - f"pivot shows ~{math.floor(visible)} of {c.row_limit} rows at {h:g} units " - f"({header:g} header units); raise height to " - f"~{math.ceil(target)} or lower row_limit", - fix=ctx.fix_height(c, math.ceil(target)) if target <= 20 else None, + f"pivot at {h:g} units: " + (f"its {held} rows beside " if held else "") + + f"{frame} need ~{math.ceil(needed)} units; raise height to ~{math.ceil(needed)}", + fix=ctx.fix_height(c, math.ceil(needed)) if needed <= 20 else None, height_driven=True, ) + continue + # A floor only, so no fix: one that still hides rows would only move the warning + # to smoke. The count that sizes the pivot comes from --profile or the data. + first = "the first row" if held == 1 else f"the first {held} rows" + per = " x ".join(repr(d) for d in dims) + " value" + ( + " and metric" if len(c.metrics) > 1 and PIVOT_METRIC in pivot_axes(c)[0] else "") + yield Finding( + "size.pivot-window", "warn", c.name, ctx.where(c.name), + f"pivot at {h:g} units holds {math.floor(grid_rows_visible(h, header, row))} " + f"full rows beside {frame}; with a row per {per} it needs " + f"~{math.ceil(needed)} units for {first} and {row:g} for each more: " + f"advise --profile counts them", + height_driven=True, + ) @rule("size.grid-fit", "warn", "table/pivot heights must fit their data-driven row counts (they grow after authoring)", fixable=True, data_aware=True, since="2") -def grid_fit(ctx: RuleContext): +def grid_fit_rule(ctx: RuleContext): """Pre-apply half of issue #1 (the apply-time half lives in smoke): probe the actual dimension cardinality and check the configured height fits. - Honest scope: single row-dimension pivots and single-groupby aggregate - tables -- multi-dim leaf counts aren't knowable from per-column probes.""" + Exact for one row dimension (a single-groupby aggregate table, a pivot with one + row dimension, its metrics laid out as rows or not); with more, per-column probes + give only a lower bound (the largest count), reported without a fix.""" if ctx.prober is None: return for c in ctx.spec.charts: - if c.type == "pivot_table" and len(c.rows) == 1: - dim = c.rows[0] - elif c.type == "table" and (c.groupby or []) and len(c.groupby) == 1 and not c.columns: - dim = c.groupby[0] - else: - continue - ds = ctx.dataset_for(c) - if ds is None: - continue - cap = c.row_limit or 60 - n = ctx.prober.count_up_to(ds, dim, min(cap, 60)) - if n is None: + probed = _probed_rows(ctx, c) + if probed is None: continue - n = min(n, cap) - # Shared grid model (chartwright/spec.py), same numbers smoke uses: the - # header counts the controls, pager, header rows and totals the chart draws. - header, row = grid_header(c, n) - page = _table_page(c) if c.type == "table" else None - if page is not None: - n = min(n, page) # one page (its pager is in the header), not every row - needed = math.ceil(grid_units_for_rows(n, header, row)) + leaf, subtotals, exact = probed + # Shared grid model (chartwright/spec.py), the grid_fit smoke calls: the header + # counts the controls, pager, header rows and totals the chart draws, and a + # paged table holds one page (its pager is in the header), not every row. + held, header, row = grid_fit(c, leaf + subtotals) + needed = math.ceil(grid_units_for_rows(held, header, row)) h = ctx.height(c.name) if needed <= h: continue # A brain-filled page follows the height: report, but leave the fix to the fill. - brain = page is not None and _brain_page(ctx, c) + brain = c.type == "table" and table_page(c) is not None and _brain_page(ctx, c) + totals = pivot_frame(c)[2] if c.type == "pivot_table" else c.show_totals + about = "~" if exact else "at least " + paged = held != leaf + subtotals # more rows than a page: the page is exact yield Finding( "size.grid-fit", "warn", c.name, ctx.where(c.name), - f"{dim!r} yields ~{n} rendered rows needing ~{needed} units; height {h:g} " - f"hides the tail behind an inner scrollbar -- and row counts grow with the " - f"data, so this only gets worse", - fix=ctx.fix_height(c, needed) if needed <= 20 and not brain else None, + f"{_counted(c, _row_dims(c))} yields {about}{leaf} rendered rows" + + (f", {subtotals} subtotal rows" if subtotals else "") + + (" and a totals row" if totals else "") + + (f"; its {held}-row page needs ~{needed} units" if paged + else f" needing {about}{needed} units") + + f"; height {h:g} hides the tail behind an inner scrollbar -- and row counts " + f"grow with the data, so this only gets worse", + fix=ctx.fix_height(c, needed) if (exact or paged) and needed <= 20 and not brain + else None, height_driven=True, ) @@ -492,6 +621,53 @@ def section_headers(ctx: RuleContext): # -- chart: encoding choice ---------------------------------------------------- +# A sub-tab is a click: it earns one when its question needs several charts. A sub-tab +# holding one or two reads better as a section of one scrolling page under a header row. +THIN_SUBTAB_CHARTS = 2 + + +@rule("layout.thin-subtabs", "info", + f"a sub-tab holding {THIN_SUBTAB_CHARTS} charts or fewer reads better as a section of one page", + since="17") +def thin_subtabs(ctx: RuleContext): + for sec in ctx.body_sections: + if not sec.title or " > " not in sec.title: + continue + n = sum(len(b.chart_names) for b in sec.bands) + kpis = sum(1 for b in sec.bands for name in b.chart_names if ctx.charts[name].type in KPI_TYPES) + if n - kpis <= THIN_SUBTAB_CHARTS: + parent = sec.title.split(" > ")[0] + yield Finding( + "layout.thin-subtabs", "info", None, sec.label, + f"holds {n - kpis} chart(s) besides KPIs; readers click for little. Put tab " + f"{parent!r}'s sub-tabs on one scrolling page, each under a header row", + ) + + +# Every axis chart names what its axes measure: the value axis its measure and unit, the +# category or time axis its dimension. Superset draws neither unless the chart sets it, and a +# reader of a screenshot or an export can't hover for the unit. The heatmap has no title +# control (Heatmap/controlPanel.tsx at 4.1.4, 5.0.0 and 6.1.0), and a waterfall's steps name +# themselves while its y title collides with its labels (chart.waterfall-axis-titles). +@rule("chart.axis-titles", "info", "axis charts name their axes: the measure and unit, and the dimension", + since="17") +def axis_titles(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type not in AXIS_TYPES or c.type == "heatmap" or not hasattr(c, "y_axis_title"): + continue + if not c.y_axis_title: + yield Finding( + "chart.axis-titles", "info", c.name, ctx.where(c.name), + "no y_axis_title: name the measure and its unit (e.g. \"GW\", \"Share of supply\"), " + "so the axis reads without the title or a tooltip", + ) + if not c.x_axis_title: + yield Finding( + "chart.axis-titles", "info", c.name, ctx.where(c.name), + "no x_axis_title: name the dimension (e.g. \"Month\", \"Grid operator\")", + ) + + @rule("chart.vbar-categories", "warn", "vertical bars drop labels past ~8 categories; rank with horizontal bars", fixable=True) def vbar_categories(ctx: RuleContext): p = ctx.params @@ -649,6 +825,204 @@ def funnel_stages(ctx: RuleContext): ) +# -- chart: waterfalls --------------------------------------------------------------- + +# Aggregates whose values don't add: a running total of them means nothing (two +# averages don't make the average of both; distinct counts overlap). +_NON_ADDITIVE = ("AVG", "MIN", "MAX", "COUNT_DISTINCT") +WATERFALL_MAX_STEPS = 12 +# A chart card's background in Superset's light theme, and the contrast WCAG 2.1 asks of +# a graphic against what is next to it (1.4.11 Non-text Contrast). +PANEL_RGB = (255, 255, 255) +GRAPHIC_CONTRAST = 3.0 +_BAR_COLOURS = ("increase_color", "decrease_color", "total_color") + + +def _on_panel(hex_colour: str) -> float: + rgb = hex_to_rgb(hex_colour) + return contrast((rgb["r"], rgb["g"], rgb["b"]), PANEL_RGB) + + +@rule("chart.waterfall-additive", "warn", + "a waterfall's steps must add up: SUM or COUNT of one measure, never AVG, MIN, MAX or " + "COUNT_DISTINCT", since="9") +def waterfall_additive(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type != "waterfall": + continue + agg = (parse_metric(c.metric) or {}).get("aggregate") + if agg in _NON_ADDITIVE: + yield Finding( + "chart.waterfall-additive", "warn", c.name, ctx.where(c.name), + f"metric {c.metric!r}: a waterfall adds every step to a running total and " + f"draws that as the closing, and {agg} values don't add up; chart a SUM or " + f"COUNT of the change, or show the {agg} as plain bars", + ) + + +@rule("chart.waterfall-order", "info", + "a bridge reads in its own order, and Superset draws a waterfall's steps A to Z " + "unless steps orders them", since="9") +def waterfall_order(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type != "waterfall" or c.steps is not None or c.time_grain: + continue + ds = ctx.dataset_for(c) + if ds is not None and (ds.is_temporal(c.x_column) or ds.column_types.get(c.x_column) == 0): + continue # periods or numbers: their own order is the reading order + yield Finding( + "chart.waterfall-order", "info", c.name, ctx.where(c.name), + f"the steps of {c.x_column!r} draw A to Z, then Superset's total; for a bridge, " + f"list them in steps, opening first, and name the closing row in closing " + f"(Superset 6.1.0 or later)", + ) + + +@rule("chart.waterfall-colors", "info", + "a waterfall's rising, falling and total bars take colours set for the dashboard, " + "each at least 3:1 against the panel", since="9", severities=("info", "warn")) +def waterfall_colors(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type != "waterfall": + continue + low = [(f, c.colour_hex(f)) for f in _BAR_COLOURS + if set_value(c, f) is not None and _on_panel(c.colour_hex(f)) < GRAPHIC_CONTRAST] + for field, hexed in low: + yield Finding( + "chart.waterfall-colors", "warn", c.name, ctx.where(c.name), + f"{field} {hexed} is {_on_panel(hexed):.1f}:1 against the white panel, under " + f"the {GRAPHIC_CONTRAST:g}:1 WCAG asks of a graphic; darken it", + ) + stock = [f for f in _BAR_COLOURS if set_value(c, f) is None] + if stock: + faint = [f"{f} {WATERFALL_DEFAULT_HEX[f]} ({_on_panel(WATERFALL_DEFAULT_HEX[f]):.1f}:1)" + for f in stock if _on_panel(WATERFALL_DEFAULT_HEX[f]) < GRAPHIC_CONTRAST] + yield Finding( + "chart.waterfall-colors", "info", c.name, ctx.where(c.name), + f"{', '.join(stock)} {'is' if len(stock) == 1 else 'are'} Superset's stock " + + (f"colour; {', '.join(faint)} falls under the {GRAPHIC_CONTRAST:g}:1 a graphic " + f"needs against the white panel. " if faint else "colour. ") + + "Set all three: green and red (the text shades, " + f"{_on_panel('#1B7F3B'):.1f}:1 and {_on_panel('#B3261E'):.1f}:1) or your " + "palette's, and a neutral total that reads as a total", + ) + + +@rule("chart.waterfall-axis-titles", "info", + "a waterfall's axis titles sit where Superset puts them, over wide tick labels", + since="9") +def waterfall_axis_titles(ctx: RuleContext): + # Waterfall/transformProps.ts, all three releases (6.1.0 :445-464): both titles sit + # mid-axis with no nameGap (ECharts' 15 px) and a 16 or 20 px pad, and the panel has + # no margin control for either. Seen on 6.1.0: '$3.00M' under the y title. + for c in ctx.spec.charts: + if c.type != "waterfall": + continue + if c.y_axis_title: + yield Finding( + "chart.waterfall-axis-titles", "info", c.name, ctx.where(c.name), + f"y_axis_title {c.y_axis_title!r}: Superset draws a waterfall's y-axis title " + f"about 40 px left of the axis, inside the column of tick labels, with no " + f"margin to set, so a label wider than about five characters ('$4.72M') can " + f"run through it; name the unit in the chart's title or its number_format", + ) + if c.x_axis_title and c.x_label_rotation in (45, 90): + yield Finding( + "chart.waterfall-axis-titles", "info", c.name, ctx.where(c.name), + f"x_axis_title {c.x_axis_title!r} with labels rotated {c.x_label_rotation} " + f"degrees: the rotated labels hang over the title, which Superset keeps about " + f"30 px under the axis; drop the title or the rotation", + ) + + +@rule("chart.waterfall-steps", "warn", + f"a waterfall past ~{WATERFALL_MAX_STEPS} steps stops reading as a bridge", since="9") +def waterfall_steps(ctx: RuleContext): + # Offline it counts a bridge's own steps; with a live resolution (advise --profile) it + # probes the values of an x column that steps doesn't order, as vbar-categories does. + for c in ctx.spec.charts: + if c.type != "waterfall" or c.groupby: + continue + if c.steps is not None: + n = len(c.steps) + if n > WATERFALL_MAX_STEPS: + yield Finding( + "chart.waterfall-steps", "warn", c.name, ctx.where(c.name), + f"{n} steps between the opening and the closing; past ~" + f"{WATERFALL_MAX_STEPS} the bars thin and their labels drop: fold the " + f"small ones into one 'Other' row", + ) + continue + if ctx.prober is None or c.time_grain: + continue + if c.row_limit is not None and c.row_limit <= WATERFALL_MAX_STEPS: + continue + ds = ctx.dataset_for(c) + if ds is None or ds.is_temporal(c.x_column): + continue + if ctx.prober.more_than(ds, c.x_column, WATERFALL_MAX_STEPS): + yield Finding( + "chart.waterfall-steps", "warn", c.name, ctx.where(c.name), + f"{c.x_column!r} has more than {WATERFALL_MAX_STEPS} values, one bar each; past " + f"~{WATERFALL_MAX_STEPS} the bars thin and their labels drop: fold the small " + f"ones into one 'Other' row or filter", + ) + + +# -- chart: box plots ------------------------------------------------------------------ + +BOX_PLOT_MAX_GROUPS = 20 +# Time grains from finest to coarsest, and the period a group column's name says it is. +# A day column is left out: day of week and day of month read alike by name. +_GRAIN_RANK = {"PT1S": 0, "PT1M": 0, "PT1H": 0, "P1D": 1, "P1W": 2, "P1M": 3, "P3M": 4, "P1Y": 5} +_PERIOD_COLUMN = re.compile(r"(?:^|_)(week|month|quarter|qtr|year)(?:_|$|id$|name$)", re.I) +_PERIOD_RANK = {"week": 2, "month": 3, "quarter": 4, "qtr": 4, "year": 5} + + +@rule("chart.box-plot-observations", "warn", + "a box needs many observations: distribute across a finer grain than the groups", + since="9") +def box_plot_observations(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type != "box_plot" or not c.time_grain: + continue + grain = _GRAIN_RANK.get("P1W" if "P1W" in c.time_grain else c.time_grain) + for col in c.groupby: + m = _PERIOD_COLUMN.search(col) + if grain is None or not m or grain < _PERIOD_RANK[m.group(1).lower()]: + continue + yield Finding( + "chart.box-plot-observations", "warn", c.name, ctx.where(c.name), + f"observations at {c.time_grain} grouped by {col!r}: each box holds one " + f"observation per {m.group(1).lower()}, too few for quartiles; distribute " + f"across a finer grain, e.g. P1D", + ) + break + + +@rule("chart.box-plot-groups", "warn", + f"past ~{BOX_PLOT_MAX_GROUPS} boxes each thins to a sliver and the labels drop", + since="9") +def box_plot_groups(ctx: RuleContext): + # Offline the groups are unknown; with a live resolution it probes one group column, + # as vbar-categories does. + if ctx.prober is None: + return + for c in ctx.spec.charts: + if c.type != "box_plot" or len(c.groupby) != 1: + continue + ds = ctx.dataset_for(c) + if ds is None: + continue + if ctx.prober.more_than(ds, c.groupby[0], BOX_PLOT_MAX_GROUPS): + yield Finding( + "chart.box-plot-groups", "warn", c.name, ctx.where(c.name), + f"{c.groupby[0]!r} has more than {BOX_PLOT_MAX_GROUPS} values, a box each; " + f"group coarser, filter to the groups that matter, or chart the spread per " + f"group as a table", + ) + + @rule("chart.heatmap-grid", "warn", "a heatmap past ~400 cells is unreadable at any size", data_aware=True) def heatmap_grid(ctx: RuleContext): if ctx.prober is None: @@ -717,39 +1091,100 @@ def pivot_columns(ctx: RuleContext): ) +def _rule_ranges(r) -> list[tuple[float, bool, float, bool]]: + """The values a colour rule colours, as (low, low included, high, high included) + intervals: "=" is one point, "!=" everything either side of it.""" + inf = float("inf") + if r.operator in FORMAT_RANGE_OPERATORS: + closed = r.operator == "between_inclusive" + return [(r.target_left, closed, r.target_right, closed)] + t = r.target + return {"<": [(-inf, False, t, False)], "<=": [(-inf, False, t, True)], + ">": [(t, False, inf, False)], ">=": [(t, True, inf, False)], + "=": [(t, True, t, True)], + "!=": [(-inf, False, t, False), (t, False, inf, False)]}[r.operator] + + +def _overlap(a, b) -> bool: + """Whether two such intervals share a value; a shared bound counts only when both + take it in (>= 80 and <= 80 share 80, > 80 and <= 80 share nothing).""" + (a0, ai, a1, aj), (b0, bi, b1, bj) = a, b + lo, lo_in = (a0, ai) if a0 > b0 else (b0, bi) if b0 > a0 else (a0, ai and bi) + hi, hi_in = (a1, aj) if a1 < b1 else (b1, bj) if b1 < a1 else (a1, aj and bj) + return lo < hi or (lo == hi and lo_in and hi_in) + + @rule("chart.format-bands", "warn", "conditional-formatting bands must tell one coherent story per metric", since="2", severities=("warn", "info")) def format_bands(ctx: RuleContext): for c in ctx.spec.charts: - if c.type != "pivot_table" or not c.conditional_formatting: + if c.type not in ("pivot_table", "big_number_total") or not c.conditional_formatting: continue + kpi = c.type == "big_number_total" by_metric: dict[str, list] = {} for r in c.conditional_formatting: - lo, hi = ((r.target_left, r.target_right) if r.operator == "between" - else (float("-inf"), r.target) if r.operator == "<" - else (r.target, float("inf"))) # compared by the shade painted, so "green" and its own hex agree - by_metric.setdefault(r.metric, []).append((lo, hi, r.color, r.paint_hex())) + by_metric.setdefault(c.metric if kpi else r.metric, []).append( + (_rule_ranges(r), r.color, r.paint_hex())) for metric, bands in by_metric.items(): for i in range(len(bands)): for j in range(i + 1, len(bands)): - (a0, a1, ca, ha), (b0, b1, cb, hb) = bands[i], bands[j] - if a0 < b1 and b0 < a1 and ha != hb: + (ra, ca, ha), (rb, cb, hb) = bands[i], bands[j] + if ha != hb and any(_overlap(a, b) for a in ra for b in rb): + # Superset paints a big number with the last rule that matches. + detail = (f"{ca} and {cb} bands overlap (the number takes the later " + f"rule's colour where they meet, not one the value earns)" + if kpi else + f"metric {metric!r}: {ca} and {cb} bands overlap " + f"(cell color depends on rule order, not the value)") yield Finding( "chart.format-bands", "warn", c.name, ctx.where(c.name), - f"metric {metric!r}: {ca} and {cb} bands overlap " - f"(cell color depends on rule order, not the value); " - f"make the ranges disjoint", + f"{detail}; make the ranges disjoint", ) - if len(bands) == 1: + # A KPI coloured only where it needs attention (red outside a tolerance) is + # exception highlighting, not decoration: a lone band is a signal there. + if len(bands) == 1 and not kpi: yield Finding( "chart.format-bands", "info", c.name, ctx.where(c.name), - f"metric {metric!r} has a single {bands[0][2]} band: one color is " + f"metric {metric!r} has a single {bands[0][1]} band: one color is " f"decoration, not a signal; band the full green/amber/red story " f"or drop it", ) +# WCAG 2 AA minimums: large text and graphics (a big number, a trendline: 1.4.3 and +# 1.4.11) need 3:1, body text (a table or pivot cell's text) 4.5:1. Measured against +# the white chart card of Superset's default theme (PANEL_RGB, as the waterfall's are). +_LARGE_CONTRAST, _TEXT_CONTRAST = GRAPHIC_CONTRAST, 4.5 + + +@rule("chart.color-contrast", "warn", + "a colour painted as text or a line reads on the white card: 3:1 for a big number or " + "a trendline, 4.5:1 for table and pivot text (WCAG AA)", since="14") +def color_contrast(ctx: RuleContext): + for c in ctx.spec.charts: + painted: list[tuple[str, str, float]] = [] # (what, hex painted, minimum ratio) + if c.type == "big_number_total": + painted = [(f"its {r.color} rule", r.paint_hex(), _LARGE_CONTRAST) + for r in c.conditional_formatting] + elif c.type == "big_number_trend" and set_value(c, "trend_color"): + painted = [("trend_color", FORMAT_TEXT_HEX.get(c.trend_color, c.trend_color), + _LARGE_CONTRAST)] + elif c.type in ("table", "pivot_table"): + painted = [(f"its {r.color} text rule on {r.metric!r}", r.paint_hex(), _TEXT_CONTRAST) + for r in c.conditional_formatting if r.paint == "text"] + seen: set[str] = set() + for what, hexed, need in painted: + ratio = _on_panel(hexed) + if ratio < need and hexed not in seen: + seen.add(hexed) + yield Finding( + "chart.color-contrast", "warn", c.name, ctx.where(c.name), + f"{what} paints {hexed}, {ratio:.1f}:1 on the white card, below {need:g}:1: " + f"too pale to read; use a darker shade (green, amber and red pass)", + ) + + _ORDINAL_RE = re.compile(r"(^|_)(day|weekday|dow|month|quarter|hour)(_|$|name)", re.I) @@ -759,13 +1194,27 @@ def dims(c): if c.type == "bar": return [c.x_column] if c.type == "heatmap": - return [c.x_column, c.y_column] + # An axis ordered by value is ordered on purpose, not by label. + return [col for col, order in ((c.x_column, c.x_order), (c.y_column, c.y_order)) + if order not in ("value_asc", "value_desc")] if c.type == "pivot_table": return [*c.rows, *c.columns] + if c.type == "box_plot": + return list(c.groupby) # the boxplot step groups and sorts by them return [] + def own_order(c, column) -> bool: + # A column the dataset reports numeric or temporal sorts in its own order + # (1..12); only names sort alphabetically. Known with a live resolution only. + ds = ctx.dataset_for(c) + return ds is not None and ds.column_types.get(column) in (0, 2) + for c in ctx.spec.charts: hits = [d for d in dims(c) if d and _ORDINAL_RE.search(d)] + if c.type == "bar" and c.sort_by: + continue # ranked by a measure the author chose: the order is deliberate + if c.type != "bar": + hits = [d for d in hits if not own_order(c, d)] if hits and c.type == "bar" and not c.category_sort: # A bar sorts by its first metric unless category_sort is set, so an # order-encoded label alone changes nothing. @@ -863,6 +1312,21 @@ def top_n_sort(ctx: RuleContext): "the query by the series ranking, so the axis shows gaps; drop row_limit, " "or drop series_limit so the query is ordered by the category", ) + elif (c.type == "bar" and c.sort_by == "total" and c.row_limit is not None + and c.row_limit < DEFAULT_ROW_LIMIT["bar"]): + # "total" is the plugin's sort of the rows it gets (SortSeriesType.Sum); the + # query is still ordered by the first metric (normalizeOrderBy.ts), so the + # row limit cuts by that metric before the bars are ranked by their sum. + remedy = ("write the total as a metric (e.g. \"SQL(SUM(a) + SUM(b)) AS Total\") " + "and sort_by it, which orders the query too" if not c.groupby else + "with a groupby the limit also counts each bar's segments, not bars: " + "drop row_limit, or filter the categories") + yield Finding( + "data.top-n-sort", "warn", c.name, ctx.where(c.name), + f"row_limit {c.row_limit} with sort_by \"total\" keeps the rows with the " + f"largest {c.metrics[0]} and then ranks them by their total, so a bar with " + f"a large total but a small {c.metrics[0]} can be left out; {remedy}", + ) _RANGE_DAYS = {"day": 1, "week": 7, "month": 30.4, "quarter": 91, "year": 365} @@ -937,6 +1401,52 @@ def grain_vs_range(ctx: RuleContext): ) +def _loaded_range(ctx: RuleContext, c) -> tuple[str | None, str]: + """The time range a chart loads with, and where it comes from: a defaulted dashboard + time_range filter in its scope replaces the chart's own (the filter's extra form data + overrides time_range), else the chart's time_range.""" + for f in ctx.spec.filters: + if f.type == "time_range" and f.default and (f.charts is None or c.name in f.charts): + return f.default, f"the time_range filter {f.name!r} loads it with" + return c.time_range, "its time_range" + + +@rule("data.rolling-window-span", "warn", + "a rolling trendline KPI's time range must hold its window and its comparison", since="9") +def rolling_window_span(ctx: RuleContext): + """A trailing-12-month total over "Last year" draws one point: the backend keeps only + windows that hold rolling_min_periods steps, and compare_lag needs that many points + more. With too few, the trendline is a dot and no change shows, silently.""" + for c in ctx.spec.charts: + if c.type != "big_number_trend" or c.rolling_type in (None, "cumsum"): + continue + time_range, source = _loaded_range(ctx, c) + span = _span_days(time_range) if time_range else None + eff = c.time_grain or DEFAULT_TIME_GRAIN + grain = _grain_days(eff) + if span is None or grain is None: + continue + least = max(1, c.rolling_periods if c.rolling_min_periods is None else c.rolling_min_periods) + lag = c.compare_lag or 0 + # A range of N grain steps touches about N + 1 buckets. The backend keeps the + # windows from bucket `least` on, and the comparison needs `lag` points before the + # latest; without one, a line still needs two points. + buckets = round(span / grain) + 1 + points = max(0, buckets - least + 1) + wanted = max(lag + 1, 2) + if points >= wanted: + continue + needs = least + wanted - 1 + yield Finding( + "data.rolling-window-span", "warn", c.name, ctx.where(c.name), + f"{source} {time_range!r}: about {buckets} {eff} buckets, but a {least}-step " + f"window" + (f" with compare_lag {lag}" if lag else "") + f" needs {needs}: the " + f"trendline draws {points} point(s)" + (" and no change shows" if lag else "") + + "; widen the range" + + (" or leave this chart out of that filter's `charts`" if "filter" in source else ""), + ) + + _FINE_GRAINS = (None, "PT1S", "PT1M", "PT1H", "P1D") @@ -1041,17 +1551,75 @@ def title_style(ctx: RuleContext): def big_number_format(ctx: RuleContext): from .defaults import FILLS # function-level: defaults imports this module - counts = FILLS["default.count-format"] + counts, dates = FILLS["default.count-format"], FILLS["default.date-tile"] for c in ctx.spec.charts: - if c.type in KPI_TYPES and c.number_format is None: + if c.type in KPI_TYPES and c.number_format is None and not getattr(c, "date_format", None): if counts.decide(ctx, c)[0] is not None: continue # default.count-format offers this remedy with a fix; one finding + if c.type == "big_number_total" and dates.decide(ctx, c)[0] is not None: + continue # a date: default.date-tile formats it as one yield Finding( "narrative.big-number-format", "info", c.name, ctx.where(c.name), "no number_format: raw float precision on a hero number; ',.0f' or '.3s' read better", ) +# A number as a text writes it: digits with thousands commas, and a %, k, M or B after. +_STATED_NUMBER = re.compile(r"(\d[\d,]*(?:\.\d+)?|\.\d+)\s*(%|[kKmMbB](?![A-Za-z]))?") +_SCALE = {"k": 1e3, "m": 1e6, "b": 1e9} + + +def _numbers_stated(text: str) -> set[float]: + """Every number a text states, as written and as what it stands for: '2%' is 2 and + 0.02, '1.2M' is 1.2 and 1,200,000. Signs are dropped: '+/-2%' states -2% and 2%.""" + out: set[float] = set() + for digits, unit in _STATED_NUMBER.findall(text): + try: + n = float(digits.replace(",", "")) + except ValueError: + continue + out.add(n) + if unit: + out.add(n / 100 if unit == "%" else n * _SCALE[unit.lower()]) + return out + + +def _threshold(v: float) -> str: + return f"{v:,.12g}" # 10,000,000 and 0.02, never 1e+07 + + +def _band_text(r) -> str: + if r.operator in FORMAT_RANGE_OPERATORS: + how = "between" if r.operator == "between" else "from" + join = "and" if r.operator == "between" else "to" + return f"{r.color} {how} {_threshold(r.target_left)} {join} {_threshold(r.target_right)}" + word = {"<": "below", ">": "above", "=": "at", "<=": "at or below", ">=": "at or above", + "!=": "except at"}[r.operator] + return f"{r.color} {word} {_threshold(r.target)}" + + +@rule("narrative.kpi-thresholds", "info", + "a KPI coloured by status states its thresholds in its subtitle or description " + "(colour alone can't say what red means)", since="14") +def kpi_thresholds(ctx: RuleContext): + for c in ctx.spec.charts: + if c.type != "big_number_total" or not c.conditional_formatting: + continue + stated = _numbers_stated(" ".join(t for t in (c.subtitle, c.description) if t)) + thresholds = {abs(t) for r in c.conditional_formatting + for t in (r.target, r.target_left, r.target_right) if t is not None} + missing = sorted(t for t in thresholds + if not any(math.isclose(t, n, rel_tol=1e-9, abs_tol=1e-12) for n in stated)) + if missing: + yield Finding( + "narrative.kpi-thresholds", "info", c.name, ctx.where(c.name), + f"colours its number ({'; '.join(_band_text(r) for r in c.conditional_formatting)}) " + f"but neither its subtitle nor its description states " + f"{', '.join(_threshold(t) for t in missing)}; say there what each colour means, " + f"so viewers can read it without guessing", + ) + + @rule("narrative.filtered-title", "info", "a filtered chart's title should say what it shows") def filtered_title(ctx: RuleContext): for c in ctx.spec.charts: @@ -1191,11 +1759,12 @@ def color_scheme(ctx: RuleContext): ) -@rule("layout.markdown-height", "info", "a one-line markdown header doesn't need a chart-sized block", fixable=True, since="2") -def markdown_height(ctx: RuleContext): - # Operates on the raw layout indices so the fix can address the block - # (markdown has no name to key on). A container address is None (layout - # rows), a tab index, [tab, sub-tab], "header" or "footer"; fix.py reads the same. +def _markdown_blocks(ctx: RuleContext): + """Every markdown block in the layout's rows, as (container, label, row, item, + block). Raw layout indices, so a fix can address the block (markdown has no name + to key on): the container is None (layout rows), a tab index, [tab, sub-tab], + "header" or "footer", and the row index counts headers and dividers. fix.py reads + the same.""" lay = ctx.spec.layout sources = [] if lay.header: @@ -1211,20 +1780,117 @@ def markdown_height(ctx: RuleContext): sources.append(([ti, si], f"tab {title!r}", sub.rows)) if lay.footer: sources.append(("footer", "footer", lay.footer)) - owned = ctx.standard_rows() for addr, label, rows in sources: for ri, row in enumerate(rows): - if addr in ("header", "footer") and (addr, ri) in owned: - continue # a standard's row: the standard is its one owner, not a repair - # ri indexes the raw rows (headers and dividers included): the fix reads the same list. for ii, item in enumerate(row_items(row) or []): - if isinstance(item, str): - continue - lines = [l for l in item.markdown.splitlines() if l.strip()] - h = item.height or 4 - if len(lines) <= 1 and h >= 3: - yield Finding( - "layout.markdown-height", "info", None, f"{label} row {ri}", - f"one-line markdown block at {h} units; 2 is plenty for a header", - fix={"md": [addr, ri, ii], "set": {"height": 2}}, - ) + if not isinstance(item, str): + yield addr, label, ri, ii, item + + +def _markdown_estimate(ctx: RuleContext, item) -> Fit: + """How the block's text fits at its width under the dashboard's CSS, on the + releases the spec can go to: a field only newer releases take (a theme, say) + leaves out the older ones. The filter bar narrows the grid when it opens by + default: on the left, beside a dashboard with native filters. A horizontal bar + sits above the grid, but 4.1.4 and 5.0.0 draw it on the left without the + HORIZONTAL_FILTER_BAR flag, so on them that case is bounded both ways.""" + d = ctx.spec.dashboard + floor = max((parse_version(g.since) for g, _ in gated_fields_used(ctx.spec) + if g.severity == "error"), default=None) + releases = releases_from(floor) + if not ctx.spec.filters: + bar = False + elif d.filter_bar_orientation != "horizontal": + bar = True + else: + bar = None if any(r.flag_for_horizontal_bar for r in releases) else False + return estimate(item.markdown, ctx.spec.resolved_item_width(item), bar, + css=read_css(d.css), releases=releases) + + +@rule("layout.markdown-height", "info", "a one-line markdown header doesn't need a chart-sized block", fixable=True, since="2") +def markdown_height(ctx: RuleContext): + owned = ctx.standard_rows() + for addr, label, ri, ii, item in _markdown_blocks(ctx): + if addr in ("header", "footer") and (addr, ri) in owned: + continue # a standard's row: the standard is its one owner, not a repair + lines = [l for l in item.markdown.splitlines() if l.strip()] + h = item.height or DEFAULT_HEIGHT["markdown"] + if len(lines) > 1 or h < 3: + continue + # The height the line takes, padding and margins included (size.markdown-fit's + # estimate): a level-1 heading needs 2.4 units, a plain line 1.6. + fit = _markdown_estimate(ctx, item) + if fit.media or h <= fit.units: + continue + yield Finding( + "layout.markdown-height", "info", None, f"{label} row {ri}", + f"one-line markdown block at {h:g} units; {fit.units:g} fits it", + fix={"md": [addr, ri, ii], "set": {"height": fit.units}}, + ) + + +@rule("size.markdown-fit", "warn", + "a markdown block must be tall enough for its text: Superset cuts off the rest, " + "with no scrollbar on macOS", fixable=True, since="11", severities=("warn", "info")) +def markdown_fit(ctx: RuleContext): + """The text's height, estimated from its markdown, its width and the dashboard's + CSS (markdown_fit.py) in a 1440 px wide window, against the block's. Warns only + when the lower bound of the estimate already cuts letters off (8 px, one grid row, + past the bottom edge): text Superset hides with no sign in a screenshot. A block + whose last line merely touches the edge, or whose padding doesn't fit, gets an + info: it scrolls a few px, and Windows draws a scrollbar in it. That is how a + one-line strip under 1.6 units, or a heading strip under 2, falls short. + + The fix raises the height to the upper bound, which fits on every release. None + when the block holds an image (its height is unknown), when the text needs more + than the 100-unit maximum, or in a header or footer row a standard owns (its + height is the standard's; only a cut-off text is reported there). The finding + names CSS that could size markdown in a way the estimate doesn't read, and a + theme, which can change Superset's own text sizes and spacing.""" + d = ctx.spec.dashboard + unread = list(dict.fromkeys(sel for sel, _ in read_css(d.css).unread)) + caveat = "" + if unread: + more = f" and {len(unread) - 1} more" if len(unread) > 1 else "" + caveat = (f"; the dashboard's CSS also styles markdown in a way this estimate " + f"doesn't read ({unread[0]}{more}), so the height may be off") + if d.theme: + caveat += (f"; this assumes Superset's own text sizes and spacing, which the theme " + f"{d.theme!r} may change") + owned = ctx.standard_rows() + for addr, label, ri, ii, item in _markdown_blocks(ctx): + h = item.height or DEFAULT_HEIGHT["markdown"] + box = box_px(h) + fit = _markdown_estimate(ctx, item) + if not fit.short(box): + continue + cut = fit.text_low - box >= 8 + standard = addr in ("header", "footer") and (addr, ri) in owned + if standard and not cut: + continue + w = ctx.spec.resolved_item_width(item) + need = (f"at least ~{fit.units:g} units with its image" if fit.media + else f"~{fit.units:g} units") + if cut: + hidden = max(1, round((fit.text_low - box) / 22)) + lead = (f"markdown text runs past its block: about {hidden} line" + f"{'s' if hidden > 1 else ''} cut off at the bottom edge, hidden behind " + f"an inner scrollbar macOS doesn't draw") + else: + lead = (f"markdown block {fit.need_low - box:.0f} px short: its last line or its " + f"padding reaches the bottom edge, so the block scrolls, and Windows draws " + f"a scrollbar in it") + if standard: + tail = "; a standard owns this row, so its height is the standard's to change" + elif fit.units > 100: + tail = "; past the 100-unit maximum, so split the text or give it a tab of its own" + else: + tail = f"; raise the height{' or widen it' if w < 12 else ''}" + yield Finding( + "size.markdown-fit", "warn" if cut else "info", None, f"{label} row {ri}", + f"{lead}; it needs {need} at {w}/12 in a {VIEWPORT} px wide window, has {h:g}" + f"{tail}{caveat}", + fix=({"md": [addr, ri, ii], "set": {"height": fit.units}} + if not (standard or fit.media or fit.units > 100) else None), + ) diff --git a/chartwright/resolver.py b/chartwright/resolver.py index 05856f2..48d5453 100644 --- a/chartwright/resolver.py +++ b/chartwright/resolver.py @@ -43,6 +43,7 @@ class ResolutionError: # | superset_version_too_old | superset_version_unknown (chartwright.versions) # | owner_not_found | owner_ambiguous | owner_account_unknown (chartwright.owners) # | theme_not_found | theme_ambiguous | theme_lookup_failed (dashboard.theme) + # | waterfall_opening_axis (a bridge's opening, on a theme without the axis fix) chart: str | None ref: str detail: str @@ -77,6 +78,9 @@ class Resolution: # the spec names no theme. theme_id: int | None = None theme_uuid: str | None = None + # Its json_data, parsed: the tokens and ECharts overrides the readability rules read + # (design/readability.py). None when the spec names no theme or the JSON won't parse. + theme_json: dict | None = None @property def ok(self) -> bool: @@ -131,6 +135,8 @@ def dataset_for(ref: DatasetRef) -> ResolvedDataset | None: _check_column(f.default, where, ds, res, "time_column default") if spec.dashboard.theme is not None and not any(e.ref == "theme" for e in res.errors): _resolve_theme(spec.dashboard.theme, client, res) + if res.theme_id is not None: + _check_opening_axis(spec, client, res) if spec.dashboard.owners is not None: from .owners import resolve_owners @@ -175,6 +181,7 @@ def _resolve_theme(name: str, client: SupersetClient, res: Resolution) -> None: found = [t for t in themes if t.get("theme_name") == name] if len(found) == 1: res.theme_id, res.theme_uuid = found[0]["id"], str(found[0].get("uuid")) + res.theme_json = _theme_json(found[0].get("json_data")) return names = sorted({str(t.get("theme_name")) for t in themes if t.get("theme_name")}) if found: @@ -192,6 +199,68 @@ def _resolve_theme(name: str, client: SupersetClient, res: Resolution) -> None: f"{', '.join(names) if names else 'none'}", near)) +def keeps_zero_on_waterfalls(theme_json) -> bool: + """A theme's JSON sets the waterfall's value axis to keep zero (ECharts yAxis.scale + false), for waterfalls or every chart. Superset 6.x merges these over each chart's + own options (plugin-chart-echarts components/Echart.tsx, mergeEchartsThemeOverrides + at 6.1.0, from SupersetTheme echartsOptionsOverrides and ...ByChartType).""" + import json + + try: + config = json.loads(theme_json) if isinstance(theme_json, str) else (theme_json or {}) + except ValueError: + return False + for overrides in ((config.get("echartsOptionsOverridesByChartType") or {}).get("waterfall"), + config.get("echartsOptionsOverrides")): + y_axis = (overrides or {}).get("yAxis") if isinstance(overrides, dict) else None + if isinstance(y_axis, dict) and y_axis.get("scale") is False: + return True + return False + + +def _check_opening_axis(spec: DashboardSpec, client: SupersetClient, res: Resolution) -> None: + """A bridge drawn with an opening total needs a theme that keeps zero on the value + axis: the plugin's own axis floats up to the smallest total (defaultYAxis scale: true) + and cuts the opening away (seen on 6.1.0).""" + opened = [c.name for c in spec.charts if getattr(c, "opening", None)] + if not opened: + return + from .client import SupersetAPIError + + try: + theme_json = client.get(f"/api/v1/theme/{res.theme_id}")["result"].get("json_data") + except SupersetAPIError as e: + theme_json, failed = None, str(e) + else: + failed = None + if keeps_zero_on_waterfalls(theme_json): + return + why = (f"its JSON could not be read ({failed})" if failed else + "its JSON doesn't keep zero on a waterfall's value axis") + for name in opened: + res.errors.append(ResolutionError( + "waterfall_opening_axis", name, "opening", + f"opening draws the bridge's first bar as a total, which needs zero on the value " + f"axis, and dashboard.theme {spec.dashboard.theme!r} can't give it: {why}. Add " + f"{{\"echartsOptionsOverridesByChartType\": {{\"waterfall\": {{\"yAxis\": " + f"{{\"scale\": false}}}}}}}} to the theme's JSON in Superset (Settings > Themes), " + f"or drop opening and the first step rises from zero")) + + +def _theme_json(raw) -> dict | None: + """A theme's json_data: a JSON string on the REST API (ThemeRestApi list_columns, + themes/api.py:110-120 at 6.0.0 and 6.1.0), parsed; None when it isn't an object.""" + import json + + if isinstance(raw, dict): + return raw + try: + data = json.loads(raw) if isinstance(raw, str) else None + except ValueError: + return None + return data if isinstance(data, dict) else None + + def _check_where(filters, where: str, ds: "ResolvedDataset", res: "Resolution", what: str) -> None: for f in filters: if f.sql is not None: @@ -317,6 +386,8 @@ def _check_chart_fields(chart, ds: ResolvedDataset, res: Resolution) -> None: elif t == "bar": for m in chart.metrics: _check_metric(m, chart.name, ds, res) + if chart.sort_metric() and chart.sort_metric() not in chart.metrics: + _check_metric(chart.sort_metric(), chart.name, ds, res) _check_column(chart.x_column, chart.name, ds, res, "x_column") if chart.groupby: _check_column(chart.groupby, chart.name, ds, res, "groupby") @@ -369,3 +440,15 @@ def _check_chart_fields(chart, ds: ResolvedDataset, res: Resolution) -> None: _check_metric(chart.metric, chart.name, ds, res) for g in chart.groupby: _check_column(g, chart.name, ds, res, "groupby") + elif t == "waterfall": + _check_metric(chart.metric, chart.name, ds, res) + _check_column(chart.x_column, chart.name, ds, res, "x_column") + if chart.groupby: + _check_column(chart.groupby, chart.name, ds, res, "groupby") + elif t == "box_plot": + for m in chart.metrics: + _check_metric(m, chart.name, ds, res) + for c in chart.distribute_across: + _check_column(c, chart.name, ds, res, "distribute_across") + for g in chart.groupby: + _check_column(g, chart.name, ds, res, "groupby") diff --git a/chartwright/sketch.py b/chartwright/sketch.py index 87779f0..1b8d95d 100644 --- a/chartwright/sketch.py +++ b/chartwright/sketch.py @@ -14,6 +14,13 @@ - Each sketch line adds `line_units` spec height units (1 unit = 40 px); repeat a line to make its regions taller. - Every symbol's cells must form a solid rectangle. +- A legend symbol names a chart, or stands for a markdown or header block + (BlockRef here; the block itself stays in the spec's legend). Blocks are drawn + like charts, with one rule of their own: a header is ONE line (Superset sizes it + to its text). Superset nests a header in a GRID, TAB or COLUMN, never in a ROW + (dashboard/util/isValidChild.ts :75 and :97 at 4.1.4, 5.0.0 and 6.1.0), so a + header across the whole sketch is a band of its own (SketchRow.header_band) and + any other header sits in a column, stacked or alone. """ from __future__ import annotations @@ -22,6 +29,15 @@ from functools import lru_cache +@dataclass(frozen=True) +class BlockRef: + """A legend entry that is a block, not a chart: its kind, "markdown" or "header". + Hashable, so the parse stays memoized; the block itself is read from the legend + by its symbol.""" + + kind: str + + @dataclass class SketchChart: name: str @@ -29,15 +45,38 @@ class SketchChart: height: int # spec height units +@dataclass +class SketchBlock: + symbol: str # the legend key: holder.legend[symbol] is the block + kind: str # "markdown" | "header" + width: int # twelfths + height: int # spec height units, as drawn + + @dataclass class SketchColumn: width: int # twelfths - children: list[SketchChart] = field(default_factory=list) + children: list[SketchChart | SketchBlock] = field(default_factory=list) @dataclass class SketchRow: - children: list[SketchChart | SketchColumn] = field(default_factory=list) + children: list[SketchChart | SketchBlock | SketchColumn] = field(default_factory=list) + + @property + def header_band(self) -> SketchBlock | None: + """The header, when this band is one header across the whole sketch: Superset's + HEADER between rows, as `{"header": ...}` is in rows.""" + if len(self.children) == 1: + only = self.children[0] + if isinstance(only, SketchBlock) and only.kind == "header" and only.width == 12: + return only + return None + + +def sketch_items(child) -> list: + """The charts and blocks of one band slot: a column's stack, or the item itself.""" + return list(child.children) if isinstance(child, SketchColumn) else [child] HOLE = "." # reserved: a deliberately empty cell (grid-template-areas prior art) @@ -65,7 +104,7 @@ def _widths_to_twelfths(cell_counts: list[int], total_cells: int) -> list[int]: @lru_cache(maxsize=64) -def parse_sketch_cached(lines: tuple[str, ...], legend_items: tuple[tuple[str, str], ...], +def parse_sketch_cached(lines: tuple[str, ...], legend_items: tuple[tuple[str, str | BlockRef], ...], line_units: int) -> list[SketchRow]: """Memoized parse: geometry lookups hit parsed_sketch per chart per rule, which measured ~cubic on sketch dashboards before caching. Callers treat @@ -73,7 +112,8 @@ def parse_sketch_cached(lines: tuple[str, ...], legend_items: tuple[tuple[str, s return parse_sketch(list(lines), dict(legend_items), line_units) -def parse_sketch(lines: list[str], legend: dict[str, str], line_units: int) -> list[SketchRow]: +def parse_sketch(lines: list[str], legend: dict[str, str | BlockRef], + line_units: int) -> list[SketchRow]: if not lines: raise SketchError("sketch is empty") grid = [[ch for ch in line if ch != " "] for line in lines] @@ -106,6 +146,21 @@ def parse_sketch(lines: list[str], legend: dict[str, str], line_units: int) -> l if len(cells) != (r1 - r0 + 1) * (c1 - c0 + 1): raise SketchError(f"symbol {s!r} does not form a solid rectangle") boxes[s] = (r0, r1, c0, c1) + if isinstance(legend[s], BlockRef) and legend[s].kind == "header" and r1 != r0: + raise SketchError( + f"header {s!r} spans {r1 - r0 + 1} lines: a header is one sketch line " + "(Superset sizes it to its text)") + + def item(s: str, w: int) -> SketchChart | SketchBlock: + r0, r1 = boxes[s][0], boxes[s][1] + ref = legend[s] + if isinstance(ref, BlockRef): + return SketchBlock(s, ref.kind, w, (r1 - r0 + 1) * line_units) + return SketchChart(ref, w, (r1 - r0 + 1) * line_units) + + def label(s: str) -> str: + ref = legend[s] + return f"{ref.kind} {s!r}" if isinstance(ref, BlockRef) else f"chart {ref!r}" # horizontal bands: cut where no rectangle spans the boundary cuts = [0] @@ -120,7 +175,7 @@ def parse_sketch(lines: list[str], legend: dict[str, str], line_units: int) -> l if not band: raise SketchError( f"line {b0 + 1} is entirely empty: Superset has no vertical spacer; " - "use more lines on neighbors or a markdown block in rows mode" + "use more lines on neighbors or draw a markdown block there" ) # vertical slices: group symbols sharing column extents slices: dict[tuple[int, int], list[str]] = {} @@ -165,10 +220,10 @@ def parse_sketch(lines: list[str], legend: dict[str, str], line_units: int) -> l r0, r1 = boxes[s][0], boxes[s][1] if r0 != b0: raise SketchError( - f"empty space above chart {legend[s]!r} (line {b0 + 1}): Superset " + f"empty space above {label(s)} (line {b0 + 1}): Superset " "packs upward; empty cells in a column go at the bottom" ) - row.children.append(SketchChart(legend[s], w, (r1 - r0 + 1) * line_units)) + row.children.append(item(s, w)) else: members.sort(key=lambda s: boxes[s][0]) # stacked members must tile downward from the band top with no @@ -184,7 +239,7 @@ def parse_sketch(lines: list[str], legend: dict[str, str], line_units: int) -> l "empty cells may only end a stack, not interrupt it)" ) expect = r1 + 1 - col.children.append(SketchChart(legend[s], w, (r1 - r0 + 1) * line_units)) + col.children.append(item(s, w)) row.children.append(col) rows.append(row) return rows diff --git a/chartwright/smoke.py b/chartwright/smoke.py index 2a9cce8..a37d8fe 100644 --- a/chartwright/smoke.py +++ b/chartwright/smoke.py @@ -7,19 +7,26 @@ replicate each viz's post-processing (binning, pivoting, normalization); that stays the renderer's job. +Each chart is queried as the dashboard queries it on load: with the default of +every native filter in its scope (filter_defaults), so row counts, the empty +check and the height fit read the rows a viewer first sees. + Pass = HTTP 200 AND non-empty result payload. Empty = warning naming the -chart, not a silent pass. +chart, not a silent pass; so are values an IN filter lists on a column the +chart groups or draws by that return no rows (they would draw nothing). """ from __future__ import annotations import math -from dataclasses import dataclass +from dataclasses import asdict, dataclass, field from .client import SupersetClient -from .compiler import _metric_payload, mixed_time_axis, time_binding +from .compiler import (_metric_payload, _num, _range_default_mask, mixed_time_axis, time_binding, + waterfall_time_axis) from .resolver import Resolution -from .spec import DashboardSpec, grid_header, grid_units_for_rows +from .spec import (DEFAULT_ROW_LIMIT, DashboardSpec, grid_fit, grid_header, grid_units_for_rows, + metric_label, pivot_frame, pivot_rows) def _fit_warning(chart, spec: DashboardSpec, result: list) -> str | None: @@ -36,29 +43,165 @@ def _fit_warning(chart, spec: DashboardSpec, result: list) -> str | None: if not data: return None if chart.type == "pivot_table": - if not chart.rows: - return None # columns-only pivot: a single metric band, height-safe - leaf = len({tuple(r.get(d) for d in chart.rows) for r in data}) - header, row = grid_header(chart) - what = f"pivot renders ~{leaf} leaf rows" + (" and its totals row" if chart.column_totals else "") + # The rows the pivot draws, not the records the query returned (rows x columns). + leaf, subtotals = pivot_rows(chart, data) + rows = leaf + subtotals + what = ("pivot renders only its totals row" if not rows else # no row attribute + f"pivot renders ~{leaf} leaf rows" + + (f", {subtotals} subtotal rows" if subtotals else "") + + (" and its totals row" if pivot_frame(chart)[2] else "")) + remedy = "filter it to fewer rows" # row_limit counts records, not pivot rows else: - leaf = len(data) # already capped by the query's row_limit - header, row = grid_header(chart, leaf) - what = f"table renders ~{leaf} rows" - if chart.page_length and leaf > chart.page_length: - # A paged table renders one page and its page controls (size.table-window's model). - leaf = chart.page_length - what = f"table renders a {chart.page_length}-row page and its pager" + rows = len(data) # already capped by the query's row_limit + what = f"table renders ~{rows} rows" + (" and its totals row" if chart.show_totals else "") + remedy = "cap row_limit" + held, header, row = grid_fit(chart, rows) + if held != rows: + # A paged table renders one page and its page controls (size.table-window's model). + what = f"table renders a {held}-row page and its pager" height = spec.resolved_height(chart.name) - # Shared grid model (chartwright/spec.py): the design critic's - # size.grid-fit and size.table-window read the same numbers, so pre-apply - # advice and this post-apply warning can never contradict each other. - needed = grid_units_for_rows(leaf, header, row) + # Shared grid model (chartwright/spec.py): the design critic's size.grid-fit, + # size.pivot-window and size.table-window ask grid_fit the same question, so + # pre-apply advice and this post-apply warning can never contradict each other. + needed = grid_units_for_rows(held, header, row) if needed <= height: return None return (f"{what} (~{needed * 40:.0f}px) but height={height:g} ({height * 40:.0f}px): " f"rows will hide behind an inner scrollbar; raise height to " - f"~{math.ceil(needed)} units or cap row_limit") + f"~{math.ceil(needed)} units or {remedy}") + + +def _window_warning(chart, rows: int) -> str | None: + """A rolling trendline KPI whose data holds too few time buckets for its window and + comparison (the design brain's data.rolling-window-span, from the real row count). + The backend keeps the windows from bucket rolling_min_periods on (pandas_postprocessing/ + rolling.py), and compare_lag reads that many points back from the latest.""" + if chart.type != "big_number_trend" or chart.rolling_type in (None, "cumsum"): + return None + least = max(1, chart.rolling_periods if chart.rolling_min_periods is None + else chart.rolling_min_periods) + lag = chart.compare_lag or 0 + points = max(0, rows - least + 1) + if points >= max(lag + 1, 2): + return None + return (f"a {least}-step rolling window" + (f" with compare_lag {lag}" if lag else "") + + f" over {rows} time buckets draws {points} trendline point(s)" + + (" and no change" if lag else "") + "; widen the chart's time range") + + +def _bridge_warning(chart, result: list) -> str | None: + """A bridge (a waterfall with steps) checked against its rows: every step and the + closing has one, nothing else does, and the closing reconciles. Superset draws the + closing as the running total, not the row's own value, so a closing that doesn't + add up still looks right on the chart; this is where it shows.""" + if chart.type != "waterfall" or chart.steps is None: + return None + label = metric_label(chart.metric) + values: dict[str, float | None] = {} + for q in result: + for row in q.get("data") or []: + key = str(row.get(chart.x_column)) + value = row.get(label) + values[key] = value if isinstance(value, (int, float)) else None + out = [] + if chart.opening is not None and chart.opening not in values: + out.append(f"no row for opening {chart.opening!r}, so the bridge starts at zero") + missing = [s for s in chart.steps if s not in values] + if missing: + out.append(f"no rows for steps {missing}") + if chart.closing not in values: + out.append(f"no row for closing {chart.closing!r}, so Superset draws no closing total") + extra = sorted(v for v in values if v not in chart.steps and v not in (chart.closing, chart.opening)) + if extra: + out.append(f"{extra} are not in steps: Superset draws them after the listed steps, A to Z") + closing = values.get(chart.closing) + steps = [v for k, v in values.items() if k != chart.closing] + if closing is not None and all(v is not None for v in steps): + added = sum(steps) + # A millionth: floating-point sums of a reconciled bridge differ far less, a + # closing that doesn't reconcile far more. + if not math.isclose(closing, added, rel_tol=1e-6, abs_tol=1e-6): + out.append(f"the closing row holds {closing:,.6g} but the steps add to {added:,.6g}: " + f"Superset draws {added:,.6g} as {chart.closing!r}") + return "bridge: " + "; ".join(out) if out else None + + +# Superset 6.x fills an unset box plot row limit with the control's default +# (sharedControls.tsx row_limit, 6.1.0 :236); smoke asks for as many. +BOX_PLOT_ROWS = 10000 + + +def _observations_warning(chart, rows: int) -> str | None: + """A box plot's query returns its observations before the boxplot step makes + quartiles of them; a query that stops at its row limit leaves some out.""" + if chart.type != "box_plot" or rows < (chart.row_limit or BOX_PLOT_ROWS): + return None + where = ("its row_limit" if chart.row_limit else + f"{BOX_PLOT_ROWS:,}, where Superset 6.0.0 or later stops an unset row_limit") + return (f"the observations reached {where} ({rows:,} rows), so the boxes may leave " + f"some out; raise row_limit or coarsen distribute_across") + + +def _value_key(v) -> str: + """A filter value or a returned one, compared as text: a list of [2, 4] matches the 2 + and 4 a numeric column returns, and 2.0 reads as 2.""" + if isinstance(v, bool): + return str(v).lower() + if isinstance(v, float) and v.is_integer(): + return str(int(v)) + return str(v) + + +def _series_limited(chart, column: str) -> bool: + """The chart keeps only its top N series by `column`, so listed values it drops are + the limit's doing, not missing data.""" + holders = (chart.a, chart.b) if chart.type == "mixed" else (chart,) + return any(getattr(h, "series_limit", None) and getattr(h, "groupby", None) == column + for h in holders) + + +def _empty_values_warning(chart, queries: list[dict], result: list, ds) -> str | None: + """Values a chart lists with IN on a column it groups or draws by, which come back + with no rows: a line chart that names four zones where two have data draws two lines + and a legend of two, and nothing on the dashboard says the other two are missing. + Only a column the query returns can be checked, since the data says which values + came back; a time column comes back as timestamps, so it is left alone.""" + notes = [] + for f in chart.filters: + if f.op != "IN" or f.sql is not None or not isinstance(f.value, list): + continue + column = f.column + pairs = [(q, r) for q, r in zip(queries, result) if column in q.get("columns", [])] + if not pairs or ds.is_temporal(column) or _series_limited(chart, column): + continue + back = {_value_key(row.get(column)) for _, r in pairs for row in r.get("data") or []} + missing = [v for v in f.value if _value_key(v) not in back] + if not missing: + continue + capped = any(len(r.get("data") or []) >= q.get("row_limit", math.inf) for q, r in pairs) + names = ", ".join(repr(v) for v in missing) + notes.append(f"{column} IN lists {names}, which return{'s' if len(missing) == 1 else ''}" + f" no rows" + (" within the row limit" if capped else "") + + f", so the chart draws nothing for {'it' if len(missing) == 1 else 'them'}") + return "; ".join(notes) or None + + +def _zero_colour_warning(chart, result: list) -> str | None: + """A big number at exactly 0 under a colour rule that takes 0. Superset colours the + number only when it is truthy (BigNumberViz.tsx `bigNumber ? getColorFromValue(...) + : false`, 4.1.4 :144, 5.0.0 :145, 6.1.0 :216), so the rule's colour never shows and + the number keeps the default text colour. Smoke already holds the value.""" + if chart.type != "big_number_total" or not chart.conditional_formatting: + return None + data = (result[0].get("data") or []) if result else [] + value = data[0].get(metric_label(chart.metric)) if data and isinstance(data[0], dict) else None + if isinstance(value, bool) or value != 0: + return None + taken = [r for r in chart.conditional_formatting if r.matches(0)] + if not taken: + return None + return (f"the number is 0, which Superset never colours, so it shows in the default " + f"colour, not the {taken[-1].color} its rules give 0") @dataclass @@ -67,39 +210,50 @@ class SmokeResult: ok: bool warning: bool detail: str + # The dashboard filter defaults the query ran with ("Carrier = All carriers"). + filter_defaults: list[str] = field(default_factory=list) + + def as_dict(self) -> dict: + """The JSON shape: filter_defaults only when there were some, so a dashboard + without filter defaults reports exactly as before.""" + out = asdict(self) + if not self.filter_defaults: + del out["filter_defaults"] + return out -def _filters_payload(chart) -> list[dict]: - return [{"col": f.column, "op": f.op, "val": f.value} for f in chart.filters if f.sql is None] +def _filters_payload(filters) -> list[dict]: + return [{"col": f.column, "op": f.op, "val": f.value} for f in filters if f.sql is None] -def _sql_where(chart) -> str | None: - """A chart's custom-SQL filters as the query's extra WHERE text (extras.where, - accepted by the chart data API in every supported release).""" - parts = [f"({f.sql})" for f in chart.filters if f.sql is not None] +def _sql_where(filters) -> str | None: + """Custom-SQL filters as the query's extra WHERE text (extras.where, accepted + by the chart data API in every supported release).""" + parts = [f"({f.sql})" for f in filters if f.sql is not None] return " AND ".join(parts) or None def _with_where(q: dict, chart) -> dict: - where = _sql_where(chart) + where = _sql_where(chart.filters) if where: q["extras"]["where"] = where return q -def _with_time_range(q: dict, chart, ds) -> dict: - """The chart's own time_range, bound the way the compiled chart binds it - (compiler.time_binding): a TEMPORAL_RANGE filter on a time axis, else the - dataset's main time column as granularity. A chart without a time_range - keeps the unfiltered query.""" - binding = time_binding(chart, ds) if chart.time_range else None +def _with_time_range(q: dict, chart, ds, time_range: str | None = None) -> dict: + """A time range (a dashboard filter's default, else the chart's own + time_range), bound the way the compiled chart binds it (compiler.time_binding): + a TEMPORAL_RANGE filter on a time axis, else the dataset's main time column as + granularity. Without a time range the query stays unfiltered.""" + time_range = time_range or chart.time_range + binding = time_binding(chart, ds) if time_range else None if binding is None: return q kind, column = binding - q["time_range"] = chart.time_range + q["time_range"] = time_range if kind == "axis": q["filters"] = q["filters"] + [ - {"col": column, "op": "TEMPORAL_RANGE", "val": chart.time_range}] + {"col": column, "op": "TEMPORAL_RANGE", "val": time_range}] else: q["granularity"] = column return q @@ -117,11 +271,16 @@ def _time_axis(column: str, grain: str | None) -> dict: def _query_for(chart, spec: DashboardSpec) -> dict: slug = spec.dashboard.slug + # A table or pivot asks for the rows its compiled chart asks for (the compiler's + # default when row_limit is unset), so the height fit counts every row it draws: a + # 1,000-record cap misses leaf rows of a pivot riding its 10,000 default. + grid = chart.type in ("table", "pivot_table") q: dict = { - "filters": _filters_payload(chart), + "filters": _filters_payload(chart.filters), "extras": {"time_grain_sqla": getattr(chart, "time_grain", None) or "P1D"}, "time_range": "No filter", - "row_limit": getattr(chart, "row_limit", None) or 1000, + "row_limit": getattr(chart, "row_limit", None) or ( + DEFAULT_ROW_LIMIT[chart.type] if grid else 1000), "columns": [], "metrics": [], "orderby": [], @@ -144,6 +303,9 @@ def metric(m: str): elif t == "bar": q["metrics"] = [metric(m) for m in chart.metrics] q["columns"] = [chart.x_column] + ([chart.groupby] if chart.groupby else []) + if chart.sort_metric() and chart.sort_metric() not in chart.metrics: + # The chart queries a sort metric it doesn't draw (extractExtraMetrics.ts). + q["metrics"].append(metric(chart.sort_metric())) elif t == "pie": q["metrics"] = [metric(chart.metric)] q["columns"] = [chart.groupby] @@ -171,6 +333,39 @@ def metric(m: str): return _with_where(q, chart) +def _waterfall_query(chart, spec: DashboardSpec, ds) -> dict: + """One row per step (Waterfall/buildQuery.ts: the x axis, then the breakdown). A + bridge is queried by its step column, so its rows can be checked by name.""" + x = (_time_axis(chart.x_column, chart.time_grain) + if chart.time_grain and waterfall_time_axis(chart, ds) else chart.x_column) + return _with_where({ + "filters": _filters_payload(chart.filters), + "extras": {"time_grain_sqla": chart.time_grain or "P1D"}, + "time_range": "No filter", + "row_limit": chart.row_limit or 1000, + "columns": [x] + ([chart.groupby] if chart.groupby else []), + "metrics": [_metric_payload(chart.metric, spec.dashboard.slug, chart.name)], + "orderby": [], + }, chart) + + +def _box_plot_query(chart, spec: DashboardSpec, ds) -> dict: + """The observations, as BoxPlot/buildQuery.ts asks for them before its boxplot + step: each time column of distribute_across at the grain, then the groups.""" + across = [_time_axis(c, chart.time_grain) + if chart.time_grain and ds.is_temporal(c) is not False else c + for c in chart.distribute_across] + return _with_where({ + "filters": _filters_payload(chart.filters), + "extras": {"time_grain_sqla": chart.time_grain or "P1D"}, + "time_range": "No filter", + "row_limit": chart.row_limit or BOX_PLOT_ROWS, + "columns": [*across, *chart.groupby], + "metrics": [_metric_payload(m, spec.dashboard.slug, chart.name) for m in chart.metrics], + "orderby": [], + }, chart) + + def _mixed_queries(chart, spec: DashboardSpec, ds) -> list[dict]: """Query A and query B, as MixedTimeseries/buildQuery.ts sends them.""" x = (_time_axis(chart.x_column, chart.time_grain) @@ -178,7 +373,7 @@ def _mixed_queries(chart, spec: DashboardSpec, ds) -> list[dict]: out = [] for series in (chart.a, chart.b): out.append(_with_where({ - "filters": _filters_payload(chart), + "filters": _filters_payload(chart.filters), "extras": {"time_grain_sqla": chart.time_grain or "P1D"}, "time_range": "No filter", "row_limit": chart.row_limit or 1000, @@ -189,30 +384,260 @@ def _mixed_queries(chart, spec: DashboardSpec, ds) -> list[dict]: return out -def smoke_chart(chart, spec: DashboardSpec, resolution: Resolution, client: SupersetClient) -> SmokeResult: +# -- the dashboard's filter defaults --------------------------------------------- +# +# How Superset applies a native filter's value on load (6.1.0 source): every chart in +# the filter's scope gets the filter's extraFormData (getFormDataWithExtraFilters.ts: +# 465-476, the scope being its chartsInScope, activeAllDashboardFilters.ts:113-122). +# Its `filters` are appended to the chart's query (buildQueryObject.ts:69-74 and 90); +# time_range overrides the query's and time_grain_sqla its extras +# (processExtraFormData.ts:31-44, constants.ts:30-53), which become the time axis' +# grain (normalizeTimeColumn.ts:57-71). The backend then applies a filter only when +# the chart's own dataset has a column of that name and drops it otherwise +# (superset/models/helpers.py:3024-3026 and 3069, reported as rejected at 3484-3492; +# the same check at helpers.py:1758 and 1765 in 4.1.4, 1769 and 1776 in 5.0.0). + + +@dataclass +class FilterDefault: + """One native filter's value on load, as it reaches the charts in its scope.""" + + label: str # as a result names it: "Carrier = All carriers" + charts: list[str] | None # the filter's scope; None is every chart + column: str | None = None # applies only to charts whose dataset has it + filters: list[dict] = field(default_factory=list) + time_range: str | None = None + time_grain: str | None = None + time_column: str | None = None + unapplied: str | None = None # why smoke can't apply it, said in the result + + +def _show(value) -> str: + return "" if value is None else str(_num(value) if isinstance(value, float) else value) + + +def _select_default(f, values: list, suffix: str = "") -> FilterDefault: + # The plugin rebuilds the default's data mask on load, and under inverse selection + # it excludes the values (SelectFilterPlugin.tsx:197-225 and 349-352, + # filters/utils.ts:33-58), whatever the saved mask says. + op = "NOT IN" if f.inverse_selection else "IN" + label = (f"{f.name} excludes {' and '.join(_show(v) for v in values)}" if f.inverse_selection + else f"{f.name} = {' or '.join(_show(v) for v in values)}") + return FilterDefault(label + suffix, f.charts, f.column, [{"col": f.column, "op": op, "val": values}]) + + +def _range_label(f) -> str: + ge, le = (None if b is None else _num(b) for b in (f.ge, f.le)) + if ge is not None and le is not None: + return f"{f.name} = {le}" if ge == le else f"{ge} <= {f.name} <= {le}" + return f"{f.name} <= {le}" if le is not None else f"{f.name} >= {ge}" + + +def _ancestors(name: str, by_name: dict) -> list[str]: + """Every filter `name` depends on, nearest parent last, as Superset walks them + (nativeFilters/dependencyGraph.ts:52-74 at 6.1.0): the nearest time range wins.""" + out: list[str] = [] + seen = {name} + + def visit(n: str) -> None: + for p in getattr(by_name[n], "dependencies", None) or []: + if p not in seen: + seen.add(p) + visit(p) + out.append(p) + + visit(name) + return out + + +def _first_value(f, by_name: dict, resolved: dict, resolution: Resolution, + client: SupersetClient) -> FilterDefault | None: + """`default_to_first`: the filter's first value, read with the query the Select + filter sends (Select/buildQuery.ts:56-66: its column, ordered by itself or by + sort_metric, ascending unless sort_descending, types.ts:80) and picked as the + plugin does, the first row (SelectFilterPlugin.tsx:359-366). The query carries + the filter's pre-filter (nativeFilters/utils.ts:91-99) and the values its + dependencies load with (FilterControls/state.ts:47-62). One row is enough: the + ordering decides which value comes first.""" + ds = resolution.datasets[f.dataset.key()] + q: dict = { + "columns": [f.column], + "metrics": [f.sort_metric] if f.sort_metric else [], + "orderby": [[f.sort_metric or f.column, not f.sort_descending]], + "filters": _filters_payload(f.pre_filter), + "extras": {}, + "time_range": f.time_range or "No filter", + "row_limit": 1, + } + where = _sql_where(f.pre_filter) + if where: + q["extras"]["where"] = where + if f.time_column: + q["granularity"] = f.time_column + for parent in _ancestors(f.name, by_name): + d = resolved.get(parent) + if d is None: + continue + if d.unapplied: + return FilterDefault(f"{f.name}'s first value", f.charts, f.column, + unapplied=f"it depends on {parent}, which smoke could not apply") + if d.filters and d.column in ds.columns: + q["filters"] = q["filters"] + d.filters + if d.time_range: + q["time_range"] = d.time_range + r = client.chart_data({"datasource": {"id": ds.id, "type": "table"}, "queries": [q], + "result_format": "json", "result_type": "full"}) + if r.status_code != 200: + return FilterDefault(f"{f.name}'s first value", f.charts, f.column, + unapplied=f"reading it failed: HTTP {r.status_code}") + try: + data = r.json()["result"][0].get("data") or [] + except Exception: # noqa: BLE001 - malformed body: say so, whatever the shape + return FilterDefault(f"{f.name}'s first value", f.charts, f.column, + unapplied="its value query returned an unreadable response") + if not data or f.column not in data[0]: + return None # no values: the dashboard selects none either + return _select_default(f, [data[0][f.column]], " (its first value)") + + +def _default_of(f, by_name: dict, resolved: dict, resolution: Resolution, + client: SupersetClient) -> FilterDefault | None: + if f.type == "select": + if f.default: + return _select_default(f, list(f.default)) + if f.default_to_first: + return _first_value(f, by_name, resolved, resolution, client) + return None + if f.type == "range": + mask = _range_default_mask(f) # the bounds the bundle's default applies + if mask is None: + return None + return FilterDefault(_range_label(f), f.charts, f.column, mask["extraFormData"]["filters"]) + if not f.default: + return None + label = f"{f.name} = {f.default}" + if f.type == "time_range": + return FilterDefault(label, f.charts, time_range=f.default) + if f.type == "time_grain": + return FilterDefault(label, f.charts, time_grain=f.default) + return FilterDefault(label, f.charts, time_column=f.default) + + +def filter_defaults(spec: DashboardSpec, resolution: Resolution, + client: SupersetClient) -> list[FilterDefault]: + """Each native filter's value on load. Only `default_to_first` reads data, one + small query per filter; a filter's parents resolve before it.""" + by_name = {f.name: f for f in spec.filters} + resolved: dict[str, FilterDefault | None] = {} + + def resolve(name: str) -> FilterDefault | None: + if name not in resolved: + f = by_name[name] + for parent in getattr(f, "dependencies", None) or []: + resolve(parent) + resolved[name] = _default_of(f, by_name, resolved, resolution, client) + return resolved[name] + + return [d for d in (resolve(f.name) for f in spec.filters) if d is not None] + + +def _axes(queries: list[dict]) -> list[dict]: + return [c for q in queries for c in q["columns"] + if isinstance(c, dict) and c.get("columnType") == "BASE_AXIS"] + + +def _with_filter_defaults(queries: list[dict], chart, ds, + defaults: list[FilterDefault]) -> tuple[list[dict], list[str], list[str]]: + """The chart's queries with the defaults in its scope, and which were applied + and which smoke could not apply. A default that changes nothing in this chart's + query (a column its dataset lacks, a time range it has no time binding for, a + grain without a time axis) is neither: the dashboard doesn't apply it either.""" + binding = time_binding(chart, ds) + applied: list[str] = [] + unapplied: list[str] = [] + # A later time range or grain overrides an earlier one (nativeFilters/utils.ts:125-134). + overrides: dict[str, tuple[str, str]] = {} # "time_range" | "time_grain" -> (value, label) + time_columns: list[str] = [] + ranged = chart.time_range not in (None, "No filter") + for d in defaults: + if d.charts is not None and chart.name not in d.charts: + continue + if d.column is not None and d.column not in ds.columns: + continue + if d.unapplied: + unapplied.append(f"{d.label} ({d.unapplied})") + elif d.filters: + for q in queries: + q["filters"] = q["filters"] + d.filters + applied.append(d.label) + elif d.time_range is not None: + ranged = ranged or d.time_range != "No filter" + if binding is not None: + overrides["time_range"] = (d.time_range, d.label) + elif d.time_grain is not None and _axes(queries): + overrides["time_grain"] = (d.time_grain, d.label) + elif d.time_column is not None: + time_columns.append(d.label) + time_range = overrides.get("time_range", (None,))[0] + if "time_grain" in overrides: + grain = overrides["time_grain"][0] + for q in queries: + q["extras"]["time_grain_sqla"] = grain + for axis in _axes(queries): + axis["timeGrain"] = grain + applied += [label for _, label in overrides.values()] + queries = [_with_time_range(q, chart, ds, time_range) for q in queries] + if time_columns and (ranged or (binding is not None and binding[0] == "axis")): + # A time column filter moves the chart's time range, and its time axis, onto + # another column (query_context_factory.py:235-309 at 6.1.0); smoke doesn't. + unapplied += [f"{label} (smoke keeps the chart's own time column)" for label in time_columns] + return queries, applied, unapplied + + +def smoke_chart(chart, spec: DashboardSpec, resolution: Resolution, client: SupersetClient, + defaults: list[FilterDefault] | None = None) -> SmokeResult: ds = resolution.for_chart(chart.dataset) + if chart.type == "mixed": + queries = _mixed_queries(chart, spec, ds) + elif chart.type == "waterfall": + queries = [_waterfall_query(chart, spec, ds)] + elif chart.type == "box_plot": + queries = [_box_plot_query(chart, spec, ds)] + else: + queries = [_query_for(chart, spec)] + queries, applied, unapplied = _with_filter_defaults(queries, chart, ds, defaults or []) ctx = { "datasource": {"id": ds.id, "type": "table"}, - "queries": [_with_time_range(q, chart, ds) for q in ( - _mixed_queries(chart, spec, ds) if chart.type == "mixed" else [_query_for(chart, spec)])], + "queries": queries, "result_format": "json", "result_type": "full", } + with_defaults = f" with the dashboard's filter defaults: {', '.join(applied)}" if applied else "" + not_applied = f"; not applied: {', '.join(unapplied)}" if unapplied else "" + + def outcome(ok: bool, warning: bool, detail: str) -> SmokeResult: + return SmokeResult(chart.name, ok, warning, detail + not_applied, applied) + r = client.chart_data(ctx) if r.status_code != 200: - return SmokeResult(chart.name, False, False, f"HTTP {r.status_code}: {r.text[:500]}") + return outcome(False, False, f"HTTP {r.status_code}: {r.text[:500]}" + + (f"; queried{with_defaults}" if applied else "")) try: result = r.json()["result"] rows = sum(len(q.get("data") or []) for q in result) except Exception as e: # noqa: BLE001 - malformed body is a failure, whatever the shape - return SmokeResult(chart.name, False, False, f"unparseable chart/data response: {e}") + return outcome(False, False, f"unparseable chart/data response: {e}") if rows == 0: - return SmokeResult(chart.name, True, True, "query succeeded but returned 0 rows") - fit = _fit_warning(chart, spec, result) - if fit: - return SmokeResult(chart.name, True, True, f"{rows} rows; {fit}") - return SmokeResult(chart.name, True, False, f"{rows} rows") + return outcome(True, True, f"query succeeded but returned 0 rows{with_defaults}") + notes = [w for w in ((_fit_warning(chart, spec, result) or _bridge_warning(chart, result) + or _observations_warning(chart, rows) or _window_warning(chart, rows) + or _zero_colour_warning(chart, result)), + _empty_values_warning(chart, queries, result, ds)) if w] + if notes: + return outcome(True, True, f"{rows} rows{with_defaults}; " + "; ".join(notes)) + return outcome(True, False, f"{rows} rows{with_defaults}") def smoke(spec: DashboardSpec, resolution: Resolution, client: SupersetClient) -> list[SmokeResult]: - return [smoke_chart(c, spec, resolution, client) for c in spec.charts] + defaults = filter_defaults(spec, resolution, client) + return [smoke_chart(c, spec, resolution, client, defaults) for c in spec.charts] diff --git a/chartwright/spec.py b/chartwright/spec.py index 9912ad3..43eb07c 100644 --- a/chartwright/spec.py +++ b/chartwright/spec.py @@ -3,7 +3,7 @@ Anything not expressible here does not exist. Validation errors are the only feedback channel an LLM caller gets; keep messages precise and actionable. -Surface: 15 chart types, per-chart WHERE filters, a dashboard-level native +Surface: 17 chart types, per-chart WHERE filters, a dashboard-level native filter bar (select, time range, numeric range, time grain, time column), markdown blocks, headers, dividers, and tabs. """ @@ -12,6 +12,7 @@ import math import re +import unicodedata import uuid as _uuid from typing import Annotated, Any, ClassVar, Literal, Union @@ -24,14 +25,16 @@ "timeseries_line": 10000, "timeseries_bar": 10000, "timeseries_area": 10000, "timeseries_scatter": 10000, "bar": 10000, "pie": 100, "table": 1000, "pivot_table": 10000, "heatmap": 10000, "histogram": 10000, "funnel": 10, - "treemap": 100, "mixed": 10000, + "treemap": 100, "mixed": 10000, "waterfall": 10000, } DEFAULT_TIME_GRAIN = "P1D" # How a rendered table or pivot spends spec height. ONE model, read by the design # critic (size.table-window, size.pivot-window, size.grid-fit), the page-length and # search-box fills, and apply-time smoke, so offline advice, data-aware advice, the -# fills and the apply warning can never disagree about the same chart. +# fills and the apply warning can never disagree about the same chart. Every consumer +# asks grid_fit() what a chart drawing N body rows needs; they differ only in where N +# comes from (the rows a query returned, a probed count, or the row_limit ceiling). # # Measured 2026-10-03 on rendered Superset 4.1.4, 5.0.0 and 6.1.0 (1600 px viewport, # default theme; docs/DESIGN-BRAIN.md, "Calibration"). Each constant is the largest @@ -42,14 +45,23 @@ GRID_CONTROLS_UNITS = 1.05 # the bar above the rows that search_box or any page_length # adds: 41.1 px / 39.1 px (6.1.0 draws 33.1 px for a page size alone) GRID_PAGER_UNITS = 1.45 # the pager under a table of more than one page: 57.9 px / 42.9 px -PIVOT_UNITS_PER_ROW = 0.65 # pivot body row: 25.8 px on average (19 rows: 490.1 px) +GRID_TOTALS_UNITS = 0.74 # show_totals' Summary row: the table foot, which DataTable keeps + # out of the scrolling body (6.1.0 plugin-chart-table DataTable/ + # hooks/useSticky.tsx:183,217); not measured, so one body row +PIVOT_UNITS_PER_ROW = 0.65 # pivot body row: 25.8 px on average (19 rows: 490.1 px), as 6.1.0's + # styles give it: 12 px text at line-height 1.4, 4 px padding above + # and below and a 1 px rule (react-pivottable/Styles.ts:27,33,113,115) PIVOT_FRAME_UNITS = 2.55 # pivot card title, margins and padding: 101 px -PIVOT_HEADER_ROW_UNITS = 0.66 # one pivot header row: 26.3 px; a pivot draws one per column - # dimension plus two (the metric row and the row-dimension names) +PIVOT_HEADER_ROW_UNITS = 0.66 # one pivot header row: 26.3 px (pivot_frame counts them) PIVOT_TOTALS_UNITS = 0.72 # the totals row column_totals pins over the bottom rows: 28.8 px + # (sticky at the bottom, Styles.ts:55-59) PIVOT_HSCROLL_UNITS = 0.45 # the horizontal scrollbar under a pivot whose column dimensions # outgrow the panel: 11 px in Chromium's classic scrollbars, and - # Windows draws 17 px; overlay scrollbars (macOS) take none + # Windows draws 17 px; overlay scrollbars (macOS) take none. + # 6.1.0 styles its own at 8 px (PivotTableChart.tsx:70-73) +# The metric names: one more attribute of a pivot's rows or columns, which Superset adds +# even for a single metric (6.1.0 plugin-chart-pivot-table PivotTableChart.tsx:98,382-386). +PIVOT_METRIC = "\x00metric" def grid_units_for_rows(rows: float, header_units: float = GRID_HEADER_UNITS, @@ -64,35 +76,119 @@ def grid_rows_visible(height: float, header_units: float = GRID_HEADER_UNITS, return max(0.0, (height - header_units) / row_units) -def table_header_units(*, controls: bool = False, pager: bool = False) -> float: +def table_header_units(*, controls: bool = False, pager: bool = False, + totals: bool = False) -> float: """Everything a table draws besides its body rows: card title, column header and padding, plus the bar above the rows (`controls`: a search box, or any - page_length, which draws a page-size picker even on one page) and the pager under - them (`pager`: more rows than one page holds).""" + page_length, which draws a page-size picker even on one page), the pager under + them (`pager`: more rows than one page holds) and the Summary row (`totals`: + show_totals).""" return (GRID_HEADER_UNITS + (GRID_CONTROLS_UNITS if controls else 0) - + (GRID_PAGER_UNITS if pager else 0)) + + (GRID_PAGER_UNITS if pager else 0) + (GRID_TOTALS_UNITS if totals else 0)) + + +def pivot_axes(chart) -> tuple[list[str], list[str]]: + """The (row, column) attributes a pivot draws, as 6.1.0's PivotTableChart.tsx:377-386 + builds them: transpose swaps the dimensions, and the metric names (PIVOT_METRIC) are + one more attribute, on the columns unless metrics_layout is rows; outermost, or + innermost with combine_metric.""" + rows, cols = ((list(chart.columns), list(chart.rows)) if chart.transpose + else (list(chart.rows), list(chart.columns))) + side = rows if chart.metrics_layout == "rows" else cols + side.insert(len(side) if chart.combine_metric else 0, PIVOT_METRIC) + return rows, cols + + +def pivot_frame(chart) -> tuple[int, bool, bool]: + """(header rows, horizontal-scrollbar room, totals row) of a pivot as configured. + 6.1.0's TableRenderers.tsx draws a header row per column attribute, and one naming + the row attributes when there are any (:1418-1423); the totals row with + column_totals, and always when there are no row attributes (:367, :1429). Column + dimensions, not metric names, are what can outgrow the panel.""" + rows, cols = pivot_axes(chart) + return (len(cols) + (1 if rows else 0), any(a != PIVOT_METRIC for a in cols), + chart.column_totals or not rows) + + +def pivot_header_units(header_rows: int, *, hscroll: bool = False, + totals: bool = False) -> float: + """Everything a pivot draws besides its body rows: card frame, its header rows, + room for a horizontal scrollbar (`hscroll`) and the pinned totals row (`totals`).""" + return (PIVOT_FRAME_UNITS + PIVOT_HEADER_ROW_UNITS * header_rows + + (PIVOT_HSCROLL_UNITS if hscroll else 0) + + (PIVOT_TOTALS_UNITS if totals else 0)) -def pivot_header_units(column_dims: int, *, totals: bool = False) -> float: - """Everything a pivot draws besides its body rows: card frame, a header row per - column dimension plus two, room for a horizontal scrollbar when column dimensions - can outgrow the panel, and the pinned totals row (`column_totals`).""" - return (PIVOT_FRAME_UNITS + PIVOT_HEADER_ROW_UNITS * (column_dims + 2) - + (PIVOT_HSCROLL_UNITS if column_dims else 0) - + (PIVOT_TOTALS_UNITS if totals else 0)) +def pivot_rows(chart, records: list[dict]) -> tuple[int, int]: + """(leaf rows, subtotal rows) a pivot draws for its query's records: a body row per + distinct row key, never per record (the query returns rows x columns records). The + metric names count as a row attribute when laid out as rows, and row_subtotals adds + a row per distinct key prefix (6.1.0 react-pivottable/utilities.ts:957-975). With no + row attribute there is no body row: the pivot is its totals row.""" + rows, _ = pivot_axes(chart) + if not rows: + return 0, 0 + labels = [metric_label(m) for m in chart.metrics] if PIVOT_METRIC in rows else [None] + keys = {tuple(label if a == PIVOT_METRIC else r.get(a) for a in rows) + for r in records for label in labels} + subtotals = (len({k[:i] for k in keys for i in range(1, len(rows))}) + if chart.row_subtotals else 0) + return len(keys), subtotals + + +def pivot_rows_from_counts(chart, counts: dict[str, int]) -> tuple[int, int, bool]: + """(leaf rows, subtotal rows, exact) a pivot draws, from the distinct values of each + row dimension counted one column at a time (`counts`). Exact with at most one row + dimension; with more, the combinations number at least the largest count, so the + leaf rows are a lower bound and subtotals go uncounted (exact False).""" + rows, _ = pivot_axes(chart) + if not rows: + return 0, 0, True + size = {a: len(chart.metrics) if a == PIVOT_METRIC else counts[a] for a in rows} + dims = [a for a in rows if a != PIVOT_METRIC] + if len(dims) > 1: + per = len(chart.metrics) if PIVOT_METRIC in rows else 1 + return max(size[a] for a in dims) * per, 0, False + subtotals = (sum(math.prod(size[a] for a in rows[:i]) for i in range(1, len(rows))) + if chart.row_subtotals else 0) + return math.prod(size.values()), subtotals, True + + +def table_page(chart) -> int | None: + """Rows on one page of a paged table (page_length > 0), never more than row_limit; + None when every row is on one page.""" + if not getattr(chart, "page_length", None): + return None + return min(chart.page_length, chart.row_limit) if chart.row_limit else chart.page_length def grid_header(chart, rows: int | None = None) -> tuple[float, float]: """(header units, units per body row) of a table or pivot chart as configured, drawing `rows` body rows (None: unknown, so a paged table counts its pager).""" if chart.type == "pivot_table": - return (pivot_header_units(len(chart.columns), totals=chart.column_totals), + header_rows, hscroll, totals = pivot_frame(chart) + return (pivot_header_units(header_rows, hscroll=hscroll, totals=totals), PIVOT_UNITS_PER_ROW) page = chart.page_length or 0 return (table_header_units(controls=bool(page) or chart.search_box, - pager=bool(page) and (rows is None or rows > page)), + pager=bool(page) and (rows is None or rows > page), + totals=chart.show_totals), GRID_UNITS_PER_ROW) + +def grid_fit(chart, rows: int | None) -> tuple[int | None, float, float]: + """(body rows the panel must hold, header units, units per body row) of a table or + pivot drawing `rows` body rows: the one estimate that smoke (the rows the query + returned), size.grid-fit (a probed count), size.pivot-window (what the spec alone + fixes) and size.table-window (the row_limit ceiling) each compare a height with. A + paged table holds one page beside its pager. None: unknown (a paged table still + holds one page).""" + header, row = grid_header(chart, rows) + page = table_page(chart) if chart.type == "table" else None + if page is not None and (rows is None or rows > page): + rows = page + return rows, header, row + ADHOC_AGGREGATES = ("SUM", "AVG", "COUNT", "COUNT_DISTINCT", "MIN", "MAX") _ADHOC_RE = re.compile(r"^(SUM|AVG|COUNT|COUNT_DISTINCT|MIN|MAX)\((.+?)\)(?:\s+AS\s+(.+))?$") # A custom-SQL metric: SQL() AS