From d7d9b906f57e841b1507509b9e3b3a8a985c6e5a Mon Sep 17 00:00:00 2001 From: Thomas Lin Pedersen Date: Fri, 4 Sep 2026 13:43:56 +0200 Subject: [PATCH 01/21] Upgrade to hephaestus 0.4.0 --- CHANGELOG.md | 3 + Cargo.lock | 15 +- ggsql-cli/Cargo.toml | 6 + ggsql-cli/examples/visual_test.rs | 283 +++++++++++- src/Cargo.toml | 16 +- src/writer/hephaestus/canvas.rs | 169 +++++++ src/writer/hephaestus/compose.rs | 351 ++++++++++++++ src/writer/hephaestus/facet.rs | 2 +- src/writer/hephaestus/geom/densified.rs | 2 +- src/writer/hephaestus/geom/mod.rs | 2 +- src/writer/hephaestus/mod.rs | 586 +++++------------------- src/writer/hephaestus/raster.rs | 59 +++ src/writer/hephaestus/wiring.rs | 22 +- src/writer/mod.rs | 16 +- src/writer/options.rs | 50 ++ 15 files changed, 1070 insertions(+), 512 deletions(-) create mode 100644 src/writer/hephaestus/canvas.rs create mode 100644 src/writer/hephaestus/compose.rs create mode 100644 src/writer/hephaestus/raster.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec8c0c1..40508bd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ the png writer draws them. ### Changed +- The png writer now records its render resolution in the PNG itself, so a + figure rendered above 96 dpi reports its true physical size instead of being + read as 72 dpi by whatever opens it. - Dodging now only takes effect where groups actually meet on a position. A layer whose grouping gives every group a position of its own — `colour` mapped to the same column as the discrete axis, say — is drawn at its full width diff --git a/Cargo.lock b/Cargo.lock index 788e814d..5025f0ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2282,6 +2282,7 @@ dependencies = [ "anyhow", "clap", "ggsql", + "png", "regex", "serde_json", "termimad", @@ -2505,9 +2506,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hephaestus" -version = "0.1.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca883b65468b673aab28f1c7a86690ad5a00c31a4547c93577ecfd569c47e9b4" +checksum = "91f060494e66d6e8a9d6cbf513f0f39e212d37a89293b91fc40f61aeb461f9e5" dependencies = [ "bytemuck", "clipper2-rust", @@ -5881,7 +5882,6 @@ dependencies = [ "thiserror 2.0.18", "wgpu-core-deps-apple", "wgpu-core-deps-emscripten", - "wgpu-core-deps-wasm", "wgpu-core-deps-windows-linux-android", "wgpu-hal", "wgpu-naga-bridge", @@ -5906,15 +5906,6 @@ dependencies = [ "wgpu-hal", ] -[[package]] -name = "wgpu-core-deps-wasm" -version = "29.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2f2fb042f36920771deb0b966543c5751b18f3d327760ffc90f74e20b2dcd4" -dependencies = [ - "wgpu-hal", -] - [[package]] name = "wgpu-core-deps-windows-linux-android" version = "29.0.3" diff --git a/ggsql-cli/Cargo.toml b/ggsql-cli/Cargo.toml index 20c96f7c..5d1ac49d 100644 --- a/ggsql-cli/Cargo.toml +++ b/ggsql-cli/Cargo.toml @@ -21,6 +21,12 @@ name = "visual_test" path = "examples/visual_test.rs" required-features = ["png", "duckdb", "vegalite", "builtin-data"] +[dev-dependencies] +# Decoding renders back to pixels, so `visual_test --baseline` can compare +# pictures rather than files. Dev-only: nothing in the shipped binary reads a +# PNG. Same major as the encoder hephaestus uses, so no second copy is built. +png = "0.18" + [dependencies] ggsql = { workspace = true } diff --git a/ggsql-cli/examples/visual_test.rs b/ggsql-cli/examples/visual_test.rs index 7dbd7fe5..69bee936 100644 --- a/ggsql-cli/examples/visual_test.rs +++ b/ggsql-cli/examples/visual_test.rs @@ -66,6 +66,15 @@ struct Args { /// Render resolution, which also scales the plot chrome #[arg(long, default_value_t = 300.0)] dpi: f64, + + /// A previous `--out` directory to diff this run's renders against. + /// + /// Every cell is labelled unchanged / changed / new in the report, and the + /// header carries the counts. A hephaestus bump changes behaviour on + /// purpose, so the point is not a zero diff — it is turning "eyeball 190 + /// cells" into "eyeball the ones that moved". + #[arg(long, value_name = "DIR")] + baseline: Option, } // ============================================================================ @@ -199,12 +208,195 @@ fn front_matter_title(text: &str) -> Option { // Running the corpus // ============================================================================ +/// How a render compares with the same cell in a `--baseline` run. +#[derive(Clone, Copy)] +enum Delta { + /// Same pixels, exactly. + Identical, + /// Different pixels, but the whole picture matches under a small + /// translation — the signature of a chrome-width change nudging the panel. + /// Carries the residual left at the best alignment. + Shifted(f64), + /// Differs by more than a shift explains. These are the cells to look at. + Changed(f64), + /// The baseline has no render for this cell. + New, +} + +/// Mean absolute grey difference, 0–255, below which an aligned pair counts as +/// the same picture. Antialiasing alone puts a genuine match a little above +/// zero, so this cannot be `0.0`; it is set from the observed spread across the +/// doc corpus rather than derived. +const SHIFT_TOLERANCE: f64 = 2.0; + +/// How far to search for an alignment, in pixels of the full-size render. +const MAX_SHIFT: i32 = 24; + +/// Factor the comparison downsamples by before searching. The search cost is +/// quadratic in both the shift range and the resolution, and a panel shift is a +/// whole-image effect that survives a box filter. +const COMPARE_SCALE: u32 = 8; + +impl Delta { + /// Compare freshly rendered bytes with the baseline's copy of `name`. + /// + /// Exact equality is close to useless across a dependency bump: a few + /// pixels of chrome-width change shifts every panel, so almost every cell + /// "differs" while looking identical. So the comparison aligns first — + /// searching a small translation and keeping the best residual — which + /// collapses a uniform shift to nearly nothing and leaves anything + /// structural large. + fn against(baseline: &Path, name: &str, bytes: &[u8]) -> Self { + let Ok(old) = fs::read(baseline.join("assets").join(name)) else { + return Delta::New; + }; + if old == bytes { + return Delta::Identical; + } + match (Grey::decode(&old), Grey::decode(bytes)) { + (Some(a), Some(b)) => match a.aligned_residual(&b) { + // Different dimensions: not a shift, and not comparable. + None => Delta::Changed(f64::INFINITY), + Some(r) if r <= SHIFT_TOLERANCE => Delta::Shifted(r), + Some(r) => Delta::Changed(r), + }, + // Undecodable, so fall back to saying it moved rather than + // claiming a match we cannot support. + _ => Delta::Changed(f64::INFINITY), + } + } + + fn label(self) -> String { + match self { + Delta::Identical => "identical".to_string(), + Delta::Shifted(r) => format!("shifted · {r:.2}"), + Delta::Changed(r) if r.is_finite() => format!("changed · {r:.2}"), + Delta::Changed(_) => "changed".to_string(), + Delta::New => "new".to_string(), + } + } + + /// CSS class, and the bucket the header counts by. + fn class(self) -> &'static str { + match self { + Delta::Identical => "identical", + Delta::Shifted(_) => "shifted", + Delta::Changed(_) => "changed", + Delta::New => "new", + } + } + + /// Whether a human still needs to look at this cell. + fn needs_review(self) -> bool { + matches!(self, Delta::Changed(_)) + } +} + +/// A render reduced to one grey byte per pixel, downsampled for comparison. +struct Grey { + width: usize, + height: usize, + px: Vec, +} + +impl Grey { + /// Decode a PNG and reduce it to a downsampled grey plane. + fn decode(bytes: &[u8]) -> Option { + let decoder = png::Decoder::new(std::io::Cursor::new(bytes)); + let mut reader = decoder.read_info().ok()?; + let mut buf = vec![0; reader.output_buffer_size()?]; + let info = reader.next_frame(&mut buf).ok()?; + let channels = match info.color_type { + png::ColorType::Rgba => 4, + png::ColorType::Rgb => 3, + png::ColorType::Grayscale => 1, + png::ColorType::GrayscaleAlpha => 2, + png::ColorType::Indexed => return None, + }; + // Composite onto white as a viewer would, so a transparent background + // does not read as black and swamp the residual. + let grey_at = |i: usize| -> f64 { + let p = &buf[i * channels..]; + let (r, g, b, a) = match channels { + 1 => (p[0], p[0], p[0], 255), + 2 => (p[0], p[0], p[0], p[1]), + 3 => (p[0], p[1], p[2], 255), + _ => (p[0], p[1], p[2], p[3]), + }; + let lum = 0.299 * r as f64 + 0.587 * g as f64 + 0.114 * b as f64; + let a = a as f64 / 255.0; + lum * a + 255.0 * (1.0 - a) + }; + + let scale = COMPARE_SCALE as usize; + let width = (info.width as usize).div_ceil(scale); + let height = (info.height as usize).div_ceil(scale); + let mut px = vec![0u8; width * height]; + for by in 0..height { + for bx in 0..width { + let mut sum = 0.0; + let mut n = 0.0; + for y in by * scale..((by + 1) * scale).min(info.height as usize) { + for x in bx * scale..((bx + 1) * scale).min(info.width as usize) { + sum += grey_at(y * info.width as usize + x); + n += 1.0; + } + } + px[by * width + bx] = (sum / n).round() as u8; + } + } + Some(Grey { width, height, px }) + } + + /// The smallest mean absolute difference over a search of translations. + /// + /// `None` when the two have different dimensions, which no translation + /// reconciles. + fn aligned_residual(&self, other: &Grey) -> Option { + if self.width != other.width || self.height != other.height { + return None; + } + let reach = MAX_SHIFT / COMPARE_SCALE as i32; + let mut best = f64::INFINITY; + for dy in -reach..=reach { + for dx in -reach..=reach { + // Only the overlap is compared, so a shift is not penalised for + // the sliver it moves off the canvas. + let mut sum = 0.0f64; + let mut n = 0usize; + for y in 0..self.height as i32 { + let oy = y + dy; + if oy < 0 || oy >= self.height as i32 { + continue; + } + for x in 0..self.width as i32 { + let ox = x + dx; + if ox < 0 || ox >= self.width as i32 { + continue; + } + let a = self.px[y as usize * self.width + x as usize] as f64; + let b = other.px[oy as usize * self.width + ox as usize] as f64; + sum += (a - b).abs(); + n += 1; + } + } + if n > 0 { + best = best.min(sum / n as f64); + } + } + } + Some(best) + } +} + /// What a cell turned out to be, and what came of running it. enum Outcome { /// A query with a `VISUALISE` clause: the renders it produced. Plot { png: Option, png_error: Option, + /// How the render compares with `--baseline`, when one was given. + delta: Option, /// Vega-Lite JSON, inlined into the report when `--compare` is on vegalite: Option, vegalite_error: Option, @@ -312,9 +504,14 @@ fn run_cells(source: Source, args: &Args, assets: &Path) -> SourceResult { Ok(spec) => { warnings.extend(spec.warnings().iter().map(|w| w.message.clone())); + let mut delta = None; let (png, png_error) = match capture(|| png_writer.render(&spec)) { Ok(bytes) => { let name = format!("{}-{:02}.png", slug(&label), cell.index); + delta = args + .baseline + .as_deref() + .map(|b| Delta::against(b, &name, &bytes)); match fs::write(assets.join(&name), &bytes) { Ok(()) => (Some(name), None), Err(e) => (None, Some(format!("could not write PNG: {e}"))), @@ -335,6 +532,7 @@ fn run_cells(source: Source, args: &Args, assets: &Path) -> SourceResult { Outcome::Plot { png, png_error, + delta, vegalite: vl, vegalite_error: vl_error, } @@ -451,11 +649,27 @@ fn write_report(results: &[SourceResult], args: &Args, out: &Path) -> std::io::R .flat_map(|r| &r.cells) .filter(|c| c.is_problem()) .count(); + let bucket = |class: &str| { + results + .iter() + .flat_map(|r| &r.cells) + .filter( + |c| matches!(&c.outcome, Outcome::Plot { delta: Some(d), .. } if d.class() == class), + ) + .count() + }; + let drift = args.baseline.as_deref().map(|_| { + ( + bucket("identical") + bucket("shifted"), + bucket("changed"), + bucket("new"), + ) + }); let aspect = format!("{} / {}", args.width, args.height); let mut html = String::new(); - html.push_str(&report_head(total, plots, problems, args)); + html.push_str(&report_head(total, plots, problems, drift, args)); html.push_str("