From 472ce998fcd23d162f08a8b728b77c4b2ae2475b Mon Sep 17 00:00:00 2001 From: Mike Yumatov Date: Thu, 17 Sep 2026 08:03:39 +0300 Subject: [PATCH] feat(meta)!: add page-local attrs for integrations Preserve strictly validated, shallowly merged attrs through canonical metadata, storage and NAPI responses without changing rendering or cache dependencies. Protect wire roundtrips with bounded nesting and exact float decoding. BREAKING CHANGE: Rust Meta struct literals must initialize attrs. --- CHANGELOG.md | 10 + CLAUDE.md | 10 + Cargo.lock | 4 + README.md | 2 +- crates/rw-meta/Cargo.toml | 1 + crates/rw-meta/src/attrs.rs | 66 ++++ crates/rw-meta/src/fields.rs | 44 ++- crates/rw-meta/src/lib.rs | 281 +++++++++++++++++- crates/rw-napi/Cargo.toml | 3 + crates/rw-napi/src/attrs.rs | 67 +++++ crates/rw-napi/src/lib.rs | 2 + crates/rw-napi/src/types.rs | 8 +- crates/rw-site/benches/page_rendering.rs | 43 +++ crates/rw-site/benches/site_structure.rs | 43 +++ crates/rw-site/src/page.rs | 37 +++ crates/rw-site/src/site.rs | 3 + crates/rw-site/src/site_state.rs | 130 ++++++++ crates/rw-storage-fs/src/lib.rs | 59 +++- crates/rw-storage-s3/Cargo.toml | 1 + crates/rw-storage-s3/src/format.rs | 102 +++++++ crates/rw-storage-s3/src/publisher.rs | 68 +++++ crates/rw-storage/Cargo.toml | 4 +- crates/rw-storage/src/mock.rs | 3 +- crates/rw-storage/src/storage.rs | 94 +++++- docs/embedding.md | 42 ++- docs/metadata.md | 92 +++++- packages/core/index.d.ts | 2 + packages/core/package.json | 1 + .../core/test/helpers/attrs-s3-fixture.mjs | 170 +++++++++++ .../core/test/render-page-attrs-s3.test.mjs | 130 ++++++++ .../core/test/render-page-metadata.test.mjs | 102 +++++++ 31 files changed, 1596 insertions(+), 28 deletions(-) create mode 100644 crates/rw-meta/src/attrs.rs create mode 100644 crates/rw-napi/src/attrs.rs create mode 100644 packages/core/test/helpers/attrs-s3-fixture.mjs create mode 100644 packages/core/test/render-page-attrs-s3.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 586efa61..9a0ca6f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **Page-local attrs for integrations** — JSON-compatible `attrs` in frontmatter and selected YAML sidecars survive S3 publication and appear as optional `meta.attrs` in NAPI/core `renderPage()` responses. Frontmatter overlays top-level keys; nested values replace whole, null remains data, and attrs never inherit. Empty attrs are omitted. Authored nesting is bounded for manifest/cache transport; Rust JSON byte roundtrips preserve supported finite `f64` values. No built-in HTTP/viewer or rendering semantics change. See [Page Metadata](docs/metadata.md#attrs-page-local-integration-data). + +### Changed + +- **Breaking (pre-1.0, Rust source):** Public `rw_meta::Meta` literals must add `attrs: Default::default()` (or use `Meta::resolve`). S3 manifests remain version 1; upgrade readers before integrations rely on attrs, since older readers ignore and lose them on reserialization. Populated attrs increase manifest/cache payloads and selected-page conversion work. JavaScript numeric identifiers requiring exact large integers should be strings. + ## [0.1.36] - 2026-09-07 ### New Features diff --git a/CLAUDE.md b/CLAUDE.md index 7f2d34cc..ac7ee282 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,6 +62,16 @@ uses effective `Section.name`; explicitly named, kind-declaring roots opt in, with eligibility included in the existing resolution fingerprint. Storage keeps the flattened wire with optional, omitted-when-absent `name`; S3 stays version 1, requiring readers to upgrade before publishers enable explicit names. +`Meta.attrs` is page-local JSON data: selected sidecar plus top-level frontmatter +overlay, whole nested replacement, literal null, no inheritance. Invalid source +attrs are discarded as a whole field. The same Arc carries attrs through cached +and virtual renders; flattened Document/structure/S3 wires omit empty attrs. +Only NAPI/core `renderPage().meta.attrs` exposes them (omitted empty), with normal +JS number precision. HTTP/viewer/search/navigation and renderer semantics stay +unchanged. Attrs add neither fetch paths nor render fingerprints nor refresh +guarantees; future renderer consumption must extend actual cache dependencies. +Upgrade S3 readers before consumers rely on attrs; old readers discard them on +reserialization. Public Rust Meta literals need `attrs: Default::default()`. ## Key Technical Details diff --git a/Cargo.lock b/Cargo.lock index 2f3e90e5..4eeec70f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4468,6 +4468,7 @@ version = "0.1.36" dependencies = [ "pulldown-cmark", "rw-sections", + "serde_json", "serde_yaml", ] @@ -4481,11 +4482,13 @@ dependencies = [ "rw-cache", "rw-cache-s3", "rw-config", + "rw-meta", "rw-renderer", "rw-site", "rw-storage", "rw-storage-fs", "rw-storage-s3", + "serde_json", "tokio", ] @@ -4634,6 +4637,7 @@ dependencies = [ "chrono", "parking_lot", "pretty_assertions", + "rw-meta", "rw-parser", "rw-plantuml", "rw-storage", diff --git a/README.md b/README.md index 490e7df5..63af13bd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Publish the same markdown to Confluence pages or embed in Backstage with native - **Status badges** — inline colored pill labels with Confluence status-macro parity - **GitHub-style alerts** — `[!NOTE]`, `[!TIP]`, `[!WARNING]`, and more - **Navigation and TOC** — automatic sidebar, breadcrumbs, and table of contents -- **Page metadata** — YAML frontmatter or sidecar files for titles, descriptions, navigation order, and explicit section/catalog and diagram names independent of URLs +- **Page metadata** — YAML frontmatter or sidecar files for titles, descriptions, navigation order, explicit section/catalog and diagram names independent of URLs, and page-local integration attrs exposed by NAPI/core - **Confluence rendering** — produce publish-ready bundles (XHTML + diagrams) for any Confluence publishing tool - **Backstage integration** — embed docs with native Backstage plugins diff --git a/crates/rw-meta/Cargo.toml b/crates/rw-meta/Cargo.toml index 1eabaa08..28c93b84 100644 --- a/crates/rw-meta/Cargo.toml +++ b/crates/rw-meta/Cargo.toml @@ -13,3 +13,4 @@ workspace = true pulldown-cmark = { workspace = true } rw-sections = { workspace = true } serde_yaml = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/rw-meta/src/attrs.rs b/crates/rw-meta/src/attrs.rs new file mode 100644 index 00000000..4e90162b --- /dev/null +++ b/crates/rw-meta/src/attrs.rs @@ -0,0 +1,66 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; +use serde_yaml::Value as Yaml; + +// Leave room in serde_json's nesting budget for the surrounding wire envelope: +// root, document collection, document, and attrs map. +const MAX_VALUE_NESTING_DEPTH: usize = 127 - 4; + +pub(crate) fn from_yaml(value: &Yaml) -> Result, String> { + let Yaml::Mapping(values) = value else { + return Err("expected an attrs mapping".to_owned()); + }; + values + .iter() + .map(|entry| to_json_entry(entry, MAX_VALUE_NESTING_DEPTH)) + .collect() +} + +fn to_json_entry( + (key, value): (&Yaml, &Yaml), + remaining_containers: usize, +) -> Result<(String, Json), String> { + let Yaml::String(key) = key else { + return Err("expected string object keys".to_owned()); + }; + Ok((key.clone(), to_json(value, remaining_containers)?)) +} + +fn to_json(value: &Yaml, remaining_containers: usize) -> Result { + let remaining_containers = if matches!(value, Yaml::Sequence(_) | Yaml::Mapping(_)) { + remaining_containers + .checked_sub(1) + .ok_or_else(|| "attrs nesting exceeds the supported transport depth".to_owned())? + } else { + remaining_containers + }; + match value { + Yaml::Null => Ok(Json::Null), + Yaml::Bool(value) => Ok(Json::Bool(*value)), + Yaml::String(value) => Ok(Json::String(value.clone())), + Yaml::Number(value) => { + let number = if let Some(value) = value.as_i64() { + Some(serde_json::Number::from(value)) + } else if let Some(value) = value.as_u64() { + Some(serde_json::Number::from(value)) + } else { + value.as_f64().and_then(serde_json::Number::from_f64) + }; + number + .map(Json::Number) + .ok_or_else(|| "expected a finite JSON number".to_owned()) + } + Yaml::Sequence(values) => values + .iter() + .map(|value| to_json(value, remaining_containers)) + .collect::, _>>() + .map(Json::Array), + Yaml::Mapping(values) => values + .iter() + .map(|entry| to_json_entry(entry, remaining_containers)) + .collect::, String>>() + .map(Json::Object), + Yaml::Tagged(_) => Err("custom YAML tags are not supported in attrs".to_owned()), + } +} diff --git a/crates/rw-meta/src/fields.rs b/crates/rw-meta/src/fields.rs index e9f8a111..4e8f092f 100644 --- a/crates/rw-meta/src/fields.rs +++ b/crates/rw-meta/src/fields.rs @@ -1,4 +1,7 @@ -use std::{borrow::Cow, collections::HashSet}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashSet}, +}; use serde_yaml::{Mapping, Value}; @@ -12,13 +15,14 @@ pub(crate) struct MetaFields { pub description: Option, pub pages: Option>, pub name: Option, + pub attrs: BTreeMap, } impl MetaFields { /// Extract fields from one YAML source; a failing field drops only itself, /// while a source that fails to parse (invalid YAML, non-mapping root) /// contributes nothing and yields one `Severity::Error` diagnostic. - /// Extraction order is fixed (kind, namespace, title, description, pages, name) + /// Extraction order is fixed (kind, namespace, title, description, pages, name, attrs) /// so diagnostics come back deterministic. pub(crate) fn from_yaml_with_diagnostics( yaml: &str, @@ -70,11 +74,12 @@ impl MetaFields { description: string_field(&mapping, "description", source, &mut diagnostics), pages: pages_field(&mapping, source, &mut diagnostics), name: name_field(&mapping, source, &mut diagnostics), + attrs: attrs_field(&mapping, source, &mut diagnostics), }; (fields, diagnostics) } - /// Merge `other` onto self. `other` fields win when Some. + /// Merge `other` onto self. Optional fields win when Some; attrs overlay by key. pub(crate) fn merge(mut self, other: Self) -> Self { self.kind = other.kind.or(self.kind); self.namespace = other.namespace.or(self.namespace); @@ -82,11 +87,42 @@ impl MetaFields { self.description = other.description.or(self.description); self.pages = other.pages.or(self.pages); self.name = other.name.or(self.name); + self.attrs.extend(other.attrs); self } } -const KNOWN_KEYS: [&str; 6] = ["kind", "namespace", "title", "description", "pages", "name"]; +const KNOWN_KEYS: [&str; 7] = [ + "kind", + "namespace", + "title", + "description", + "pages", + "name", + "attrs", +]; + +fn attrs_field( + mapping: &Mapping, + source: DiagnosticSource, + diagnostics: &mut Vec, +) -> BTreeMap { + let Some(value) = mapping.get("attrs") else { + return BTreeMap::new(); + }; + match crate::attrs::from_yaml(value) { + Ok(attrs) => attrs, + Err(message) => { + diagnostics.push(Diagnostic { + source, + field: Some("attrs".to_owned()), + severity: Severity::Warning, + message, + }); + BTreeMap::new() + } + } +} fn normalize_known_keys(mapping: &Mapping) -> Result, &'static str> { let mut seen = HashSet::new(); diff --git a/crates/rw-meta/src/lib.rs b/crates/rw-meta/src/lib.rs index f9e28b6c..50abbaeb 100644 --- a/crates/rw-meta/src/lib.rs +++ b/crates/rw-meta/src/lib.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +mod attrs; mod diagnostic; mod fields; mod head; @@ -8,7 +11,7 @@ use head::Head; /// Resolved page metadata from all sources. /// -/// Rust struct literals must include `name: None` when no name is declared; +/// Rust struct literals must include `name: None` and empty `attrs` when absent; /// prefer [`Meta::resolve`] when constructing metadata from document sources. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Meta { @@ -27,6 +30,12 @@ pub struct Meta { /// Declared page-local name; never inherited. Overrides section identity only /// when this page declares `kind`; otherwise retained but not effective. pub name: Option, + /// Page-local JSON data; never inherited. + /// + /// Source resolution bounds array/object nesting for manifest/cache transport. + /// Callers constructing attrs directly must also respect the wire readers' + /// nesting limits. + pub attrs: BTreeMap, } /// Resolution result: canonical fields plus every recoverable problem found @@ -46,7 +55,8 @@ impl Meta { /// Internally: /// 1. Parses meta.yaml into base fields /// 2. Extracts frontmatter and first H1 from markdown via pulldown-cmark - /// 3. Merges frontmatter over meta.yaml (frontmatter wins per field) + /// 3. Overlays valid frontmatter fields onto meta.yaml; attrs merge by + /// top-level key, while other supplied fields replace their sidecar values /// 4. Resolves title: frontmatter title, else `meta.yaml` title, else H1, /// else titlecased filename stem, else stem verbatim, else `"Untitled"` #[must_use] @@ -99,6 +109,7 @@ impl Meta { description: merged.description, pages: merged.pages, name: merged.name, + attrs: merged.attrs, }, diagnostics, } @@ -146,6 +157,272 @@ fn titlecase_from_slug(slug: &str) -> String { mod tests { use super::*; + #[test] + fn attrs_shallow_overlay_preserves_null_and_sidecar_keys() { + let resolved = Meta::resolve_with_diagnostics( + Some("---\nattrs:\n owner: null\n nested: {new: true}\n tags: [new]\n---\n# Page"), + Some("attrs:\n owner: old\n kept: 42\n nested: {old: true}\n tags: [old]"), + "page.md", + ); + assert_eq!(resolved.meta.attrs["owner"], serde_json::Value::Null); + assert_eq!(resolved.meta.attrs["kept"], serde_json::json!(42)); + assert_eq!( + resolved.meta.attrs["nested"], + serde_json::json!({"new": true}) + ); + assert_eq!(resolved.meta.attrs["tags"], serde_json::json!(["new"])); + assert!(resolved.diagnostics.is_empty()); + } + + #[test] + fn attrs_preserve_json_types_and_deterministic_keys() { + let resolved = Meta::resolve_with_diagnostics( + None, + Some( + "attrs:\n z: [null, true, false, text, -9223372036854775808, 18446744073709551615, 1.5, {}, []]\n a: {z: 1, a: {z: 2, a: 3}}\n text: '42'", + ), + "page.md", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved + .meta + .attrs + .keys() + .map(String::as_str) + .collect::>(), + ["a", "text", "z"] + ); + assert_eq!(resolved.meta.attrs["text"], serde_json::json!("42")); + assert_eq!( + resolved.meta.attrs["z"], + serde_json::json!([null, true, false, "text", i64::MIN, u64::MAX, 1.5, {}, []]) + ); + assert_eq!( + serde_json::to_string(&resolved.meta.attrs["a"]).unwrap(), + r#"{"a":{"a":3,"z":2},"z":1}"# + ); + } + + #[test] + fn attrs_absent_and_empty_add_no_overrides() { + for yaml in ["title: Kept", "attrs: {}"] { + let markdown = format!("---\n{yaml}\n---"); + let empty = Meta::resolve_with_diagnostics(Some(&markdown), None, "p"); + assert!(empty.meta.attrs.is_empty()); + assert!(empty.diagnostics.is_empty()); + let base = + Meta::resolve_with_diagnostics(Some(&markdown), Some("attrs: {kept: true}"), "p"); + assert_eq!(base.meta.attrs["kept"], serde_json::json!(true)); + assert!(base.diagnostics.is_empty()); + let sidecar = Meta::resolve_with_diagnostics(None, Some(yaml), "p"); + assert!(sidecar.meta.attrs.is_empty()); + assert!(sidecar.diagnostics.is_empty()); + } + assert!(Meta::resolve(None, None, "p").attrs.is_empty()); + } + + #[test] + fn attrs_transport_depth_preserves_field_and_source_recovery() { + // Both container kinds count, even when the deepest container is empty. + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + for depth in [123, 124, 125, 126, 127] { + let mut expected = leaf_value.clone(); + let leaf_depth = usize::from(leaf != "null"); + for level in leaf_depth..depth { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let fields = + format!("title: Kept\nattrs: {{kept: changed, value: {value}}}"); + let markdown = format!("---\n{fields}\n---"); + let fallback = "attrs: {kept: fallback}\ntitle: Fallback"; + let fallback_markdown = format!("---\n{fallback}\n---"); + let result = match source { + DiagnosticSource::Sidecar => Meta::resolve_with_diagnostics( + Some(&fallback_markdown), + Some(&fields), + "p", + ), + DiagnosticSource::Frontmatter => { + Meta::resolve_with_diagnostics(Some(&markdown), Some(fallback), "p") + } + }; + let context = format!("{shape}/{leaf}/{depth}/{source:?}"); + if depth == 123 { + assert!(result.diagnostics.is_empty(), "{context}"); + assert_eq!(result.meta.attrs["value"], expected, "{context}"); + } else { + assert_eq!(result.meta.attrs.len(), 1, "{context}"); + assert_eq!( + result.meta.attrs["kept"], + serde_json::json!("fallback"), + "{context}" + ); + assert_eq!(result.diagnostics.len(), 1, "{context}"); + let diagnostic = &result.diagnostics[0]; + assert_eq!(diagnostic.source, source, "{context}"); + if depth == 127 { + // The existing YAML parser rejects the entire source first. + assert_eq!(diagnostic.field, None, "{context}"); + assert_eq!(diagnostic.severity, Severity::Error, "{context}"); + assert_eq!(result.meta.title, "Fallback", "{context}"); + } else { + assert_eq!(diagnostic.field.as_deref(), Some("attrs"), "{context}"); + assert_eq!(diagnostic.severity, Severity::Warning, "{context}"); + } + } + if depth < 127 && source == DiagnosticSource::Frontmatter { + assert_eq!(result.meta.title, "Kept", "{context}"); + } + } + // With no overlay, a valid sibling in the sidecar also survives. + if (124..=126).contains(&depth) { + let fields = format!("title: Kept\nattrs: {{value: {value}}}"); + let result = Meta::resolve_with_diagnostics(None, Some(&fields), "p"); + assert_eq!(result.meta.title, "Kept"); + assert!(result.meta.attrs.is_empty()); + } + } + } + } + } + + #[test] + fn attrs_invalid_values_drop_whole_field_per_source() { + for yaml in [ + "text", + "42", + "true", + "[]", + "null", + "{kept: changed, 1: value}", + "{kept: changed, !custom key: value}", + "{kept: changed, bad: {1: value}}", + "{kept: changed, bad: {true: value}}", + "{kept: changed, bad: {null: value}}", + "{kept: changed, bad: {[a]: value}}", + "{kept: changed, bad: { !custom key: value }}", + "{kept: changed, bad: .nan}", + "{kept: changed, bad: .inf}", + "{kept: changed, bad: -.inf}", + "!custom scalar", + "!custom {key: value}", + "{kept: changed, bad: !custom scalar}", + "{kept: changed, bad: !custom {key: value}}", + "{kept: changed, bad: [!custom scalar]}", + "{kept: changed, bad: [1, {2: value}]}", + ] { + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let fields = format!("attrs: {yaml}\ntitle: Kept"); + let markdown = format!("---\n{fields}\n---"); + let result = match source { + DiagnosticSource::Sidecar => { + Meta::resolve_with_diagnostics(None, Some(&fields), "p") + } + DiagnosticSource::Frontmatter => Meta::resolve_with_diagnostics( + Some(&markdown), + Some("attrs: {kept: fallback}"), + "p", + ), + }; + assert_eq!(result.meta.title, "Kept", "{source:?}: {yaml}"); + if source == DiagnosticSource::Sidecar { + assert!(result.meta.attrs.is_empty(), "{yaml}"); + } else { + assert_eq!(result.meta.attrs.len(), 1, "{yaml}"); + assert_eq!( + result.meta.attrs["kept"], + serde_json::json!("fallback"), + "{yaml}" + ); + } + assert_eq!(result.diagnostics.len(), 1, "{source:?}: {yaml}"); + assert_eq!( + result.diagnostics[0].field.as_deref(), + Some("attrs"), + "{source:?}: {yaml}" + ); + assert_eq!(result.diagnostics[0].source, source, "{yaml}"); + assert_eq!(result.diagnostics[0].severity, Severity::Warning, "{yaml}"); + } + } + } + + #[test] + fn attrs_tagged_known_key_and_duplicate_source_recovery() { + let result = Meta::resolve_with_diagnostics( + None, + Some("? !custom attrs\n: {kept: true}\ntitle: !custom 42"), + "p", + ); + assert_eq!(result.meta.attrs["kept"], serde_json::json!(true)); + assert_eq!(result.meta.title, "42"); + assert!(result.diagnostics.is_empty()); + for yaml in [ + "attrs: {one: 1}\nattrs: {two: 2}\ntitle: Dropped", + "attrs: {one: 1}\n? !custom attrs\n: {two: 2}\ntitle: Dropped", + ] { + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let markdown = format!("---\n{yaml}\n---"); + let result = match source { + DiagnosticSource::Sidecar => { + Meta::resolve_with_diagnostics(None, Some(yaml), "p") + } + DiagnosticSource::Frontmatter => Meta::resolve_with_diagnostics( + Some(&markdown), + Some("attrs: {kept: true}\ntitle: Fallback"), + "p", + ), + }; + if source == DiagnosticSource::Sidecar { + assert!(result.meta.attrs.is_empty()); + assert_eq!(result.meta.title, "P"); + } else { + assert_eq!(result.meta.attrs.len(), 1); + assert_eq!(result.meta.attrs["kept"], serde_json::json!(true)); + assert_eq!(result.meta.title, "Fallback"); + } + assert_eq!(result.diagnostics.len(), 1); + assert_eq!(result.diagnostics[0].field, None); + assert_eq!(result.diagnostics[0].source, source); + assert_eq!(result.diagnostics[0].severity, Severity::Error); + } + } + } + + #[test] + fn attrs_diagnostics_follow_name_and_source_order() { + let result = Meta::resolve_with_diagnostics( + Some("---\nattrs: []\nname: []\n---"), + Some("attrs: []\nname: []"), + "p", + ); + assert_eq!( + result + .diagnostics + .iter() + .map(|d| (d.source, d.field.as_deref())) + .collect::>(), + [ + (DiagnosticSource::Sidecar, Some("name")), + (DiagnosticSource::Sidecar, Some("attrs")), + (DiagnosticSource::Frontmatter, Some("name")), + (DiagnosticSource::Frontmatter, Some("attrs")), + ] + ); + } + #[test] fn declared_name_scalar_and_identifier_boundaries() { for (yaml, expected) in [ diff --git a/crates/rw-napi/Cargo.toml b/crates/rw-napi/Cargo.toml index ac05812e..73a9f48f 100644 --- a/crates/rw-napi/Cargo.toml +++ b/crates/rw-napi/Cargo.toml @@ -19,12 +19,15 @@ napi-derive = "3" rw-cache = { workspace = true } rw-cache-s3 = { workspace = true } rw-config = { workspace = true } +rw-meta = { workspace = true } rw-renderer = { workspace = true } rw-site = { workspace = true } rw-storage = { workspace = true } rw-storage-fs = { workspace = true } rw-storage-s3 = { workspace = true } +serde_json = { workspace = true } + tokio = { version = "1", features = ["rt", "rt-multi-thread"] } [build-dependencies] diff --git a/crates/rw-napi/src/attrs.rs b/crates/rw-napi/src/attrs.rs new file mode 100644 index 00000000..7e7f059e --- /dev/null +++ b/crates/rw-napi/src/attrs.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; + +use napi::bindgen_prelude::{JsObjectValue, Null, Object, ToNapiValue}; +use napi::{Env, JsValue, Property, Result, Unknown, sys}; +use serde_json::Value; + +/// Output-only binding adapter retaining the selected page's canonical metadata. +/// JSON traversal is deferred until napi converts the response on the JS thread. +pub struct Attrs(pub(crate) Arc); + +impl ToNapiValue for Attrs { + /// Converts attrs to a JavaScript object with ordinary data properties. + /// + /// # Errors + /// Returns an error if a number cannot convert to `f64`, an array exceeds + /// `u32::MAX` elements, or a Node-API operation fails. + /// + /// # Safety + /// `env` must be valid on its JavaScript thread with an active handle scope. + unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { + let env = Env::from_raw(env); + Ok(object(&env, val.0.attrs.iter())?.raw()) + } +} + +fn object<'env, 'value>( + env: &'env Env, + entries: impl Iterator, +) -> Result> { + let mut object = Object::new(env)?; + for (key, value) in entries { + let converted_value = json_value(env, value)?; + // A JS string key preserves embedded NUL. Defining a data property also + // avoids invoking Object.prototype's __proto__ setter. + let property = Property::new() + .with_name(env, key.as_str())? + .with_napi_value(env, converted_value)?; + object.define_properties(&[property])?; + } + Ok(object) +} + +fn json_value<'env>(env: &'env Env, value: &Value) -> Result> { + match value { + Value::Null => Null.into_unknown(env), + Value::Bool(value) => value.into_unknown(env), + Value::String(value) => value.as_str().into_unknown(env), + Value::Number(value) => { + // All JSON numbers are JS Numbers, including u64 (with ordinary JS + // precision loss), never napi's serde-json unsigned string fallback. + let number = value + .as_f64() + .ok_or_else(|| napi::Error::from_reason("Invalid JSON number in page attrs"))?; + number.into_unknown(env) + } + Value::Array(values) => { + let len = u32::try_from(values.len()) + .map_err(|_| napi::Error::from_reason("Page attrs array is too large"))?; + let mut array = env.create_array(len)?; + for (index, value) in (0..len).zip(values) { + array.set(index, json_value(env, value)?)?; + } + array.into_unknown(env) + } + Value::Object(values) => object(env, values.iter())?.into_unknown(env), + } +} diff --git a/crates/rw-napi/src/lib.rs b/crates/rw-napi/src/lib.rs index 328a2cd2..b72aa794 100644 --- a/crates/rw-napi/src/lib.rs +++ b/crates/rw-napi/src/lib.rs @@ -1,3 +1,4 @@ +mod attrs; mod types; use std::collections::HashMap; @@ -418,6 +419,7 @@ fn build_page_response(site: &Site, path: &str) -> Result { last_modified, description: result.meta.description.clone(), page_kind: result.meta.kind.clone(), + attrs: (!result.meta.attrs.is_empty()).then(|| attrs::Attrs(Arc::clone(&result.meta))), section_ref, subpath, }, diff --git a/crates/rw-napi/src/types.rs b/crates/rw-napi/src/types.rs index f3ec71f4..30193d0f 100644 --- a/crates/rw-napi/src/types.rs +++ b/crates/rw-napi/src/types.rs @@ -3,6 +3,8 @@ use std::collections::HashMap; use napi_derive::napi; use rw_site::{Section, SectionAnchor}; +use crate::attrs::Attrs; + #[napi(object)] pub struct DiagramsConfig { #[napi(js_name = "krokiUrl")] @@ -164,7 +166,7 @@ pub struct NavigationResponse { pub section_ancestry: HashMap>, } -#[napi(object)] +#[napi(object, object_from_js = false)] pub struct PageMetaResponse { pub title: String, pub path: String, @@ -175,6 +177,8 @@ pub struct PageMetaResponse { pub description: Option, #[napi(js_name = "kind")] pub page_kind: Option, + #[napi(ts_type = "Record")] + pub attrs: Option, #[napi(js_name = "sectionRef")] pub section_ref: String, /// Page path relative to its section root. Stable across whole-section @@ -201,7 +205,7 @@ pub struct TocEntryResponse { pub id: String, } -#[napi(object)] +#[napi(object, object_from_js = false)] pub struct PageResponse { pub meta: PageMetaResponse, pub breadcrumbs: Vec, diff --git a/crates/rw-site/benches/page_rendering.rs b/crates/rw-site/benches/page_rendering.rs index 99d92ddf..d3aee0c3 100644 --- a/crates/rw-site/benches/page_rendering.rs +++ b/crates/rw-site/benches/page_rendering.rs @@ -131,3 +131,46 @@ fn caching(bencher: Bencher, kind: &str) { _ => unreachable!(), } } + +/// Retained Site / persistent-cache response with opaque page metadata. Identical +/// markdown and mtimes; setup and validation are untimed. This is Rust serving +/// work, not the NAPI conversion performed by an embedding host. +#[divan::bench(args = ["empty", "small"])] +fn attrs_cache_hit(bencher: Bencher, kind: &str) { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("docs"); + fs::create_dir(&source_dir).unwrap(); + let attrs = if kind == "small" { + serde_json::json!({ + "owner": "payments", "audiences": ["operators"], + "support": {"url": "https://example.org/support"}, "reviewDate": null + }) + } else { + serde_json::json!({}) + }; + for (name, body) in [ + ("cached.md", generate_markdown(10, 3)), + ( + "cached.meta.yaml", + serde_json::json!({"attrs": attrs}).to_string(), + ), + ] { + let path = source_dir.join(name); + fs::write(&path, body).unwrap(); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000)) + .unwrap(); + } + let site = create_site_with_config( + source_dir, + Arc::new(FileCache::new(dir.path().join("cache"), "bench")), + PageRendererConfig::default(), + ); + let first = site.render("cached").unwrap(); + assert_eq!(first.meta.attrs.is_empty(), kind == "empty"); + assert!(site.render("cached").unwrap().from_cache); + bencher.bench(|| site.render(black_box("cached"))); +} diff --git a/crates/rw-site/benches/site_structure.rs b/crates/rw-site/benches/site_structure.rs index e6ed59cf..9f7d3b68 100644 --- a/crates/rw-site/benches/site_structure.rs +++ b/crates/rw-site/benches/site_structure.rs @@ -104,3 +104,46 @@ fn build_from_scratch(bencher: Bencher) { .with_inputs(|| create_site(source_dir.clone())) .bench_values(|site| site.navigation(None)); } + +/// Synthetic 100-page all-site attrs cost, including filesystem metadata decode, +/// indexing and structure-cache serialization (even NullCache serializes before +/// discarding bytes). Fixture creation is untimed, with identical bodies/mtimes. +#[divan::bench(args = ["empty", "small"])] +fn attrs_build_from_scratch(bencher: Bencher, kind: &str) { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("docs"); + fs::create_dir(&source_dir).unwrap(); + let attrs = if kind == "small" { + serde_json::json!({ + "owner": "payments", "audiences": ["operators"], + "support": {"url": "https://example.org/support"}, "reviewDate": null + }) + } else { + serde_json::json!({}) + }; + let sidecar = serde_json::json!({"attrs": attrs}).to_string(); + for i in 0..100 { + for (suffix, body) in [ + ("md", "# Selected\n\nFixed body.\n"), + ("meta.yaml", sidecar.as_str()), + ] { + let path = source_dir.join(format!("page-{i:04}.{suffix}")); + fs::write(&path, body).unwrap(); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000)) + .unwrap(); + } + } + let check = create_site(source_dir.clone()); + assert_eq!(check.navigation(None).unwrap().items.len(), 100); + assert_eq!( + check.render("page-0000").unwrap().meta.attrs.is_empty(), + kind == "empty" + ); + bencher + .with_inputs(|| create_site(source_dir.clone())) + .bench_values(|site| site.navigation(None)); +} diff --git a/crates/rw-site/src/page.rs b/crates/rw-site/src/page.rs index 455d8db8..5eeacce4 100644 --- a/crates/rw-site/src/page.rs +++ b/crates/rw-site/src/page.rs @@ -568,6 +568,7 @@ struct CachedPageRef<'a> { #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -591,6 +592,7 @@ mod tests { path: path.to_owned(), has_content, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.to_owned(), description: None, @@ -921,6 +923,40 @@ mod tests { assert!(result.toc.is_empty()); } + #[test] + fn attrs_use_current_canonical_meta_on_fresh_cached_and_virtual_pages() { + let (_temp, cache) = file_cache(); + let storage = MockStorage::new() + .with_file("page", "Page", "# Page\n\nUnchanged body") + .with_mtime("page", 1000.0); + let renderer = PageRenderer::new(Arc::new(storage), cache, PageRendererConfig::default()); + let mut before = make_page("Page", "page", true); + Arc::make_mut(&mut before.meta) + .attrs + .insert("owner".into(), serde_json::json!("old")); + let mut after = before.clone(); + Arc::make_mut(&mut after.meta) + .attrs + .insert("owner".into(), serde_json::json!("new")); + assert!(!Arc::ptr_eq(&before.meta, &after.meta)); + let ctx = RenderContext::default(); + let first = renderer.render("page", &before, vec![], &ctx).unwrap(); + let second = renderer.render("page", &after, vec![], &ctx).unwrap(); + assert!(!first.from_cache); + assert!(second.from_cache); + assert_eq!(first.html, second.html); + assert!(Arc::ptr_eq(&first.meta, &before.meta)); + assert_eq!(first.meta.attrs["owner"], serde_json::json!("old")); + assert!(Arc::ptr_eq(&second.meta, &after.meta)); + assert_eq!(second.meta.attrs["owner"], serde_json::json!("new")); + + after.has_content = false; + let virtual_page = renderer.render("page", &after, vec![], &ctx).unwrap(); + assert!(!virtual_page.has_content); + assert!(Arc::ptr_eq(&virtual_page.meta, &after.meta)); + assert_eq!(virtual_page.meta.attrs["owner"], serde_json::json!("new")); + } + #[test] fn test_render_page_cache_hit() { let temp_dir = tempfile::tempdir().unwrap(); @@ -1225,6 +1261,7 @@ mod tests { let renderer = create_renderer(storage); let mut page = make_page("Test", "test", true); page.meta = Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Test".to_owned(), description: Some("A description".to_owned()), diff --git a/crates/rw-site/src/site.rs b/crates/rw-site/src/site.rs index 6763aafc..85c57eef 100644 --- a/crates/rw-site/src/site.rs +++ b/crates/rw-site/src/site.rs @@ -591,6 +591,7 @@ mod tests { // Ensure Site is Send + Sync for use with Arc static_assertions::assert_impl_all!(super::Site: Send, Sync); + use std::collections::BTreeMap; use std::sync::Arc; use rw_storage::{Document, MockStorage, StorageErrorKind}; @@ -609,6 +610,7 @@ mod tests { path: path.to_owned(), has_content: true, meta: Arc::new(rw_meta::Meta { + attrs: BTreeMap::new(), name: None, title: title.to_owned(), description: None, @@ -692,6 +694,7 @@ mod tests { path: s.path.to_owned(), has_content: s.has_content, meta: Arc::new(rw_meta::Meta { + attrs: BTreeMap::new(), name: None, title: s.title.to_owned(), description: s.description.map(ToOwned::to_owned), diff --git a/crates/rw-site/src/site_state.rs b/crates/rw-site/src/site_state.rs index dbfca21e..2c88ab57 100644 --- a/crates/rw-site/src/site_state.rs +++ b/crates/rw-site/src/site_state.rs @@ -1107,6 +1107,7 @@ impl From for SiteState { mod tests { use super::*; use rw_meta::Meta; + use std::collections::BTreeMap; fn test_document( path: impl Into, @@ -1120,6 +1121,7 @@ mod tests { path: path.into(), has_content, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.into(), description: None, @@ -2838,6 +2840,134 @@ mod tests { ); } + #[test] + fn attrs_f64_bits_roundtrip_structure_cache() { + use rw_cache::{Cache, FileCache}; + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = FileCache::new(tmp.path().join("cache"), "v1"); + let bucket = cache.bucket("site"); + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let mut document = fingerprint_document("guide", "Guide", None, true); + document.meta = Arc::new(resolved.meta); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document); + let state = builder.build(); + // In-memory Value conversion would miss byte-decoder float rounding. + state.to_cache(bucket.as_ref(), "etag"); + let restored = + SiteState::from_cache(bucket.as_ref(), "etag").expect("readable structure cache"); + assert_eq!( + restored.get_page("guide").unwrap().meta.attrs["value"] + .as_f64() + .unwrap() + .to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn attrs_transport_depth_boundary_roundtrips_structure_cache() { + use rw_cache::{Cache, FileCache}; + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = FileCache::new(tmp.path().join("cache"), "v1"); + let bucket = cache.bucket("site"); + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + let mut expected = leaf_value; + for level in usize::from(leaf != "null")..123 { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {value}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!(resolved.meta.attrs["value"], expected, "{shape}/{leaf}"); + let mut document = fingerprint_document("guide", "Guide", None, true); + document.meta = Arc::new(resolved.meta); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let state = builder.build(); + state.to_cache(bucket.as_ref(), "etag"); + let restored = SiteState::from_cache(bucket.as_ref(), "etag") + .expect("readable structure cache"); + assert_eq!( + restored.get_page("guide").unwrap().meta.attrs["value"], + expected, + "{shape}/{leaf}" + ); + assert_eq!( + restored.get_page("guide").unwrap().meta, + document.meta, + "{shape}/{leaf}" + ); + } + } + } + + #[test] + fn attrs_only_changes_preserve_fingerprint_and_roundtrip_structure_cache() { + let mut document = fingerprint_document("guide", "Guide", None, true); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let empty = builder.build(); + for owner in ["old", "new"] { + Arc::make_mut(&mut document.meta) + .attrs + .insert("owner".into(), serde_json::json!(owner)); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let state = builder.build(); + assert!(Arc::ptr_eq( + &state.get_page("guide").unwrap().meta, + &document.meta + )); + assert_eq!( + state.resolution_fingerprint(), + empty.resolution_fingerprint() + ); + let json = serde_json::to_string(&CachedSiteStateRef::from(&state)).unwrap(); + let rebuilt = SiteState::from(serde_json::from_str::(&json).unwrap()); + assert_eq!( + rebuilt.resolution_fingerprint(), + state.resolution_fingerprint() + ); + assert_eq!( + rebuilt.get_page("guide").unwrap().meta.attrs["owner"], + serde_json::json!(owner) + ); + } + } + #[test] fn fingerprint_changes_on_title() { let a = fingerprint_of( diff --git a/crates/rw-storage-fs/src/lib.rs b/crates/rw-storage-fs/src/lib.rs index fe181c99..b91c1a51 100644 --- a/crates/rw-storage-fs/src/lib.rs +++ b/crates/rw-storage-fs/src/lib.rs @@ -812,13 +812,60 @@ mod tests { ); } + #[test] + fn attrs_follow_selected_sidecar_and_frontmatter_without_inheritance() { + // Exercise every selected form, both with content and metadata-only. + // Lower-ranked files deliberately have unique attrs that must not leak. + for selected in ["page/meta.yaml", "page/index.meta.yaml", "page.meta.yaml"] { + for has_content in [false, true] { + let temp = create_test_dir(); + fs::create_dir(temp.path().join("page")).unwrap(); + let forms = ["page/meta.yaml", "page/index.meta.yaml", "page.meta.yaml"]; + let rank = forms.iter().position(|form| *form == selected).unwrap(); + for form in &forms[rank + 1..] { + fs::write(temp.path().join(form), "attrs: {unselected: true}\n").unwrap(); + } + fs::write( + temp.path().join(selected), + "title: Selected\nnamespace: parent\nattrs: {owner: sidecar, keep: true, nested: {old: true}, list: [old]}\n", + ).unwrap(); + if has_content { + fs::write(temp.path().join("page/index.md"), + "---\nattrs: {owner: null, nested: {new: true}, list: [new]}\n---\n# Body\n").unwrap(); + } + fs::write(temp.path().join("page/child.md"), "# Child\n").unwrap(); + let storage = FsStorage::new(temp.path().to_path_buf(), temp.path().to_path_buf()); + let first = storage.scan().unwrap(); + let second = storage.scan().unwrap(); + let page = first.iter().find(|d| d.path == "page").unwrap(); + let cached = second.iter().find(|d| d.path == "page").unwrap(); + assert!(std::sync::Arc::ptr_eq(&page.meta, &cached.meta)); + assert_eq!(page.has_content, has_content); + assert_eq!(page.meta.title, "Selected"); + let expected = if has_content { + serde_json::json!({"owner": null, "keep": true, "nested": {"new": true}, "list": ["new"]}) + } else { + serde_json::json!({"owner": "sidecar", "keep": true, "nested": {"old": true}, "list": ["old"]}) + }; + assert_eq!( + serde_json::to_value(&page.meta.attrs).unwrap(), + expected, + "{selected}" + ); + let child = first.iter().find(|d| d.path == "page/child").unwrap(); + assert!(child.meta.attrs.is_empty(), "{selected} must not inherit"); + assert_eq!(page.meta.namespace.as_deref(), Some("parent")); + } + } + } + #[test] fn test_sidecar_combines_metadata_and_content() { let temp_dir = create_test_dir(); fs::write(temp_dir.path().join("guide.md"), "# Original H1\n\nBody.").unwrap(); fs::write( temp_dir.path().join("guide.meta.yaml"), - "title: Sidecar Title\nkind: guide", + "title: Sidecar Title\nkind: guide\nattrs: {owner: named-leaf}", ) .unwrap(); @@ -829,6 +876,7 @@ mod tests { assert!(doc.has_content); assert_eq!(doc.meta.title, "Sidecar Title"); // sidecar wins over H1 assert_eq!(doc.meta.kind, Some("guide".to_owned())); + assert_eq!(doc.meta.attrs["owner"], "named-leaf"); // Content still served from the .md file. assert_eq!(storage.read("guide").unwrap(), "# Original H1\n\nBody."); @@ -940,7 +988,7 @@ mod tests { let temp_dir = create_test_dir(); fs::write( temp_dir.path().join("payments.meta.yaml"), - "kind: component\nnamespace: billing", + "kind: component\nnamespace: billing\nattrs: {owner: metadata-only-leaf}", ) .unwrap(); @@ -952,6 +1000,7 @@ mod tests { assert_eq!(doc.meta.title, "Payments"); // titlecased from url segment assert_eq!(doc.meta.kind, Some("component".to_owned())); assert_eq!(doc.meta.namespace, Some("billing".to_owned())); + assert_eq!(doc.meta.attrs["owner"], "metadata-only-leaf"); } #[test] @@ -2032,7 +2081,7 @@ mod tests { #[test] fn readme_fallback_frontmatter_is_preserved_and_shared_across_unchanged_scans() { let (_dir, _, storage) = create_readme_test_dir( - "---\ntitle: Readme Home\ndescription: Project docs\nkind: domain\nnamespace: docs\npages:\n - guide\n---\n# Body Title", + "---\ntitle: Readme Home\ndescription: Project docs\nkind: domain\nnamespace: docs\nattrs: {owner: project, nested: [null, true]}\npages:\n - guide\n---\n# Body Title", ); let first = storage .scan() @@ -2049,6 +2098,10 @@ mod tests { assert!(std::sync::Arc::ptr_eq(&first.meta, &second.meta)); assert_eq!(first.meta.title, "Readme Home"); + assert_eq!( + serde_json::to_value(&first.meta.attrs).unwrap(), + serde_json::json!({"owner": "project", "nested": [null, true]}) + ); assert_eq!(first.meta.description.as_deref(), Some("Project docs")); assert_eq!(first.meta.kind.as_deref(), Some("domain")); assert_eq!(first.meta.namespace.as_deref(), Some("docs")); diff --git a/crates/rw-storage-s3/Cargo.toml b/crates/rw-storage-s3/Cargo.toml index 66eaec53..73b41d5f 100644 --- a/crates/rw-storage-s3/Cargo.toml +++ b/crates/rw-storage-s3/Cargo.toml @@ -32,6 +32,7 @@ tracing = { workspace = true } [dev-dependencies] chrono = "0.4" pretty_assertions = { workspace = true } +rw-meta = { workspace = true } rw-storage = { workspace = true, features = ["mock"] } tempfile = { workspace = true } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/crates/rw-storage-s3/src/format.rs b/crates/rw-storage-s3/src/format.rs index 7443c9f1..a5fc66a2 100644 --- a/crates/rw-storage-s3/src/format.rs +++ b/crates/rw-storage-s3/src/format.rs @@ -144,6 +144,80 @@ mod tests { serde_json::from_value(wire).unwrap() } + #[test] + fn attrs_f64_bits_roundtrip_manifest_bytes() { + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = rw_meta::Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let mut document = + document_from_wire("guide", true, "Guide", None, None, None, None, None, false); + document.meta = std::sync::Arc::new(resolved.meta); + let manifest = Manifest::from(vec![document]); + let bytes = serde_json::to_vec(&manifest).unwrap(); + let decoded: Manifest = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.documents[0].meta.attrs["value"] + .as_f64() + .unwrap() + .to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn attrs_transport_depth_boundary_roundtrips_manifest_bytes() { + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + let mut expected = leaf_value; + for level in usize::from(leaf != "null")..123 { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + let resolved = rw_meta::Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {value}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!(resolved.meta.attrs["value"], expected, "{shape}/{leaf}"); + let mut document = + document_from_wire("guide", true, "Guide", None, None, None, None, None, false); + document.meta = std::sync::Arc::new(resolved.meta); + let manifest = Manifest::from(vec![document]); + let bytes = serde_json::to_vec(&manifest).unwrap(); + let decoded: Manifest = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.documents[0].meta.attrs["value"], expected, + "{shape}/{leaf}" + ); + assert_eq!(decoded, manifest, "{shape}/{leaf}"); + } + } + } + #[test] fn test_manifest_serialization_roundtrip() { let manifest = Manifest::from(vec![ @@ -175,6 +249,9 @@ mod tests { assert_eq!(serde_json::to_string(&old).unwrap(), LEGACY_MANIFEST_JSON); assert_eq!(old.documents[0].meta.name, None); + assert_eq!(FORMAT_VERSION, 1); + assert_eq!(old.version, 1); + assert!(old.documents[0].meta.attrs.is_empty()); assert_eq!(old.documents[0].meta.kind.as_deref(), Some("domain")); assert_eq!(old.documents[0].meta.namespace.as_deref(), Some("payments")); assert_eq!( @@ -228,6 +305,31 @@ mod tests { assert!(!old_reader.documents[0].is_dir); } + #[test] + fn attrs_are_additive_in_version_one_and_ignored_by_old_readers() { + let mut manifest: Manifest = serde_json::from_str(LEGACY_MANIFEST_JSON).unwrap(); + let attrs = + serde_json::json!({"owner": null, "nested": {"audiences": ["operators", true, 3]}}); + std::sync::Arc::make_mut(&mut manifest.documents[0].meta).attrs = + serde_json::from_value(attrs.clone()).unwrap(); + let json = serde_json::to_string(&manifest).unwrap(); + let wire: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(wire["version"], 1); + assert_eq!(FORMAT_VERSION, 1); + assert_eq!(wire["documents"][0]["attrs"], attrs); + let current: Manifest = serde_json::from_str(&json).unwrap(); + assert_eq!(current, manifest); + let old: LegacyManifest = serde_json::from_str(&json).unwrap(); + assert_eq!(serde_json::to_string(&old).unwrap(), LEGACY_MANIFEST_JSON); + std::sync::Arc::make_mut(&mut manifest.documents[0].meta) + .attrs + .clear(); + assert_eq!( + serde_json::to_string(&manifest).unwrap(), + LEGACY_MANIFEST_JSON + ); + } + #[test] fn declared_name_manifest_roundtrip_is_syntactic_not_legacy_semantic_parity() { let mut wire: serde_json::Value = serde_json::from_str(LEGACY_MANIFEST_JSON).unwrap(); diff --git a/crates/rw-storage-s3/src/publisher.rs b/crates/rw-storage-s3/src/publisher.rs index 79568002..bbaf0c86 100644 --- a/crates/rw-storage-s3/src/publisher.rs +++ b/crates/rw-storage-s3/src/publisher.rs @@ -352,6 +352,74 @@ A -> B assert!(!bundle.content.contains("System(")); } + #[tokio::test] + async fn publication_preserves_selected_canonical_attrs_not_raw_frontmatter() { + // The scan is authoritative: bundle content deliberately disagrees. + let document: Document = serde_json::from_value(serde_json::json!({ + "path": "selected", "title": "Selected", "has_content": true, + "attrs": {"owner": "canonical", "keep": true, "nested": [null, {"x": 1}]} + })) + .unwrap(); + let canonical = Arc::clone(&document.meta); + let virtual_document: Document = serde_json::from_value(serde_json::json!({ + "path": "virtual", "title": "Virtual", "has_content": false, + "attrs": {"owner": "metadata-only"} + })) + .unwrap(); + let storage = MockStorage::new() + .with_scanned_document(document) + .with_scanned_document(virtual_document) + .with_content( + "selected", + "---\nattrs: {owner: raw, unexpected: true}\n---\n# Body\n", + ); + let documents = storage.scan().unwrap(); + let mut bundles = Vec::new(); + let (count, warnings) = build_bundles(&storage, &documents, &[], async |key, json| { + bundles.push((key, json)); + Ok(()) + }) + .await + .unwrap(); + assert_eq!(count, 1); + assert!(warnings.is_empty()); + assert_eq!(bundles[0].0, "pages/selected.json"); + let bundle: serde_json::Value = serde_json::from_slice(&bundles[0].1).unwrap(); + assert_eq!(bundle.as_object().unwrap().len(), 1); + assert!(bundle["content"].as_str().unwrap().contains("owner: raw")); + let manifest = Manifest::from(documents); + let selected = manifest + .documents + .iter() + .find(|d| d.path == "selected") + .unwrap(); + assert!(Arc::ptr_eq(&selected.meta, &canonical)); + let reloaded: Manifest = + serde_json::from_slice(&serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert_eq!(reloaded.version, 1); + assert_eq!(reloaded, manifest); + assert_eq!( + reloaded + .documents + .iter() + .find(|d| d.path == "selected") + .unwrap() + .meta + .attrs, + canonical.attrs + ); + assert_eq!( + reloaded + .documents + .iter() + .find(|d| d.path == "virtual") + .unwrap() + .meta + .attrs["owner"], + "metadata-only" + ); + } + #[test] fn dedup_preserves_first_seen_order() { let input = [ diff --git a/crates/rw-storage/Cargo.toml b/crates/rw-storage/Cargo.toml index ba2d394f..138ade37 100644 --- a/crates/rw-storage/Cargo.toml +++ b/crates/rw-storage/Cargo.toml @@ -14,9 +14,7 @@ chrono = "0.4" parking_lot = { workspace = true, optional = true } rw-meta = { workspace = true } serde = { workspace = true } - -[dev-dependencies] -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["float_roundtrip"] } [features] default = [] diff --git a/crates/rw-storage/src/mock.rs b/crates/rw-storage/src/mock.rs index 52ab18bd..2495b91e 100644 --- a/crates/rw-storage/src/mock.rs +++ b/crates/rw-storage/src/mock.rs @@ -2,7 +2,7 @@ //! //! Provides [`MockStorage`] for unit testing without filesystem access. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc; @@ -37,6 +37,7 @@ fn meta( pages: Option>, ) -> Arc { Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.into(), description, diff --git a/crates/rw-storage/src/storage.rs b/crates/rw-storage/src/storage.rs index 4f58a405..8757538e 100644 --- a/crates/rw-storage/src/storage.rs +++ b/crates/rw-storage/src/storage.rs @@ -13,6 +13,7 @@ //! //! Storage implementations handle the mapping from URL paths to their internal storage format. +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; @@ -69,7 +70,7 @@ pub struct Document { } #[derive(Serialize, Deserialize)] -struct DocumentWire { +struct DocumentWire { path: S, title: S, has_content: bool, @@ -85,6 +86,8 @@ struct DocumentWire { origin: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pages: Option

, + #[serde(default, skip_serializing_if = "Option::is_none")] + attrs: Option, #[serde(default = "default_is_dir")] is_dir: bool, } @@ -94,7 +97,7 @@ impl Serialize for Document { where S: serde::Serializer, { - DocumentWire::<&str, &[String]> { + DocumentWire::<&str, &[String], &BTreeMap> { path: self.path.as_str(), title: self.meta.title.as_str(), has_content: self.has_content, @@ -104,6 +107,7 @@ impl Serialize for Document { description: self.meta.description.as_deref(), origin: self.origin.as_deref(), pages: self.meta.pages.as_deref(), + attrs: (!self.meta.attrs.is_empty()).then_some(&self.meta.attrs), is_dir: self.is_dir, } .serialize(serializer) @@ -115,12 +119,16 @@ impl<'de> Deserialize<'de> for Document { where D: serde::Deserializer<'de>, { - let wire = DocumentWire::>::deserialize(deserializer)?; + let wire = + DocumentWire::, BTreeMap>::deserialize( + deserializer, + )?; Ok(Self { path: wire.path, has_content: wire.has_content, meta: Arc::new(Meta { + attrs: wire.attrs.unwrap_or_default(), name: wire.name, title: wire.title, description: wire.description, @@ -507,6 +515,78 @@ mod tests { assert_eq!(converted.to_rfc3339(), "2025-07-08T18:40:00.500+00:00"); } + #[test] + fn attrs_f64_bits_roundtrip_document_bytes() { + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let document = Document { + path: "guide".into(), + has_content: true, + meta: Arc::new(resolved.meta), + origin: None, + is_dir: false, + diagnostics: Vec::new().into(), + }; + let bytes = serde_json::to_vec(&document).unwrap(); + let decoded: Document = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn document_attrs_round_trip_on_flat_wire() { + let wire = serde_json::json!({ + "path": "guide", "title": "Guide", "has_content": true, "is_dir": false, + "attrs": { + "owner": null, + "nested": {"enabled": true}, + "values": [false, "text", -42, u64::MAX, 1.5, {}, []] + } + }); + let document: Document = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&document).unwrap(), wire); + assert_eq!(document.meta.attrs["owner"], serde_json::Value::Null); + assert_eq!( + document.meta.attrs["nested"], + serde_json::json!({"enabled": true}) + ); + assert!(document.diagnostics.is_empty()); + } + + #[test] + fn document_empty_attrs_are_omitted_on_flat_wire() { + for attrs in [None, Some(serde_json::json!({}))] { + let expected = serde_json::json!({ + "path": "guide", "title": "Guide", "has_content": true, "is_dir": true + }); + let mut wire = expected.clone(); + if let Some(attrs) = attrs { + wire["attrs"] = attrs; + } + let document: Document = serde_json::from_value(wire).unwrap(); + assert!(document.meta.attrs.is_empty()); + assert_eq!(serde_json::to_value(document).unwrap(), expected); + } + } + #[test] fn document_retains_declared_name_on_wire() { let wire = serde_json::json!({"path":"guide", "title":"Guide", "has_content":true, "is_dir":true, "name":"payments-api"}); @@ -520,6 +600,7 @@ mod tests { path: String::new(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Home".to_owned(), description: None, @@ -544,6 +625,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: None, @@ -568,6 +650,7 @@ mod tests { path: "domain/billing".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Billing".to_owned(), description: None, @@ -591,6 +674,7 @@ mod tests { path: "domains".to_owned(), has_content: false, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Domains".to_owned(), description: None, @@ -773,6 +857,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), @@ -822,6 +907,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), @@ -842,6 +928,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: None, @@ -891,6 +978,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), diff --git a/docs/embedding.md b/docs/embedding.md index 6c39b150..3ae6203f 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -1,9 +1,9 @@ # Embedding -This page covers three concerns for embedding `@rwdocs/viewer` (or rendering pages +This page covers integration concerns for embedding `@rwdocs/viewer` (or rendering pages through `@rwdocs/core`) in a host application: pointing `@rwdocs/core` at a site -with `projectDir`, resolving cross-entity links via `resolveSectionRefs`, and -durable comment keys for hosts that store their own comments. +with `projectDir`, resolving cross-entity links via `resolveSectionRefs`, +durable comment keys for hosts that store their own comments, and page attrs. When you embed the viewer in a host application that stores **its own** comments — for example the Backstage plugin pair — each comment needs a stable @@ -23,6 +23,42 @@ directory, and it does not consult the Node process's working directory. A host that mounts several sites can therefore point each `createSite` call at its own directory without one site's configuration leaking into another. +## Consuming page attrs + +Only the NAPI/core page boundary exposes [page-local `attrs`](metadata.md#attrs-page-local-integration-data): + +```js +const page = await site.renderPage("guide"); +const owner = page.meta.attrs?.owner; +if (typeof owner === "string") { + // Validate against your integration's schema before using it. +} +``` + +The core declarations describe JSON values (including nested null), not `any`. +`meta.attrs` is omitted when empty, and never inherits. Use string identifiers +when exact large integers matter: JavaScript number precision applies. Rust JSON +byte roundtrips preserve the chosen supported finite `f64` values; they do not +remove JavaScript's numeric limits. Source validation bounds array/object +nesting so manifest/cache readers can load the values; see +[attrs validation](metadata.md#attrs-page-local-integration-data). +Attrs are ordinary data, including keys such as `__proto__`; do not treat them as +executable configuration or authorization rules. RW does not render or interpret them, and +the built-in HTTP API, viewer, search and navigation shapes are unchanged. + +For S3, deploy version-1-compatible readers with attrs support before consumers +rely on published attrs. Older readers ignore and do not preserve them on +reserialization. Attributes increase all-site manifest/structure-cache payloads +and resident snapshot memory; selected-page conversion adds response work, not +new S3 object requests. Measure your own payloads and Site reuse/eviction cadence: +local benchmarks do not establish external-host latency. + +Metadata on fresh and cached HTML responses comes from the applicable Site +snapshot. Keep the existing host refresh strategy; attrs add no polling, watcher +delivery, same-mtime or cross-process cache correctness guarantees. An attrs-only +snapshot change does not introduce whole-site HTML invalidation; normal source +mtime invalidation still applies. No new refresh API is provided. + ## `resolveSectionRefs` must map the site-root ref `mountRw({ resolveSectionRefs })` lets the viewer turn a cross-entity link (a diff --git a/docs/metadata.md b/docs/metadata.md index 8e998efa..ff77b2e7 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -40,8 +40,87 @@ These fields are available in both frontmatter and meta.yaml: - `kind` -- page kind (e.g., `domain`, `guide`). Pages with `kind` are registered as sections. - `name` -- page-local section/catalog and diagram identifier, overriding the path-derived name when `kind` is set (see below). - `namespace` -- Backstage catalog namespace for the section (see below). +- `attrs` -- opaque, page-local JSON-compatible attributes for integrations (see below) - `pages` -- ordered list of child page slugs for navigation sidebar ordering (directory-level only) +### `attrs`: page-local integration data + +Declare attributes in frontmatter or the selected sidecar, including on the +README homepage and metadata-only pages; `kind` is not required: + +```yaml +attrs: + owner: payments + audiences: [operators] + support: {url: https://example.org/support} + reviewDate: null +``` + +- **No inheritance.** Attributes belong only to the declaring page, not its + children. They do not change section identity, namespace, titles, or URLs. +- **Top-level overlay.** Start with the selected sidecar's attrs; frontmatter + overrides matching keys. A nested object or array replaces the previous value + **whole**, without recursive merging. `attrs: {}` contributes no overrides. +- **Null is data.** `reviewDate: null` keeps that key present and replaces any + sidecar value; it is different from an absent key, not a deletion instruction. + There is no reset/deletion syntax: remove a key from every source declaring it + to remove it entirely. +- **Whole-field validation.** Attrs must be a string-keyed map of JSON-compatible + strings, booleans, finite supported numbers, nulls, arrays, and objects. Types + are preserved, not string-coerced. Keys are case-sensitive opaque strings. + YAML custom tags, non-string object keys, and non-finite numbers are not + supported. Array/object nesting is bounded so values remain readable inside + manifest and structure-cache envelopes. Empty arrays/objects count toward + nesting; scalars do not. Deeply nested values can be valid JSON but exceed + RW's supported transport bound. + Any unsupported nested value discards that source's **entire attrs + field** with a source-attributed warning, retaining sibling metadata and + valid attrs from the other source. Whole-field `attrs: null` is invalid and + falls back to valid sidecar attrs. Malformed YAML and duplicate mappings + detected by the parser keep their existing source-level failure behavior. + +`@rwdocs/core` / NAPI `renderPage()` exposes resolved attributes as optional +`meta.attrs`, omitted when empty. Built-in HTTP page responses, viewer, +navigation, search, Confluence, and rendered HTML do not expose or interpret +attrs. Integrations supply their own schema and presentation; attrs are exposed +metadata, **not a secret store or an authorization policy**. + +Rust retains numbers within serde_json's supported range, including the exact +bits of the finite `f64` chosen during source resolution through JSON byte +serialization/decoding. This does not promise exact decimal arithmetic. +JavaScript receives ordinary numbers, not BigInt or a string-number codec. Use +**strings for numeric identifiers requiring exact large integers**, especially beyond +`Number.MAX_SAFE_INTEGER` (`9007199254740991`). + +Attrs travel in the existing all-site manifest and structure cache, increasing +bytes, decoding/serialization work, and snapshot memory. The selected page also +incurs native-to-JavaScript conversion cost. There are no extra attrs objects or +fetches, nor attrs-specific render-cache fingerprints or unrelated-page HTML +invalidation. Existing source mtime changes can still invalidate an edited page. +Freshness follows the existing Site snapshot/load lifecycle: this adds no +polling, watcher, same-mtime recovery, or cross-process refresh guarantees. +See [embedding guidance](embedding.md#consuming-page-attrs). + +#### Attrs compatibility and rollout + +The flattened Document/S3 wire adds optional `attrs`, omitted when empty; +empty-attrs serialization is unchanged. Manifest `FORMAT_VERSION` remains **1**. +New readers accept old manifests; old readers ignore attrs and **lose them if +reserializing documents**. Upgrade readers before integrations rely on attrs +from published bundles. There is no migration or automatic deployment; mixed +versions do not provide semantic parity. Rollback requires disabling integration +consumption and coordinating reader/publisher changes, not deleting stored data. + +Adding `attrs: BTreeMap` to public `rw_meta::Meta` is a +**breaking (pre-1.0) Rust struct-literal change**. Existing literals must add +`attrs: Default::default()` or use `Meta::resolve`. Document syntax and optional +NAPI output are additive; this does not add an HTTP field or a CLI command. +Directly constructed Rust `Meta.attrs` is not a validated wrapper: callers must +keep values within the same transport nesting bound before publication or caching. +`rw-storage` enables serde_json's `float_roundtrip` feature for byte decoding; +Cargo feature unification also affects other JSON float decoding in the same +binary, not only attrs. + ### Migrating legacy metadata Earlier versions accepted `type` as an alias for `kind`. Rename that key to @@ -102,8 +181,9 @@ or migrated automatically. Adding `name: Option` to public `rw_meta::Meta` is a **pre-1.0 Rust struct-literal source break**: existing literals must add `name: None` (or use -`Meta::resolve` to parse declarations). HTTP/NAPI/viewer page metadata shapes -are unchanged; their existing section projections carry the effective name. +`Meta::resolve` to parse declarations). For the `name` feature, HTTP/NAPI/viewer +page metadata shapes are unchanged; their existing section projections carry +the effective name. The flattened Document/S3 wire adds optional `name`, omitted when absent; no-name serialization is unchanged. S3 manifest `FORMAT_VERSION` stays **1**. @@ -166,8 +246,8 @@ docs/guides/meta.yaml: sidecar `pages`: expected a list, found a string Warnings log at WARN level, so run `rw serve --verbose` (or set `RUST_LOG=warn`) to see them — the default verbosity hides them. -Diagnostics never reach HTTP responses or published bundles. Unknown keys -remain silently ignored. +Diagnostics never reach public responses or published bundles. Unknown top-level +metadata keys remain silently ignored; keys inside `attrs` are retained as data. ## Navigation ordering @@ -204,8 +284,8 @@ The page title is resolved in this order: ## Inheritance Metadata does not inherit from parent directories: `title`, `description`, -`kind`, `name`, and `pages` apply only to the page or directory that declares them, not -to anything beneath it. `namespace` is the one exception — it inherits down +`kind`, `name`, `attrs`, and `pages` apply only to the page or directory that +declares them, not to anything beneath it. `namespace` is the one exception — it inherits down the tree, as described above. ## Named sidecar files (`.meta.yaml`) diff --git a/packages/core/index.d.ts b/packages/core/index.d.ts index 24a508ab..b1160713 100644 --- a/packages/core/index.d.ts +++ b/packages/core/index.d.ts @@ -1,5 +1,6 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue } export declare class RwSite { getNavigation(sectionRef?: string | undefined | null): Promise listSections(): Promise> @@ -161,6 +162,7 @@ export interface PageMetaResponse { lastModified: string description?: string kind?: string + attrs?: Record sectionRef: string /** * Page path relative to its section root. Stable across whole-section diff --git a/packages/core/package.json b/packages/core/package.json index 7be6d084..aa47b877 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -10,6 +10,7 @@ "types": "index.d.ts", "napi": { "binaryName": "core", + "dtsHeader": "/* auto-generated by NAPI-RS */\n/* eslint-disable */\nexport type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }\n", "targets": [ "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", diff --git a/packages/core/test/helpers/attrs-s3-fixture.mjs b/packages/core/test/helpers/attrs-s3-fixture.mjs new file mode 100644 index 00000000..6ebe4acf --- /dev/null +++ b/packages/core/test/helpers/attrs-s3-fixture.mjs @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createServer } from "node:http"; +import { crc32 } from "node:zlib"; + +// Only the objects/operations used by the attrs addon test, not an S3 emulator. +export async function attrsS3Fixture(t, attrs) { + const objects = new Map(); + const requests = []; + const failures = []; + const prefix = "/attrs-test/docs/"; + const cacheKeys = new Set([ + "cache/site/structure", + "cache/pages/selected", + "cache/pages/unrelated", + ]); + const put = (key, value, cacheEtag) => { + const body = Buffer.isBuffer(value) ? value : Buffer.from(JSON.stringify(value)); + objects.set(key, { + body, + etag: `"${createHash("sha256").update(body).digest("hex")}"`, + cacheEtag, + }); + }; + const setAttrs = (next) => { + put("manifest.json", { + version: 1, + documents: [ + { + path: "selected", + title: "Selected", + has_content: true, + is_dir: false, + ...(Object.keys(next).length ? { attrs: next } : {}), + }, + { path: "unrelated", title: "Unrelated", has_content: true, is_dir: false }, + { + path: "virtual", + title: "Virtual", + has_content: false, + is_dir: true, + ...(Object.keys(next).length ? { attrs: next } : {}), + }, + ], + mtimes: { selected: 1700000000, unrelated: 1700000000, virtual: 1700000000 }, + }); + }; + setAttrs(attrs); + put("pages/selected.json", { content: "# Selected\n\nFixed body.\n" }); + put("pages/unrelated.json", { content: "# Unrelated\n\nOther fixed body.\n" }); + + const server = createServer(async (req, res) => { + try { + const url = new URL(req.url, "http://127.0.0.1"); + assert.ok(url.pathname.startsWith(prefix), `unexpected path: ${req.url}`); + const key = url.pathname.slice(prefix.length); + requests.push(`${req.method} ${key}`); + const operation = { GET: "GetObject", HEAD: "HeadObject", PUT: "PutObject" }[req.method]; + assert.ok(operation, `unsupported method: ${req.method}`); + for (const [name, value] of url.searchParams) { + assert.equal(name, "x-id", `unsupported query: ${req.url}`); + assert.equal(value, operation); + } + assert.match(req.headers.authorization ?? "", /Credential=rw-attrs-dummy\//); + if (req.method === "PUT") { + assert.ok(cacheKeys.has(key), `unsupported PUT: ${key}`); + const body = await decodeUpload(req); + // Invalid framing must never masquerade as an ordinary cache miss. + assert.ok(JSON.parse(body.toString()), "cache upload must be decoded JSON"); + const cacheEtag = req.headers["x-amz-meta-cache-etag"]; + assert.equal(typeof cacheEtag, "string"); + assert.ok(cacheEtag.length > 0); + put(key, body, cacheEtag); + res.writeHead(200, { ETag: objects.get(key).etag, "Content-Length": 0 }); + res.end(); + return; + } + assert.ok(objects.has(key) || cacheKeys.has(key), `unsupported GET: ${key}`); + if (req.method === "HEAD") assert.equal(key, "manifest.json"); + const object = objects.get(key); + if (!object) { + const body = "NoSuchKeyFixture cache miss"; + res.writeHead(404, { + "Content-Type": "application/xml", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); + return; + } + res.writeHead(200, { + ETag: object.etag, + "Content-Type": "application/octet-stream", + "Content-Length": object.body.length, + ...(object.cacheEtag ? { "x-amz-meta-cache-etag": object.cacheEtag } : {}), + }); + res.end(req.method === "HEAD" ? undefined : object.body); + } catch (error) { + failures.push(error.stack); + res.writeHead(400, { "Content-Length": 0 }); + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(async () => { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + server.closeAllConnections(); + }); + assert.deepEqual(failures, [], "unsupported S3 request or malformed upload"); + }); + return { + config: { + s3: { + bucket: "attrs-test", + entity: "docs", + region: "us-east-1", + endpoint: `http://127.0.0.1:${server.address().port}`, + accessKeyId: "rw-attrs-dummy", + secretAccessKey: "rw-attrs-dummy-secret", + }, + }, + setAttrs, + objects, + takeRequests() { + assert.deepEqual(failures, [], "unsupported S3 request or malformed upload"); + return requests.splice(0); + }, + }; +} + +async function decodeUpload(req) { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const wire = Buffer.concat(chunks); + // Node removes HTTP transfer-encoding, but NOT AWS's content-encoding. + assert.equal(req.headers["x-amz-sdk-checksum-algorithm"], "CRC32"); + let body = wire; + let checksum = req.headers["x-amz-checksum-crc32"]; + if (req.headers["content-encoding"] === "aws-chunked") { + assert.equal(req.headers["x-amz-trailer"], "x-amz-checksum-crc32"); + const decoded = []; + let offset = 0; + while (true) { + const end = wire.indexOf("\r\n", offset); + assert.ok(end >= offset, "missing AWS chunk length"); + const size = wire.subarray(offset, end).toString(); + assert.match(size, /^[0-9a-f]+$/i, "unsupported signed AWS chunk"); + offset = end + 2; + const length = Number.parseInt(size, 16); + if (length === 0) break; + assert.ok(offset + length + 2 <= wire.length, "truncated AWS chunk"); + decoded.push(wire.subarray(offset, offset + length)); + offset += length; + assert.equal(wire.subarray(offset, offset + 2).toString(), "\r\n"); + offset += 2; + } + const trailer = wire.subarray(offset).toString(); + const match = /^x-amz-checksum-crc32:([A-Za-z0-9+/=]+)\r\n\r\n$/.exec(trailer); + assert.ok(match, `unsupported AWS trailer: ${JSON.stringify(trailer)}`); + checksum = match[1]; + body = Buffer.concat(decoded); + assert.equal(body.length, Number(req.headers["x-amz-decoded-content-length"])); + } else { + assert.equal(req.headers["content-encoding"], undefined); + } + const expected = Buffer.alloc(4); + expected.writeUInt32BE(crc32(body)); + assert.equal(checksum, expected.toString("base64"), "AWS CRC32 must match decoded bytes"); + return body; +} diff --git a/packages/core/test/render-page-attrs-s3.test.mjs b/packages/core/test/render-page-attrs-s3.test.mjs new file mode 100644 index 00000000..66cf93f5 --- /dev/null +++ b/packages/core/test/render-page-attrs-s3.test.mjs @@ -0,0 +1,130 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { attrsS3Fixture } from "./helpers/attrs-s3-fixture.mjs"; + +const require = createRequire(import.meta.url); +const { createSite } = require("../index.js"); + +function assertAttrs(page, attrs) { + if (Object.keys(attrs).length === 0) { + assert.equal(Object.hasOwn(page.meta, "attrs"), false); + } else { + assert.deepEqual(page.meta.attrs, attrs); + } +} + +const initialLoad = ["GET manifest.json", "PUT cache/site/structure"]; +const coldPage = (page) => [ + `GET cache/pages/${page}`, + `GET pages/${page}.json`, + `PUT cache/pages/${page}`, +]; + +for (const [label, attrs] of [ + ["empty", {}], + [ + "populated", + { owner: "old", nested: { audiences: ["operators"], nullable: null }, enabled: true }, + ], +]) { + test(`S3 ${label} attrs use the same exact cold/warm requests and retain hits across reload`, async (t) => { + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + assert.deepEqual(fixture.takeRequests(), []); + const first = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), [...initialLoad, ...coldPage("selected")]); + assertAttrs(first, attrs); + const warm = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assert.deepEqual(warm, first); + const unrelated = await site.renderPage("unrelated"); + assert.deepEqual(fixture.takeRequests(), coldPage("unrelated")); + assertAttrs(unrelated, {}); + assert.deepEqual(await site.renderPage("unrelated"), unrelated); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/unrelated"]); + assertAttrs(await site.renderPage("virtual"), attrs); + assert.deepEqual(fixture.takeRequests(), []); + + assert.equal(await site.reload(false), false); + assert.deepEqual(fixture.takeRequests(), ["HEAD manifest.json"]); + // Keep one Site and fixed render inputs to isolate attrs-only refresh. + for (const [force, next] of [ + [false, { owner: "new" }], + [true, { owner: "forced", values: [null, 2] }], + [false, {}], + ]) { + const selectedCache = fixture.objects.get("cache/pages/selected"); + const unrelatedCache = fixture.objects.get("cache/pages/unrelated"); + const priorManifestEtag = fixture.objects.get("manifest.json").etag; + fixture.setAttrs(next); + assert.notEqual(fixture.objects.get("manifest.json").etag, priorManifestEtag); + assert.equal(await site.reload(force), true); + assert.deepEqual(fixture.takeRequests(), [ + ...(force ? [] : ["HEAD manifest.json"]), + "GET cache/site/structure", + "GET manifest.json", + "PUT cache/site/structure", + ]); + const refreshed = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assertAttrs(refreshed, next); + assert.equal(refreshed.content, first.content); + assert.deepEqual( + { ...refreshed, meta: { ...refreshed.meta, attrs: undefined } }, + { ...first, meta: { ...first.meta, attrs: undefined } }, + ); + assert.deepEqual(await site.renderPage("unrelated"), unrelated); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/unrelated"]); + assertAttrs(await site.renderPage("virtual"), next); + assert.deepEqual(fixture.takeRequests(), []); + assert.equal(fixture.objects.get("cache/pages/selected"), selectedCache); + assert.equal(fixture.objects.get("cache/pages/unrelated"), unrelatedCache); + assert.equal(await site.reload(false), false); + assert.deepEqual(fixture.takeRequests(), ["HEAD manifest.json"]); + } + }); + + test(`S3 ${label} metadata-only attrs need no page bundle or render-cache request even cold`, async (t) => { + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + const first = await site.renderPage("virtual"); + assert.deepEqual(fixture.takeRequests(), initialLoad); + assertAttrs(first, attrs); + assert.equal(first.content, "

Virtual

\n"); + assert.deepEqual(await site.renderPage("virtual"), first); + assert.deepEqual(fixture.takeRequests(), []); + }); +} + +// Catch extra Rust byte-decoder rounding, not ordinary JS large-integer limits. +test("finite attrs fractions agree across filesystem, S3 and structure-cache bytes", async (t) => { + const attrs = { value: 51.248178375505404, small: 0.1, negative: -0.125 }; + const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "rw-core-attrs-f64-")); + t.after(() => fs.rmSync(projectDir, { recursive: true, force: true })); + fs.writeFileSync(path.join(projectDir, "rw.toml"), ""); + fs.mkdirSync(path.join(projectDir, "docs")); + fs.writeFileSync( + path.join(projectDir, "docs", "selected.md"), + "---\nattrs: {value: 51.248178375505404, small: 0.1, negative: -0.125}\n---\n# Selected\n", + ); + assertAttrs(await createSite({ projectDir }).renderPage("selected"), attrs); + + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + assertAttrs(await site.renderPage("selected"), attrs); + assert.deepEqual(fixture.takeRequests(), [...initialLoad, ...coldPage("selected")]); + + // Change only the cache token so reload decodes the original Rust-produced float bytes. + const structure = fixture.objects.get("cache/site/structure"); + assert.equal(structure.cacheEtag, "0"); + structure.cacheEtag = "1"; + assert.equal(await site.reload(true), true); + assert.deepEqual(fixture.takeRequests(), ["GET cache/site/structure"]); + assertAttrs(await site.renderPage("selected"), attrs); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assert.equal(fixture.objects.get("cache/site/structure"), structure); +}); diff --git a/packages/core/test/render-page-metadata.test.mjs b/packages/core/test/render-page-metadata.test.mjs index 52c8e97d..3be1bbe4 100644 --- a/packages/core/test/render-page-metadata.test.mjs +++ b/packages/core/test/render-page-metadata.test.mjs @@ -74,3 +74,105 @@ test("renderPage omits undeclared and internal metadata", async (t) => { assert.deepEqual(Object.keys(page).sort(), responseKeys); assert.deepEqual(Object.keys(page.meta).sort(), requiredMetaKeys); }); + +// Losing the selected page's canonical attrs, using assignment for object keys, +// or converting unsigned JSON numbers to strings must fail at the JS boundary. +const attrsYaml = [ + "attrs:", + " audiences: [operators]", + " owner: null", + " large: 18446744073709551615", + " signed: -9223372036854775808", + " fraction: 1.25", + " enabled: true", + " disabled: false", + ' "__proto__": {marker: root}', + " constructor: root-constructor", + " prototype: root-prototype", + ' "key\\u0000tail": kept', + " recursive:", + ' - [null, false, 3.5, "text\\u0000tail"]', + ' - "__proto__": {marker: nested}', + " constructor: nested-constructor", + " prototype: nested-prototype", + ' "key\\u0000tail": nested-kept', + " - {}", + " - []", +].join("\n"); + +for (const source of ["frontmatter", "metadata-only"]) { + test(`renderPage exposes recursive attrs from ${source} as ordinary JS values`, async (t) => { + const projectDir = project(t); + const dir = path.join(projectDir, "docs", "selected"); + fs.mkdirSync(dir); + const metadata = `title: Selected\nkind: domain\nname: selected-page\n${attrsYaml}\n`; + if (source === "frontmatter") { + fs.writeFileSync(path.join(dir, "index.md"), `---\n${metadata}---\n# Body\n`); + } else { + fs.writeFileSync(path.join(dir, "meta.yaml"), metadata); + } + + const page = await createSite({ projectDir }).renderPage("selected"); + + assert.ok(page.meta.attrs, "nonempty canonical attrs must be exposed"); + assert.deepEqual(page.meta.attrs.audiences, ["operators"]); + assert.equal(page.meta.attrs.owner, null); + assert.equal(typeof page.meta.attrs.large, "number"); + assert.equal(page.meta.attrs.large, Number("18446744073709551615")); + assert.equal(typeof page.meta.attrs.signed, "number"); + assert.equal(page.meta.attrs.signed, Number("-9223372036854775808")); + assert.equal(page.meta.attrs.fraction, 1.25); + assert.equal(page.meta.attrs.enabled, true); + assert.equal(page.meta.attrs.disabled, false); + assert.equal(Object.hasOwn(page.meta.attrs, "__proto__"), true); + assert.equal(Object.getPrototypeOf(page.meta.attrs), Object.prototype); + assert.deepEqual(page.meta.attrs.__proto__, { marker: "root" }); + assert.equal(page.meta.attrs["key\u0000tail"], "kept"); + assert.deepEqual(page.meta.attrs.recursive[0], [null, false, 3.5, "text\u0000tail"]); + const nested = page.meta.attrs.recursive[1]; + assert.equal(Object.hasOwn(nested, "__proto__"), true); + assert.equal(Object.getPrototypeOf(nested), Object.prototype); + assert.deepEqual(nested.__proto__, { marker: "nested" }); + assert.equal(nested["key\u0000tail"], "nested-kept"); + for (const [object, prefix] of [ + [page.meta.attrs, "root"], + [nested, "nested"], + ]) { + for (const key of ["constructor", "prototype"]) { + assert.equal(Object.hasOwn(object, key), true); + assert.equal(object[key], `${prefix}-${key}`); + } + const descriptor = Object.getOwnPropertyDescriptor(object, "__proto__"); + assert.equal(descriptor.writable, true); + assert.equal(descriptor.enumerable, true); + assert.equal(descriptor.configurable, true); + } + assert.deepEqual(page.meta.attrs.recursive[2], {}); + assert.equal(Object.getPrototypeOf(page.meta.attrs.recursive[2]), Object.prototype); + assert.deepEqual(page.meta.attrs.recursive[3], []); + assert.equal(page.meta.title, "Selected"); + assert.equal(page.meta.path, "/selected"); + assert.equal(page.meta.sourceFile, source === "frontmatter" ? "selected" : ""); + assert.equal(page.meta.sectionRef, "domain:default/selected-page"); + assert.equal(page.meta.subpath, ""); + assert.deepEqual(Object.keys(page).sort(), responseKeys); + assert.deepEqual(Object.keys(page.meta).sort(), ["attrs", "kind", ...requiredMetaKeys]); + }); +} + +test("renderPage omits empty attrs on filesystem and metadata-only pages", async (t) => { + const projectDir = project(t); + // Parent attrs must not leak into either empty child. + fs.writeFileSync( + path.join(projectDir, "docs", "index.md"), + "---\nattrs: {parent: true}\n---\nHome\n", + ); + fs.writeFileSync(path.join(projectDir, "docs", "empty.md"), "---\nattrs: {}\n---\nEmpty\n"); + fs.mkdirSync(path.join(projectDir, "docs", "virtual")); + fs.writeFileSync(path.join(projectDir, "docs", "virtual", "meta.yaml"), "attrs: {}\n"); + const site = createSite({ projectDir }); + for (const pagePath of ["empty", "virtual"]) { + const emptyPage = await site.renderPage(pagePath); + assert.equal(Object.hasOwn(emptyPage.meta, "attrs"), false); + } +});