diff --git a/CHANGELOG.md b/CHANGELOG.md index 586efa61..9a0ca6f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **Page-local attrs for integrations** — JSON-compatible `attrs` in frontmatter and selected YAML sidecars survive S3 publication and appear as optional `meta.attrs` in NAPI/core `renderPage()` responses. Frontmatter overlays top-level keys; nested values replace whole, null remains data, and attrs never inherit. Empty attrs are omitted. Authored nesting is bounded for manifest/cache transport; Rust JSON byte roundtrips preserve supported finite `f64` values. No built-in HTTP/viewer or rendering semantics change. See [Page Metadata](docs/metadata.md#attrs-page-local-integration-data). + +### Changed + +- **Breaking (pre-1.0, Rust source):** Public `rw_meta::Meta` literals must add `attrs: Default::default()` (or use `Meta::resolve`). S3 manifests remain version 1; upgrade readers before integrations rely on attrs, since older readers ignore and lose them on reserialization. Populated attrs increase manifest/cache payloads and selected-page conversion work. JavaScript numeric identifiers requiring exact large integers should be strings. + ## [0.1.36] - 2026-09-07 ### New Features diff --git a/CLAUDE.md b/CLAUDE.md index 7f2d34cc..ac7ee282 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,6 +62,16 @@ uses effective `Section.name`; explicitly named, kind-declaring roots opt in, with eligibility included in the existing resolution fingerprint. Storage keeps the flattened wire with optional, omitted-when-absent `name`; S3 stays version 1, requiring readers to upgrade before publishers enable explicit names. +`Meta.attrs` is page-local JSON data: selected sidecar plus top-level frontmatter +overlay, whole nested replacement, literal null, no inheritance. Invalid source +attrs are discarded as a whole field. The same Arc carries attrs through cached +and virtual renders; flattened Document/structure/S3 wires omit empty attrs. +Only NAPI/core `renderPage().meta.attrs` exposes them (omitted empty), with normal +JS number precision. HTTP/viewer/search/navigation and renderer semantics stay +unchanged. Attrs add neither fetch paths nor render fingerprints nor refresh +guarantees; future renderer consumption must extend actual cache dependencies. +Upgrade S3 readers before consumers rely on attrs; old readers discard them on +reserialization. Public Rust Meta literals need `attrs: Default::default()`. ## Key Technical Details diff --git a/Cargo.lock b/Cargo.lock index 2f3e90e5..4eeec70f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4468,6 +4468,7 @@ version = "0.1.36" dependencies = [ "pulldown-cmark", "rw-sections", + "serde_json", "serde_yaml", ] @@ -4481,11 +4482,13 @@ dependencies = [ "rw-cache", "rw-cache-s3", "rw-config", + "rw-meta", "rw-renderer", "rw-site", "rw-storage", "rw-storage-fs", "rw-storage-s3", + "serde_json", "tokio", ] @@ -4634,6 +4637,7 @@ dependencies = [ "chrono", "parking_lot", "pretty_assertions", + "rw-meta", "rw-parser", "rw-plantuml", "rw-storage", diff --git a/README.md b/README.md index 490e7df5..63af13bd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Publish the same markdown to Confluence pages or embed in Backstage with native - **Status badges** — inline colored pill labels with Confluence status-macro parity - **GitHub-style alerts** — `[!NOTE]`, `[!TIP]`, `[!WARNING]`, and more - **Navigation and TOC** — automatic sidebar, breadcrumbs, and table of contents -- **Page metadata** — YAML frontmatter or sidecar files for titles, descriptions, navigation order, and explicit section/catalog and diagram names independent of URLs +- **Page metadata** — YAML frontmatter or sidecar files for titles, descriptions, navigation order, explicit section/catalog and diagram names independent of URLs, and page-local integration attrs exposed by NAPI/core - **Confluence rendering** — produce publish-ready bundles (XHTML + diagrams) for any Confluence publishing tool - **Backstage integration** — embed docs with native Backstage plugins diff --git a/crates/rw-meta/Cargo.toml b/crates/rw-meta/Cargo.toml index 1eabaa08..28c93b84 100644 --- a/crates/rw-meta/Cargo.toml +++ b/crates/rw-meta/Cargo.toml @@ -13,3 +13,4 @@ workspace = true pulldown-cmark = { workspace = true } rw-sections = { workspace = true } serde_yaml = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/rw-meta/src/attrs.rs b/crates/rw-meta/src/attrs.rs new file mode 100644 index 00000000..4e90162b --- /dev/null +++ b/crates/rw-meta/src/attrs.rs @@ -0,0 +1,66 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; +use serde_yaml::Value as Yaml; + +// Leave room in serde_json's nesting budget for the surrounding wire envelope: +// root, document collection, document, and attrs map. +const MAX_VALUE_NESTING_DEPTH: usize = 127 - 4; + +pub(crate) fn from_yaml(value: &Yaml) -> Result, String> { + let Yaml::Mapping(values) = value else { + return Err("expected an attrs mapping".to_owned()); + }; + values + .iter() + .map(|entry| to_json_entry(entry, MAX_VALUE_NESTING_DEPTH)) + .collect() +} + +fn to_json_entry( + (key, value): (&Yaml, &Yaml), + remaining_containers: usize, +) -> Result<(String, Json), String> { + let Yaml::String(key) = key else { + return Err("expected string object keys".to_owned()); + }; + Ok((key.clone(), to_json(value, remaining_containers)?)) +} + +fn to_json(value: &Yaml, remaining_containers: usize) -> Result { + let remaining_containers = if matches!(value, Yaml::Sequence(_) | Yaml::Mapping(_)) { + remaining_containers + .checked_sub(1) + .ok_or_else(|| "attrs nesting exceeds the supported transport depth".to_owned())? + } else { + remaining_containers + }; + match value { + Yaml::Null => Ok(Json::Null), + Yaml::Bool(value) => Ok(Json::Bool(*value)), + Yaml::String(value) => Ok(Json::String(value.clone())), + Yaml::Number(value) => { + let number = if let Some(value) = value.as_i64() { + Some(serde_json::Number::from(value)) + } else if let Some(value) = value.as_u64() { + Some(serde_json::Number::from(value)) + } else { + value.as_f64().and_then(serde_json::Number::from_f64) + }; + number + .map(Json::Number) + .ok_or_else(|| "expected a finite JSON number".to_owned()) + } + Yaml::Sequence(values) => values + .iter() + .map(|value| to_json(value, remaining_containers)) + .collect::, _>>() + .map(Json::Array), + Yaml::Mapping(values) => values + .iter() + .map(|entry| to_json_entry(entry, remaining_containers)) + .collect::, String>>() + .map(Json::Object), + Yaml::Tagged(_) => Err("custom YAML tags are not supported in attrs".to_owned()), + } +} diff --git a/crates/rw-meta/src/fields.rs b/crates/rw-meta/src/fields.rs index e9f8a111..4e8f092f 100644 --- a/crates/rw-meta/src/fields.rs +++ b/crates/rw-meta/src/fields.rs @@ -1,4 +1,7 @@ -use std::{borrow::Cow, collections::HashSet}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashSet}, +}; use serde_yaml::{Mapping, Value}; @@ -12,13 +15,14 @@ pub(crate) struct MetaFields { pub description: Option, pub pages: Option>, pub name: Option, + pub attrs: BTreeMap, } impl MetaFields { /// Extract fields from one YAML source; a failing field drops only itself, /// while a source that fails to parse (invalid YAML, non-mapping root) /// contributes nothing and yields one `Severity::Error` diagnostic. - /// Extraction order is fixed (kind, namespace, title, description, pages, name) + /// Extraction order is fixed (kind, namespace, title, description, pages, name, attrs) /// so diagnostics come back deterministic. pub(crate) fn from_yaml_with_diagnostics( yaml: &str, @@ -70,11 +74,12 @@ impl MetaFields { description: string_field(&mapping, "description", source, &mut diagnostics), pages: pages_field(&mapping, source, &mut diagnostics), name: name_field(&mapping, source, &mut diagnostics), + attrs: attrs_field(&mapping, source, &mut diagnostics), }; (fields, diagnostics) } - /// Merge `other` onto self. `other` fields win when Some. + /// Merge `other` onto self. Optional fields win when Some; attrs overlay by key. pub(crate) fn merge(mut self, other: Self) -> Self { self.kind = other.kind.or(self.kind); self.namespace = other.namespace.or(self.namespace); @@ -82,11 +87,42 @@ impl MetaFields { self.description = other.description.or(self.description); self.pages = other.pages.or(self.pages); self.name = other.name.or(self.name); + self.attrs.extend(other.attrs); self } } -const KNOWN_KEYS: [&str; 6] = ["kind", "namespace", "title", "description", "pages", "name"]; +const KNOWN_KEYS: [&str; 7] = [ + "kind", + "namespace", + "title", + "description", + "pages", + "name", + "attrs", +]; + +fn attrs_field( + mapping: &Mapping, + source: DiagnosticSource, + diagnostics: &mut Vec, +) -> BTreeMap { + let Some(value) = mapping.get("attrs") else { + return BTreeMap::new(); + }; + match crate::attrs::from_yaml(value) { + Ok(attrs) => attrs, + Err(message) => { + diagnostics.push(Diagnostic { + source, + field: Some("attrs".to_owned()), + severity: Severity::Warning, + message, + }); + BTreeMap::new() + } + } +} fn normalize_known_keys(mapping: &Mapping) -> Result, &'static str> { let mut seen = HashSet::new(); diff --git a/crates/rw-meta/src/lib.rs b/crates/rw-meta/src/lib.rs index f9e28b6c..50abbaeb 100644 --- a/crates/rw-meta/src/lib.rs +++ b/crates/rw-meta/src/lib.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +mod attrs; mod diagnostic; mod fields; mod head; @@ -8,7 +11,7 @@ use head::Head; /// Resolved page metadata from all sources. /// -/// Rust struct literals must include `name: None` when no name is declared; +/// Rust struct literals must include `name: None` and empty `attrs` when absent; /// prefer [`Meta::resolve`] when constructing metadata from document sources. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Meta { @@ -27,6 +30,12 @@ pub struct Meta { /// Declared page-local name; never inherited. Overrides section identity only /// when this page declares `kind`; otherwise retained but not effective. pub name: Option, + /// Page-local JSON data; never inherited. + /// + /// Source resolution bounds array/object nesting for manifest/cache transport. + /// Callers constructing attrs directly must also respect the wire readers' + /// nesting limits. + pub attrs: BTreeMap, } /// Resolution result: canonical fields plus every recoverable problem found @@ -46,7 +55,8 @@ impl Meta { /// Internally: /// 1. Parses meta.yaml into base fields /// 2. Extracts frontmatter and first H1 from markdown via pulldown-cmark - /// 3. Merges frontmatter over meta.yaml (frontmatter wins per field) + /// 3. Overlays valid frontmatter fields onto meta.yaml; attrs merge by + /// top-level key, while other supplied fields replace their sidecar values /// 4. Resolves title: frontmatter title, else `meta.yaml` title, else H1, /// else titlecased filename stem, else stem verbatim, else `"Untitled"` #[must_use] @@ -99,6 +109,7 @@ impl Meta { description: merged.description, pages: merged.pages, name: merged.name, + attrs: merged.attrs, }, diagnostics, } @@ -146,6 +157,272 @@ fn titlecase_from_slug(slug: &str) -> String { mod tests { use super::*; + #[test] + fn attrs_shallow_overlay_preserves_null_and_sidecar_keys() { + let resolved = Meta::resolve_with_diagnostics( + Some("---\nattrs:\n owner: null\n nested: {new: true}\n tags: [new]\n---\n# Page"), + Some("attrs:\n owner: old\n kept: 42\n nested: {old: true}\n tags: [old]"), + "page.md", + ); + assert_eq!(resolved.meta.attrs["owner"], serde_json::Value::Null); + assert_eq!(resolved.meta.attrs["kept"], serde_json::json!(42)); + assert_eq!( + resolved.meta.attrs["nested"], + serde_json::json!({"new": true}) + ); + assert_eq!(resolved.meta.attrs["tags"], serde_json::json!(["new"])); + assert!(resolved.diagnostics.is_empty()); + } + + #[test] + fn attrs_preserve_json_types_and_deterministic_keys() { + let resolved = Meta::resolve_with_diagnostics( + None, + Some( + "attrs:\n z: [null, true, false, text, -9223372036854775808, 18446744073709551615, 1.5, {}, []]\n a: {z: 1, a: {z: 2, a: 3}}\n text: '42'", + ), + "page.md", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved + .meta + .attrs + .keys() + .map(String::as_str) + .collect::>(), + ["a", "text", "z"] + ); + assert_eq!(resolved.meta.attrs["text"], serde_json::json!("42")); + assert_eq!( + resolved.meta.attrs["z"], + serde_json::json!([null, true, false, "text", i64::MIN, u64::MAX, 1.5, {}, []]) + ); + assert_eq!( + serde_json::to_string(&resolved.meta.attrs["a"]).unwrap(), + r#"{"a":{"a":3,"z":2},"z":1}"# + ); + } + + #[test] + fn attrs_absent_and_empty_add_no_overrides() { + for yaml in ["title: Kept", "attrs: {}"] { + let markdown = format!("---\n{yaml}\n---"); + let empty = Meta::resolve_with_diagnostics(Some(&markdown), None, "p"); + assert!(empty.meta.attrs.is_empty()); + assert!(empty.diagnostics.is_empty()); + let base = + Meta::resolve_with_diagnostics(Some(&markdown), Some("attrs: {kept: true}"), "p"); + assert_eq!(base.meta.attrs["kept"], serde_json::json!(true)); + assert!(base.diagnostics.is_empty()); + let sidecar = Meta::resolve_with_diagnostics(None, Some(yaml), "p"); + assert!(sidecar.meta.attrs.is_empty()); + assert!(sidecar.diagnostics.is_empty()); + } + assert!(Meta::resolve(None, None, "p").attrs.is_empty()); + } + + #[test] + fn attrs_transport_depth_preserves_field_and_source_recovery() { + // Both container kinds count, even when the deepest container is empty. + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + for depth in [123, 124, 125, 126, 127] { + let mut expected = leaf_value.clone(); + let leaf_depth = usize::from(leaf != "null"); + for level in leaf_depth..depth { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let fields = + format!("title: Kept\nattrs: {{kept: changed, value: {value}}}"); + let markdown = format!("---\n{fields}\n---"); + let fallback = "attrs: {kept: fallback}\ntitle: Fallback"; + let fallback_markdown = format!("---\n{fallback}\n---"); + let result = match source { + DiagnosticSource::Sidecar => Meta::resolve_with_diagnostics( + Some(&fallback_markdown), + Some(&fields), + "p", + ), + DiagnosticSource::Frontmatter => { + Meta::resolve_with_diagnostics(Some(&markdown), Some(fallback), "p") + } + }; + let context = format!("{shape}/{leaf}/{depth}/{source:?}"); + if depth == 123 { + assert!(result.diagnostics.is_empty(), "{context}"); + assert_eq!(result.meta.attrs["value"], expected, "{context}"); + } else { + assert_eq!(result.meta.attrs.len(), 1, "{context}"); + assert_eq!( + result.meta.attrs["kept"], + serde_json::json!("fallback"), + "{context}" + ); + assert_eq!(result.diagnostics.len(), 1, "{context}"); + let diagnostic = &result.diagnostics[0]; + assert_eq!(diagnostic.source, source, "{context}"); + if depth == 127 { + // The existing YAML parser rejects the entire source first. + assert_eq!(diagnostic.field, None, "{context}"); + assert_eq!(diagnostic.severity, Severity::Error, "{context}"); + assert_eq!(result.meta.title, "Fallback", "{context}"); + } else { + assert_eq!(diagnostic.field.as_deref(), Some("attrs"), "{context}"); + assert_eq!(diagnostic.severity, Severity::Warning, "{context}"); + } + } + if depth < 127 && source == DiagnosticSource::Frontmatter { + assert_eq!(result.meta.title, "Kept", "{context}"); + } + } + // With no overlay, a valid sibling in the sidecar also survives. + if (124..=126).contains(&depth) { + let fields = format!("title: Kept\nattrs: {{value: {value}}}"); + let result = Meta::resolve_with_diagnostics(None, Some(&fields), "p"); + assert_eq!(result.meta.title, "Kept"); + assert!(result.meta.attrs.is_empty()); + } + } + } + } + } + + #[test] + fn attrs_invalid_values_drop_whole_field_per_source() { + for yaml in [ + "text", + "42", + "true", + "[]", + "null", + "{kept: changed, 1: value}", + "{kept: changed, !custom key: value}", + "{kept: changed, bad: {1: value}}", + "{kept: changed, bad: {true: value}}", + "{kept: changed, bad: {null: value}}", + "{kept: changed, bad: {[a]: value}}", + "{kept: changed, bad: { !custom key: value }}", + "{kept: changed, bad: .nan}", + "{kept: changed, bad: .inf}", + "{kept: changed, bad: -.inf}", + "!custom scalar", + "!custom {key: value}", + "{kept: changed, bad: !custom scalar}", + "{kept: changed, bad: !custom {key: value}}", + "{kept: changed, bad: [!custom scalar]}", + "{kept: changed, bad: [1, {2: value}]}", + ] { + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let fields = format!("attrs: {yaml}\ntitle: Kept"); + let markdown = format!("---\n{fields}\n---"); + let result = match source { + DiagnosticSource::Sidecar => { + Meta::resolve_with_diagnostics(None, Some(&fields), "p") + } + DiagnosticSource::Frontmatter => Meta::resolve_with_diagnostics( + Some(&markdown), + Some("attrs: {kept: fallback}"), + "p", + ), + }; + assert_eq!(result.meta.title, "Kept", "{source:?}: {yaml}"); + if source == DiagnosticSource::Sidecar { + assert!(result.meta.attrs.is_empty(), "{yaml}"); + } else { + assert_eq!(result.meta.attrs.len(), 1, "{yaml}"); + assert_eq!( + result.meta.attrs["kept"], + serde_json::json!("fallback"), + "{yaml}" + ); + } + assert_eq!(result.diagnostics.len(), 1, "{source:?}: {yaml}"); + assert_eq!( + result.diagnostics[0].field.as_deref(), + Some("attrs"), + "{source:?}: {yaml}" + ); + assert_eq!(result.diagnostics[0].source, source, "{yaml}"); + assert_eq!(result.diagnostics[0].severity, Severity::Warning, "{yaml}"); + } + } + } + + #[test] + fn attrs_tagged_known_key_and_duplicate_source_recovery() { + let result = Meta::resolve_with_diagnostics( + None, + Some("? !custom attrs\n: {kept: true}\ntitle: !custom 42"), + "p", + ); + assert_eq!(result.meta.attrs["kept"], serde_json::json!(true)); + assert_eq!(result.meta.title, "42"); + assert!(result.diagnostics.is_empty()); + for yaml in [ + "attrs: {one: 1}\nattrs: {two: 2}\ntitle: Dropped", + "attrs: {one: 1}\n? !custom attrs\n: {two: 2}\ntitle: Dropped", + ] { + for source in [DiagnosticSource::Sidecar, DiagnosticSource::Frontmatter] { + let markdown = format!("---\n{yaml}\n---"); + let result = match source { + DiagnosticSource::Sidecar => { + Meta::resolve_with_diagnostics(None, Some(yaml), "p") + } + DiagnosticSource::Frontmatter => Meta::resolve_with_diagnostics( + Some(&markdown), + Some("attrs: {kept: true}\ntitle: Fallback"), + "p", + ), + }; + if source == DiagnosticSource::Sidecar { + assert!(result.meta.attrs.is_empty()); + assert_eq!(result.meta.title, "P"); + } else { + assert_eq!(result.meta.attrs.len(), 1); + assert_eq!(result.meta.attrs["kept"], serde_json::json!(true)); + assert_eq!(result.meta.title, "Fallback"); + } + assert_eq!(result.diagnostics.len(), 1); + assert_eq!(result.diagnostics[0].field, None); + assert_eq!(result.diagnostics[0].source, source); + assert_eq!(result.diagnostics[0].severity, Severity::Error); + } + } + } + + #[test] + fn attrs_diagnostics_follow_name_and_source_order() { + let result = Meta::resolve_with_diagnostics( + Some("---\nattrs: []\nname: []\n---"), + Some("attrs: []\nname: []"), + "p", + ); + assert_eq!( + result + .diagnostics + .iter() + .map(|d| (d.source, d.field.as_deref())) + .collect::>(), + [ + (DiagnosticSource::Sidecar, Some("name")), + (DiagnosticSource::Sidecar, Some("attrs")), + (DiagnosticSource::Frontmatter, Some("name")), + (DiagnosticSource::Frontmatter, Some("attrs")), + ] + ); + } + #[test] fn declared_name_scalar_and_identifier_boundaries() { for (yaml, expected) in [ diff --git a/crates/rw-napi/Cargo.toml b/crates/rw-napi/Cargo.toml index ac05812e..73a9f48f 100644 --- a/crates/rw-napi/Cargo.toml +++ b/crates/rw-napi/Cargo.toml @@ -19,12 +19,15 @@ napi-derive = "3" rw-cache = { workspace = true } rw-cache-s3 = { workspace = true } rw-config = { workspace = true } +rw-meta = { workspace = true } rw-renderer = { workspace = true } rw-site = { workspace = true } rw-storage = { workspace = true } rw-storage-fs = { workspace = true } rw-storage-s3 = { workspace = true } +serde_json = { workspace = true } + tokio = { version = "1", features = ["rt", "rt-multi-thread"] } [build-dependencies] diff --git a/crates/rw-napi/src/attrs.rs b/crates/rw-napi/src/attrs.rs new file mode 100644 index 00000000..7e7f059e --- /dev/null +++ b/crates/rw-napi/src/attrs.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; + +use napi::bindgen_prelude::{JsObjectValue, Null, Object, ToNapiValue}; +use napi::{Env, JsValue, Property, Result, Unknown, sys}; +use serde_json::Value; + +/// Output-only binding adapter retaining the selected page's canonical metadata. +/// JSON traversal is deferred until napi converts the response on the JS thread. +pub struct Attrs(pub(crate) Arc); + +impl ToNapiValue for Attrs { + /// Converts attrs to a JavaScript object with ordinary data properties. + /// + /// # Errors + /// Returns an error if a number cannot convert to `f64`, an array exceeds + /// `u32::MAX` elements, or a Node-API operation fails. + /// + /// # Safety + /// `env` must be valid on its JavaScript thread with an active handle scope. + unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { + let env = Env::from_raw(env); + Ok(object(&env, val.0.attrs.iter())?.raw()) + } +} + +fn object<'env, 'value>( + env: &'env Env, + entries: impl Iterator, +) -> Result> { + let mut object = Object::new(env)?; + for (key, value) in entries { + let converted_value = json_value(env, value)?; + // A JS string key preserves embedded NUL. Defining a data property also + // avoids invoking Object.prototype's __proto__ setter. + let property = Property::new() + .with_name(env, key.as_str())? + .with_napi_value(env, converted_value)?; + object.define_properties(&[property])?; + } + Ok(object) +} + +fn json_value<'env>(env: &'env Env, value: &Value) -> Result> { + match value { + Value::Null => Null.into_unknown(env), + Value::Bool(value) => value.into_unknown(env), + Value::String(value) => value.as_str().into_unknown(env), + Value::Number(value) => { + // All JSON numbers are JS Numbers, including u64 (with ordinary JS + // precision loss), never napi's serde-json unsigned string fallback. + let number = value + .as_f64() + .ok_or_else(|| napi::Error::from_reason("Invalid JSON number in page attrs"))?; + number.into_unknown(env) + } + Value::Array(values) => { + let len = u32::try_from(values.len()) + .map_err(|_| napi::Error::from_reason("Page attrs array is too large"))?; + let mut array = env.create_array(len)?; + for (index, value) in (0..len).zip(values) { + array.set(index, json_value(env, value)?)?; + } + array.into_unknown(env) + } + Value::Object(values) => object(env, values.iter())?.into_unknown(env), + } +} diff --git a/crates/rw-napi/src/lib.rs b/crates/rw-napi/src/lib.rs index 328a2cd2..b72aa794 100644 --- a/crates/rw-napi/src/lib.rs +++ b/crates/rw-napi/src/lib.rs @@ -1,3 +1,4 @@ +mod attrs; mod types; use std::collections::HashMap; @@ -418,6 +419,7 @@ fn build_page_response(site: &Site, path: &str) -> Result { last_modified, description: result.meta.description.clone(), page_kind: result.meta.kind.clone(), + attrs: (!result.meta.attrs.is_empty()).then(|| attrs::Attrs(Arc::clone(&result.meta))), section_ref, subpath, }, diff --git a/crates/rw-napi/src/types.rs b/crates/rw-napi/src/types.rs index f3ec71f4..30193d0f 100644 --- a/crates/rw-napi/src/types.rs +++ b/crates/rw-napi/src/types.rs @@ -3,6 +3,8 @@ use std::collections::HashMap; use napi_derive::napi; use rw_site::{Section, SectionAnchor}; +use crate::attrs::Attrs; + #[napi(object)] pub struct DiagramsConfig { #[napi(js_name = "krokiUrl")] @@ -164,7 +166,7 @@ pub struct NavigationResponse { pub section_ancestry: HashMap>, } -#[napi(object)] +#[napi(object, object_from_js = false)] pub struct PageMetaResponse { pub title: String, pub path: String, @@ -175,6 +177,8 @@ pub struct PageMetaResponse { pub description: Option, #[napi(js_name = "kind")] pub page_kind: Option, + #[napi(ts_type = "Record")] + pub attrs: Option, #[napi(js_name = "sectionRef")] pub section_ref: String, /// Page path relative to its section root. Stable across whole-section @@ -201,7 +205,7 @@ pub struct TocEntryResponse { pub id: String, } -#[napi(object)] +#[napi(object, object_from_js = false)] pub struct PageResponse { pub meta: PageMetaResponse, pub breadcrumbs: Vec, diff --git a/crates/rw-site/benches/page_rendering.rs b/crates/rw-site/benches/page_rendering.rs index 99d92ddf..d3aee0c3 100644 --- a/crates/rw-site/benches/page_rendering.rs +++ b/crates/rw-site/benches/page_rendering.rs @@ -131,3 +131,46 @@ fn caching(bencher: Bencher, kind: &str) { _ => unreachable!(), } } + +/// Retained Site / persistent-cache response with opaque page metadata. Identical +/// markdown and mtimes; setup and validation are untimed. This is Rust serving +/// work, not the NAPI conversion performed by an embedding host. +#[divan::bench(args = ["empty", "small"])] +fn attrs_cache_hit(bencher: Bencher, kind: &str) { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("docs"); + fs::create_dir(&source_dir).unwrap(); + let attrs = if kind == "small" { + serde_json::json!({ + "owner": "payments", "audiences": ["operators"], + "support": {"url": "https://example.org/support"}, "reviewDate": null + }) + } else { + serde_json::json!({}) + }; + for (name, body) in [ + ("cached.md", generate_markdown(10, 3)), + ( + "cached.meta.yaml", + serde_json::json!({"attrs": attrs}).to_string(), + ), + ] { + let path = source_dir.join(name); + fs::write(&path, body).unwrap(); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000)) + .unwrap(); + } + let site = create_site_with_config( + source_dir, + Arc::new(FileCache::new(dir.path().join("cache"), "bench")), + PageRendererConfig::default(), + ); + let first = site.render("cached").unwrap(); + assert_eq!(first.meta.attrs.is_empty(), kind == "empty"); + assert!(site.render("cached").unwrap().from_cache); + bencher.bench(|| site.render(black_box("cached"))); +} diff --git a/crates/rw-site/benches/site_structure.rs b/crates/rw-site/benches/site_structure.rs index e6ed59cf..9f7d3b68 100644 --- a/crates/rw-site/benches/site_structure.rs +++ b/crates/rw-site/benches/site_structure.rs @@ -104,3 +104,46 @@ fn build_from_scratch(bencher: Bencher) { .with_inputs(|| create_site(source_dir.clone())) .bench_values(|site| site.navigation(None)); } + +/// Synthetic 100-page all-site attrs cost, including filesystem metadata decode, +/// indexing and structure-cache serialization (even NullCache serializes before +/// discarding bytes). Fixture creation is untimed, with identical bodies/mtimes. +#[divan::bench(args = ["empty", "small"])] +fn attrs_build_from_scratch(bencher: Bencher, kind: &str) { + let dir = tempfile::tempdir().unwrap(); + let source_dir = dir.path().join("docs"); + fs::create_dir(&source_dir).unwrap(); + let attrs = if kind == "small" { + serde_json::json!({ + "owner": "payments", "audiences": ["operators"], + "support": {"url": "https://example.org/support"}, "reviewDate": null + }) + } else { + serde_json::json!({}) + }; + let sidecar = serde_json::json!({"attrs": attrs}).to_string(); + for i in 0..100 { + for (suffix, body) in [ + ("md", "# Selected\n\nFixed body.\n"), + ("meta.yaml", sidecar.as_str()), + ] { + let path = source_dir.join(format!("page-{i:04}.{suffix}")); + fs::write(&path, body).unwrap(); + fs::File::options() + .write(true) + .open(path) + .unwrap() + .set_modified(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000)) + .unwrap(); + } + } + let check = create_site(source_dir.clone()); + assert_eq!(check.navigation(None).unwrap().items.len(), 100); + assert_eq!( + check.render("page-0000").unwrap().meta.attrs.is_empty(), + kind == "empty" + ); + bencher + .with_inputs(|| create_site(source_dir.clone())) + .bench_values(|site| site.navigation(None)); +} diff --git a/crates/rw-site/src/page.rs b/crates/rw-site/src/page.rs index 455d8db8..5eeacce4 100644 --- a/crates/rw-site/src/page.rs +++ b/crates/rw-site/src/page.rs @@ -568,6 +568,7 @@ struct CachedPageRef<'a> { #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -591,6 +592,7 @@ mod tests { path: path.to_owned(), has_content, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.to_owned(), description: None, @@ -921,6 +923,40 @@ mod tests { assert!(result.toc.is_empty()); } + #[test] + fn attrs_use_current_canonical_meta_on_fresh_cached_and_virtual_pages() { + let (_temp, cache) = file_cache(); + let storage = MockStorage::new() + .with_file("page", "Page", "# Page\n\nUnchanged body") + .with_mtime("page", 1000.0); + let renderer = PageRenderer::new(Arc::new(storage), cache, PageRendererConfig::default()); + let mut before = make_page("Page", "page", true); + Arc::make_mut(&mut before.meta) + .attrs + .insert("owner".into(), serde_json::json!("old")); + let mut after = before.clone(); + Arc::make_mut(&mut after.meta) + .attrs + .insert("owner".into(), serde_json::json!("new")); + assert!(!Arc::ptr_eq(&before.meta, &after.meta)); + let ctx = RenderContext::default(); + let first = renderer.render("page", &before, vec![], &ctx).unwrap(); + let second = renderer.render("page", &after, vec![], &ctx).unwrap(); + assert!(!first.from_cache); + assert!(second.from_cache); + assert_eq!(first.html, second.html); + assert!(Arc::ptr_eq(&first.meta, &before.meta)); + assert_eq!(first.meta.attrs["owner"], serde_json::json!("old")); + assert!(Arc::ptr_eq(&second.meta, &after.meta)); + assert_eq!(second.meta.attrs["owner"], serde_json::json!("new")); + + after.has_content = false; + let virtual_page = renderer.render("page", &after, vec![], &ctx).unwrap(); + assert!(!virtual_page.has_content); + assert!(Arc::ptr_eq(&virtual_page.meta, &after.meta)); + assert_eq!(virtual_page.meta.attrs["owner"], serde_json::json!("new")); + } + #[test] fn test_render_page_cache_hit() { let temp_dir = tempfile::tempdir().unwrap(); @@ -1225,6 +1261,7 @@ mod tests { let renderer = create_renderer(storage); let mut page = make_page("Test", "test", true); page.meta = Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Test".to_owned(), description: Some("A description".to_owned()), diff --git a/crates/rw-site/src/site.rs b/crates/rw-site/src/site.rs index 6763aafc..85c57eef 100644 --- a/crates/rw-site/src/site.rs +++ b/crates/rw-site/src/site.rs @@ -591,6 +591,7 @@ mod tests { // Ensure Site is Send + Sync for use with Arc static_assertions::assert_impl_all!(super::Site: Send, Sync); + use std::collections::BTreeMap; use std::sync::Arc; use rw_storage::{Document, MockStorage, StorageErrorKind}; @@ -609,6 +610,7 @@ mod tests { path: path.to_owned(), has_content: true, meta: Arc::new(rw_meta::Meta { + attrs: BTreeMap::new(), name: None, title: title.to_owned(), description: None, @@ -692,6 +694,7 @@ mod tests { path: s.path.to_owned(), has_content: s.has_content, meta: Arc::new(rw_meta::Meta { + attrs: BTreeMap::new(), name: None, title: s.title.to_owned(), description: s.description.map(ToOwned::to_owned), diff --git a/crates/rw-site/src/site_state.rs b/crates/rw-site/src/site_state.rs index dbfca21e..2c88ab57 100644 --- a/crates/rw-site/src/site_state.rs +++ b/crates/rw-site/src/site_state.rs @@ -1107,6 +1107,7 @@ impl From for SiteState { mod tests { use super::*; use rw_meta::Meta; + use std::collections::BTreeMap; fn test_document( path: impl Into, @@ -1120,6 +1121,7 @@ mod tests { path: path.into(), has_content, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.into(), description: None, @@ -2838,6 +2840,134 @@ mod tests { ); } + #[test] + fn attrs_f64_bits_roundtrip_structure_cache() { + use rw_cache::{Cache, FileCache}; + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = FileCache::new(tmp.path().join("cache"), "v1"); + let bucket = cache.bucket("site"); + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let mut document = fingerprint_document("guide", "Guide", None, true); + document.meta = Arc::new(resolved.meta); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document); + let state = builder.build(); + // In-memory Value conversion would miss byte-decoder float rounding. + state.to_cache(bucket.as_ref(), "etag"); + let restored = + SiteState::from_cache(bucket.as_ref(), "etag").expect("readable structure cache"); + assert_eq!( + restored.get_page("guide").unwrap().meta.attrs["value"] + .as_f64() + .unwrap() + .to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn attrs_transport_depth_boundary_roundtrips_structure_cache() { + use rw_cache::{Cache, FileCache}; + + let tmp = tempfile::TempDir::new().unwrap(); + let cache = FileCache::new(tmp.path().join("cache"), "v1"); + let bucket = cache.bucket("site"); + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + let mut expected = leaf_value; + for level in usize::from(leaf != "null")..123 { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {value}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!(resolved.meta.attrs["value"], expected, "{shape}/{leaf}"); + let mut document = fingerprint_document("guide", "Guide", None, true); + document.meta = Arc::new(resolved.meta); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let state = builder.build(); + state.to_cache(bucket.as_ref(), "etag"); + let restored = SiteState::from_cache(bucket.as_ref(), "etag") + .expect("readable structure cache"); + assert_eq!( + restored.get_page("guide").unwrap().meta.attrs["value"], + expected, + "{shape}/{leaf}" + ); + assert_eq!( + restored.get_page("guide").unwrap().meta, + document.meta, + "{shape}/{leaf}" + ); + } + } + } + + #[test] + fn attrs_only_changes_preserve_fingerprint_and_roundtrip_structure_cache() { + let mut document = fingerprint_document("guide", "Guide", None, true); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let empty = builder.build(); + for owner in ["old", "new"] { + Arc::make_mut(&mut document.meta) + .attrs + .insert("owner".into(), serde_json::json!(owner)); + let mut builder = SiteStateBuilder::new(); + builder.add_document(document.clone()); + let state = builder.build(); + assert!(Arc::ptr_eq( + &state.get_page("guide").unwrap().meta, + &document.meta + )); + assert_eq!( + state.resolution_fingerprint(), + empty.resolution_fingerprint() + ); + let json = serde_json::to_string(&CachedSiteStateRef::from(&state)).unwrap(); + let rebuilt = SiteState::from(serde_json::from_str::(&json).unwrap()); + assert_eq!( + rebuilt.resolution_fingerprint(), + state.resolution_fingerprint() + ); + assert_eq!( + rebuilt.get_page("guide").unwrap().meta.attrs["owner"], + serde_json::json!(owner) + ); + } + } + #[test] fn fingerprint_changes_on_title() { let a = fingerprint_of( diff --git a/crates/rw-storage-fs/src/lib.rs b/crates/rw-storage-fs/src/lib.rs index fe181c99..b91c1a51 100644 --- a/crates/rw-storage-fs/src/lib.rs +++ b/crates/rw-storage-fs/src/lib.rs @@ -812,13 +812,60 @@ mod tests { ); } + #[test] + fn attrs_follow_selected_sidecar_and_frontmatter_without_inheritance() { + // Exercise every selected form, both with content and metadata-only. + // Lower-ranked files deliberately have unique attrs that must not leak. + for selected in ["page/meta.yaml", "page/index.meta.yaml", "page.meta.yaml"] { + for has_content in [false, true] { + let temp = create_test_dir(); + fs::create_dir(temp.path().join("page")).unwrap(); + let forms = ["page/meta.yaml", "page/index.meta.yaml", "page.meta.yaml"]; + let rank = forms.iter().position(|form| *form == selected).unwrap(); + for form in &forms[rank + 1..] { + fs::write(temp.path().join(form), "attrs: {unselected: true}\n").unwrap(); + } + fs::write( + temp.path().join(selected), + "title: Selected\nnamespace: parent\nattrs: {owner: sidecar, keep: true, nested: {old: true}, list: [old]}\n", + ).unwrap(); + if has_content { + fs::write(temp.path().join("page/index.md"), + "---\nattrs: {owner: null, nested: {new: true}, list: [new]}\n---\n# Body\n").unwrap(); + } + fs::write(temp.path().join("page/child.md"), "# Child\n").unwrap(); + let storage = FsStorage::new(temp.path().to_path_buf(), temp.path().to_path_buf()); + let first = storage.scan().unwrap(); + let second = storage.scan().unwrap(); + let page = first.iter().find(|d| d.path == "page").unwrap(); + let cached = second.iter().find(|d| d.path == "page").unwrap(); + assert!(std::sync::Arc::ptr_eq(&page.meta, &cached.meta)); + assert_eq!(page.has_content, has_content); + assert_eq!(page.meta.title, "Selected"); + let expected = if has_content { + serde_json::json!({"owner": null, "keep": true, "nested": {"new": true}, "list": ["new"]}) + } else { + serde_json::json!({"owner": "sidecar", "keep": true, "nested": {"old": true}, "list": ["old"]}) + }; + assert_eq!( + serde_json::to_value(&page.meta.attrs).unwrap(), + expected, + "{selected}" + ); + let child = first.iter().find(|d| d.path == "page/child").unwrap(); + assert!(child.meta.attrs.is_empty(), "{selected} must not inherit"); + assert_eq!(page.meta.namespace.as_deref(), Some("parent")); + } + } + } + #[test] fn test_sidecar_combines_metadata_and_content() { let temp_dir = create_test_dir(); fs::write(temp_dir.path().join("guide.md"), "# Original H1\n\nBody.").unwrap(); fs::write( temp_dir.path().join("guide.meta.yaml"), - "title: Sidecar Title\nkind: guide", + "title: Sidecar Title\nkind: guide\nattrs: {owner: named-leaf}", ) .unwrap(); @@ -829,6 +876,7 @@ mod tests { assert!(doc.has_content); assert_eq!(doc.meta.title, "Sidecar Title"); // sidecar wins over H1 assert_eq!(doc.meta.kind, Some("guide".to_owned())); + assert_eq!(doc.meta.attrs["owner"], "named-leaf"); // Content still served from the .md file. assert_eq!(storage.read("guide").unwrap(), "# Original H1\n\nBody."); @@ -940,7 +988,7 @@ mod tests { let temp_dir = create_test_dir(); fs::write( temp_dir.path().join("payments.meta.yaml"), - "kind: component\nnamespace: billing", + "kind: component\nnamespace: billing\nattrs: {owner: metadata-only-leaf}", ) .unwrap(); @@ -952,6 +1000,7 @@ mod tests { assert_eq!(doc.meta.title, "Payments"); // titlecased from url segment assert_eq!(doc.meta.kind, Some("component".to_owned())); assert_eq!(doc.meta.namespace, Some("billing".to_owned())); + assert_eq!(doc.meta.attrs["owner"], "metadata-only-leaf"); } #[test] @@ -2032,7 +2081,7 @@ mod tests { #[test] fn readme_fallback_frontmatter_is_preserved_and_shared_across_unchanged_scans() { let (_dir, _, storage) = create_readme_test_dir( - "---\ntitle: Readme Home\ndescription: Project docs\nkind: domain\nnamespace: docs\npages:\n - guide\n---\n# Body Title", + "---\ntitle: Readme Home\ndescription: Project docs\nkind: domain\nnamespace: docs\nattrs: {owner: project, nested: [null, true]}\npages:\n - guide\n---\n# Body Title", ); let first = storage .scan() @@ -2049,6 +2098,10 @@ mod tests { assert!(std::sync::Arc::ptr_eq(&first.meta, &second.meta)); assert_eq!(first.meta.title, "Readme Home"); + assert_eq!( + serde_json::to_value(&first.meta.attrs).unwrap(), + serde_json::json!({"owner": "project", "nested": [null, true]}) + ); assert_eq!(first.meta.description.as_deref(), Some("Project docs")); assert_eq!(first.meta.kind.as_deref(), Some("domain")); assert_eq!(first.meta.namespace.as_deref(), Some("docs")); diff --git a/crates/rw-storage-s3/Cargo.toml b/crates/rw-storage-s3/Cargo.toml index 66eaec53..73b41d5f 100644 --- a/crates/rw-storage-s3/Cargo.toml +++ b/crates/rw-storage-s3/Cargo.toml @@ -32,6 +32,7 @@ tracing = { workspace = true } [dev-dependencies] chrono = "0.4" pretty_assertions = { workspace = true } +rw-meta = { workspace = true } rw-storage = { workspace = true, features = ["mock"] } tempfile = { workspace = true } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/crates/rw-storage-s3/src/format.rs b/crates/rw-storage-s3/src/format.rs index 7443c9f1..a5fc66a2 100644 --- a/crates/rw-storage-s3/src/format.rs +++ b/crates/rw-storage-s3/src/format.rs @@ -144,6 +144,80 @@ mod tests { serde_json::from_value(wire).unwrap() } + #[test] + fn attrs_f64_bits_roundtrip_manifest_bytes() { + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = rw_meta::Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let mut document = + document_from_wire("guide", true, "Guide", None, None, None, None, None, false); + document.meta = std::sync::Arc::new(resolved.meta); + let manifest = Manifest::from(vec![document]); + let bytes = serde_json::to_vec(&manifest).unwrap(); + let decoded: Manifest = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.documents[0].meta.attrs["value"] + .as_f64() + .unwrap() + .to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn attrs_transport_depth_boundary_roundtrips_manifest_bytes() { + for shape in ["array", "object", "mixed"] { + for (leaf, leaf_value) in [ + ("null", serde_json::Value::Null), + ("[]", serde_json::json!([])), + ("{}", serde_json::json!({})), + ] { + let mut expected = leaf_value; + for level in usize::from(leaf != "null")..123 { + expected = if shape == "array" || (shape == "mixed" && level % 2 == 0) { + serde_json::Value::Array(vec![expected]) + } else { + serde_json::json!({"child": expected}) + }; + } + let value = serde_json::to_string(&expected).unwrap(); + let resolved = rw_meta::Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {value}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!(resolved.meta.attrs["value"], expected, "{shape}/{leaf}"); + let mut document = + document_from_wire("guide", true, "Guide", None, None, None, None, None, false); + document.meta = std::sync::Arc::new(resolved.meta); + let manifest = Manifest::from(vec![document]); + let bytes = serde_json::to_vec(&manifest).unwrap(); + let decoded: Manifest = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.documents[0].meta.attrs["value"], expected, + "{shape}/{leaf}" + ); + assert_eq!(decoded, manifest, "{shape}/{leaf}"); + } + } + } + #[test] fn test_manifest_serialization_roundtrip() { let manifest = Manifest::from(vec![ @@ -175,6 +249,9 @@ mod tests { assert_eq!(serde_json::to_string(&old).unwrap(), LEGACY_MANIFEST_JSON); assert_eq!(old.documents[0].meta.name, None); + assert_eq!(FORMAT_VERSION, 1); + assert_eq!(old.version, 1); + assert!(old.documents[0].meta.attrs.is_empty()); assert_eq!(old.documents[0].meta.kind.as_deref(), Some("domain")); assert_eq!(old.documents[0].meta.namespace.as_deref(), Some("payments")); assert_eq!( @@ -228,6 +305,31 @@ mod tests { assert!(!old_reader.documents[0].is_dir); } + #[test] + fn attrs_are_additive_in_version_one_and_ignored_by_old_readers() { + let mut manifest: Manifest = serde_json::from_str(LEGACY_MANIFEST_JSON).unwrap(); + let attrs = + serde_json::json!({"owner": null, "nested": {"audiences": ["operators", true, 3]}}); + std::sync::Arc::make_mut(&mut manifest.documents[0].meta).attrs = + serde_json::from_value(attrs.clone()).unwrap(); + let json = serde_json::to_string(&manifest).unwrap(); + let wire: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(wire["version"], 1); + assert_eq!(FORMAT_VERSION, 1); + assert_eq!(wire["documents"][0]["attrs"], attrs); + let current: Manifest = serde_json::from_str(&json).unwrap(); + assert_eq!(current, manifest); + let old: LegacyManifest = serde_json::from_str(&json).unwrap(); + assert_eq!(serde_json::to_string(&old).unwrap(), LEGACY_MANIFEST_JSON); + std::sync::Arc::make_mut(&mut manifest.documents[0].meta) + .attrs + .clear(); + assert_eq!( + serde_json::to_string(&manifest).unwrap(), + LEGACY_MANIFEST_JSON + ); + } + #[test] fn declared_name_manifest_roundtrip_is_syntactic_not_legacy_semantic_parity() { let mut wire: serde_json::Value = serde_json::from_str(LEGACY_MANIFEST_JSON).unwrap(); diff --git a/crates/rw-storage-s3/src/publisher.rs b/crates/rw-storage-s3/src/publisher.rs index 79568002..bbaf0c86 100644 --- a/crates/rw-storage-s3/src/publisher.rs +++ b/crates/rw-storage-s3/src/publisher.rs @@ -352,6 +352,74 @@ A -> B assert!(!bundle.content.contains("System(")); } + #[tokio::test] + async fn publication_preserves_selected_canonical_attrs_not_raw_frontmatter() { + // The scan is authoritative: bundle content deliberately disagrees. + let document: Document = serde_json::from_value(serde_json::json!({ + "path": "selected", "title": "Selected", "has_content": true, + "attrs": {"owner": "canonical", "keep": true, "nested": [null, {"x": 1}]} + })) + .unwrap(); + let canonical = Arc::clone(&document.meta); + let virtual_document: Document = serde_json::from_value(serde_json::json!({ + "path": "virtual", "title": "Virtual", "has_content": false, + "attrs": {"owner": "metadata-only"} + })) + .unwrap(); + let storage = MockStorage::new() + .with_scanned_document(document) + .with_scanned_document(virtual_document) + .with_content( + "selected", + "---\nattrs: {owner: raw, unexpected: true}\n---\n# Body\n", + ); + let documents = storage.scan().unwrap(); + let mut bundles = Vec::new(); + let (count, warnings) = build_bundles(&storage, &documents, &[], async |key, json| { + bundles.push((key, json)); + Ok(()) + }) + .await + .unwrap(); + assert_eq!(count, 1); + assert!(warnings.is_empty()); + assert_eq!(bundles[0].0, "pages/selected.json"); + let bundle: serde_json::Value = serde_json::from_slice(&bundles[0].1).unwrap(); + assert_eq!(bundle.as_object().unwrap().len(), 1); + assert!(bundle["content"].as_str().unwrap().contains("owner: raw")); + let manifest = Manifest::from(documents); + let selected = manifest + .documents + .iter() + .find(|d| d.path == "selected") + .unwrap(); + assert!(Arc::ptr_eq(&selected.meta, &canonical)); + let reloaded: Manifest = + serde_json::from_slice(&serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert_eq!(reloaded.version, 1); + assert_eq!(reloaded, manifest); + assert_eq!( + reloaded + .documents + .iter() + .find(|d| d.path == "selected") + .unwrap() + .meta + .attrs, + canonical.attrs + ); + assert_eq!( + reloaded + .documents + .iter() + .find(|d| d.path == "virtual") + .unwrap() + .meta + .attrs["owner"], + "metadata-only" + ); + } + #[test] fn dedup_preserves_first_seen_order() { let input = [ diff --git a/crates/rw-storage/Cargo.toml b/crates/rw-storage/Cargo.toml index ba2d394f..138ade37 100644 --- a/crates/rw-storage/Cargo.toml +++ b/crates/rw-storage/Cargo.toml @@ -14,9 +14,7 @@ chrono = "0.4" parking_lot = { workspace = true, optional = true } rw-meta = { workspace = true } serde = { workspace = true } - -[dev-dependencies] -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["float_roundtrip"] } [features] default = [] diff --git a/crates/rw-storage/src/mock.rs b/crates/rw-storage/src/mock.rs index 52ab18bd..2495b91e 100644 --- a/crates/rw-storage/src/mock.rs +++ b/crates/rw-storage/src/mock.rs @@ -2,7 +2,7 @@ //! //! Provides [`MockStorage`] for unit testing without filesystem access. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc; @@ -37,6 +37,7 @@ fn meta( pages: Option>, ) -> Arc { Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: title.into(), description, diff --git a/crates/rw-storage/src/storage.rs b/crates/rw-storage/src/storage.rs index 4f58a405..8757538e 100644 --- a/crates/rw-storage/src/storage.rs +++ b/crates/rw-storage/src/storage.rs @@ -13,6 +13,7 @@ //! //! Storage implementations handle the mapping from URL paths to their internal storage format. +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; @@ -69,7 +70,7 @@ pub struct Document { } #[derive(Serialize, Deserialize)] -struct DocumentWire { +struct DocumentWire { path: S, title: S, has_content: bool, @@ -85,6 +86,8 @@ struct DocumentWire { origin: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pages: Option

, + #[serde(default, skip_serializing_if = "Option::is_none")] + attrs: Option, #[serde(default = "default_is_dir")] is_dir: bool, } @@ -94,7 +97,7 @@ impl Serialize for Document { where S: serde::Serializer, { - DocumentWire::<&str, &[String]> { + DocumentWire::<&str, &[String], &BTreeMap> { path: self.path.as_str(), title: self.meta.title.as_str(), has_content: self.has_content, @@ -104,6 +107,7 @@ impl Serialize for Document { description: self.meta.description.as_deref(), origin: self.origin.as_deref(), pages: self.meta.pages.as_deref(), + attrs: (!self.meta.attrs.is_empty()).then_some(&self.meta.attrs), is_dir: self.is_dir, } .serialize(serializer) @@ -115,12 +119,16 @@ impl<'de> Deserialize<'de> for Document { where D: serde::Deserializer<'de>, { - let wire = DocumentWire::>::deserialize(deserializer)?; + let wire = + DocumentWire::, BTreeMap>::deserialize( + deserializer, + )?; Ok(Self { path: wire.path, has_content: wire.has_content, meta: Arc::new(Meta { + attrs: wire.attrs.unwrap_or_default(), name: wire.name, title: wire.title, description: wire.description, @@ -507,6 +515,78 @@ mod tests { assert_eq!(converted.to_rfc3339(), "2025-07-08T18:40:00.500+00:00"); } + #[test] + fn attrs_f64_bits_roundtrip_document_bytes() { + for (yaml, expected_bits) in [ + ("51.248178375505404", 0x4049_9fc4_4f1b_2f60), + ("0.1", 0.1_f64.to_bits()), + ("-0.125", (-0.125_f64).to_bits()), + ("1.2345678901234567", 1.234_567_890_123_456_7_f64.to_bits()), + ] { + let resolved = Meta::resolve_with_diagnostics( + None, + Some(&format!("attrs: {{value: {yaml}}}")), + "guide", + ); + assert!(resolved.diagnostics.is_empty()); + assert_eq!( + resolved.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits + ); + let document = Document { + path: "guide".into(), + has_content: true, + meta: Arc::new(resolved.meta), + origin: None, + is_dir: false, + diagnostics: Vec::new().into(), + }; + let bytes = serde_json::to_vec(&document).unwrap(); + let decoded: Document = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + decoded.meta.attrs["value"].as_f64().unwrap().to_bits(), + expected_bits, + "{yaml}" + ); + } + } + + #[test] + fn document_attrs_round_trip_on_flat_wire() { + let wire = serde_json::json!({ + "path": "guide", "title": "Guide", "has_content": true, "is_dir": false, + "attrs": { + "owner": null, + "nested": {"enabled": true}, + "values": [false, "text", -42, u64::MAX, 1.5, {}, []] + } + }); + let document: Document = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&document).unwrap(), wire); + assert_eq!(document.meta.attrs["owner"], serde_json::Value::Null); + assert_eq!( + document.meta.attrs["nested"], + serde_json::json!({"enabled": true}) + ); + assert!(document.diagnostics.is_empty()); + } + + #[test] + fn document_empty_attrs_are_omitted_on_flat_wire() { + for attrs in [None, Some(serde_json::json!({}))] { + let expected = serde_json::json!({ + "path": "guide", "title": "Guide", "has_content": true, "is_dir": true + }); + let mut wire = expected.clone(); + if let Some(attrs) = attrs { + wire["attrs"] = attrs; + } + let document: Document = serde_json::from_value(wire).unwrap(); + assert!(document.meta.attrs.is_empty()); + assert_eq!(serde_json::to_value(document).unwrap(), expected); + } + } + #[test] fn document_retains_declared_name_on_wire() { let wire = serde_json::json!({"path":"guide", "title":"Guide", "has_content":true, "is_dir":true, "name":"payments-api"}); @@ -520,6 +600,7 @@ mod tests { path: String::new(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Home".to_owned(), description: None, @@ -544,6 +625,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: None, @@ -568,6 +650,7 @@ mod tests { path: "domain/billing".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Billing".to_owned(), description: None, @@ -591,6 +674,7 @@ mod tests { path: "domains".to_owned(), has_content: false, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Domains".to_owned(), description: None, @@ -773,6 +857,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), @@ -822,6 +907,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), @@ -842,6 +928,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: None, @@ -891,6 +978,7 @@ mod tests { path: "guide".to_owned(), has_content: true, meta: Arc::new(Meta { + attrs: BTreeMap::new(), name: None, title: "Guide".to_owned(), description: Some("Getting started".to_owned()), diff --git a/docs/embedding.md b/docs/embedding.md index 6c39b150..3ae6203f 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -1,9 +1,9 @@ # Embedding -This page covers three concerns for embedding `@rwdocs/viewer` (or rendering pages +This page covers integration concerns for embedding `@rwdocs/viewer` (or rendering pages through `@rwdocs/core`) in a host application: pointing `@rwdocs/core` at a site -with `projectDir`, resolving cross-entity links via `resolveSectionRefs`, and -durable comment keys for hosts that store their own comments. +with `projectDir`, resolving cross-entity links via `resolveSectionRefs`, +durable comment keys for hosts that store their own comments, and page attrs. When you embed the viewer in a host application that stores **its own** comments — for example the Backstage plugin pair — each comment needs a stable @@ -23,6 +23,42 @@ directory, and it does not consult the Node process's working directory. A host that mounts several sites can therefore point each `createSite` call at its own directory without one site's configuration leaking into another. +## Consuming page attrs + +Only the NAPI/core page boundary exposes [page-local `attrs`](metadata.md#attrs-page-local-integration-data): + +```js +const page = await site.renderPage("guide"); +const owner = page.meta.attrs?.owner; +if (typeof owner === "string") { + // Validate against your integration's schema before using it. +} +``` + +The core declarations describe JSON values (including nested null), not `any`. +`meta.attrs` is omitted when empty, and never inherits. Use string identifiers +when exact large integers matter: JavaScript number precision applies. Rust JSON +byte roundtrips preserve the chosen supported finite `f64` values; they do not +remove JavaScript's numeric limits. Source validation bounds array/object +nesting so manifest/cache readers can load the values; see +[attrs validation](metadata.md#attrs-page-local-integration-data). +Attrs are ordinary data, including keys such as `__proto__`; do not treat them as +executable configuration or authorization rules. RW does not render or interpret them, and +the built-in HTTP API, viewer, search and navigation shapes are unchanged. + +For S3, deploy version-1-compatible readers with attrs support before consumers +rely on published attrs. Older readers ignore and do not preserve them on +reserialization. Attributes increase all-site manifest/structure-cache payloads +and resident snapshot memory; selected-page conversion adds response work, not +new S3 object requests. Measure your own payloads and Site reuse/eviction cadence: +local benchmarks do not establish external-host latency. + +Metadata on fresh and cached HTML responses comes from the applicable Site +snapshot. Keep the existing host refresh strategy; attrs add no polling, watcher +delivery, same-mtime or cross-process cache correctness guarantees. An attrs-only +snapshot change does not introduce whole-site HTML invalidation; normal source +mtime invalidation still applies. No new refresh API is provided. + ## `resolveSectionRefs` must map the site-root ref `mountRw({ resolveSectionRefs })` lets the viewer turn a cross-entity link (a diff --git a/docs/metadata.md b/docs/metadata.md index 8e998efa..ff77b2e7 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -40,8 +40,87 @@ These fields are available in both frontmatter and meta.yaml: - `kind` -- page kind (e.g., `domain`, `guide`). Pages with `kind` are registered as sections. - `name` -- page-local section/catalog and diagram identifier, overriding the path-derived name when `kind` is set (see below). - `namespace` -- Backstage catalog namespace for the section (see below). +- `attrs` -- opaque, page-local JSON-compatible attributes for integrations (see below) - `pages` -- ordered list of child page slugs for navigation sidebar ordering (directory-level only) +### `attrs`: page-local integration data + +Declare attributes in frontmatter or the selected sidecar, including on the +README homepage and metadata-only pages; `kind` is not required: + +```yaml +attrs: + owner: payments + audiences: [operators] + support: {url: https://example.org/support} + reviewDate: null +``` + +- **No inheritance.** Attributes belong only to the declaring page, not its + children. They do not change section identity, namespace, titles, or URLs. +- **Top-level overlay.** Start with the selected sidecar's attrs; frontmatter + overrides matching keys. A nested object or array replaces the previous value + **whole**, without recursive merging. `attrs: {}` contributes no overrides. +- **Null is data.** `reviewDate: null` keeps that key present and replaces any + sidecar value; it is different from an absent key, not a deletion instruction. + There is no reset/deletion syntax: remove a key from every source declaring it + to remove it entirely. +- **Whole-field validation.** Attrs must be a string-keyed map of JSON-compatible + strings, booleans, finite supported numbers, nulls, arrays, and objects. Types + are preserved, not string-coerced. Keys are case-sensitive opaque strings. + YAML custom tags, non-string object keys, and non-finite numbers are not + supported. Array/object nesting is bounded so values remain readable inside + manifest and structure-cache envelopes. Empty arrays/objects count toward + nesting; scalars do not. Deeply nested values can be valid JSON but exceed + RW's supported transport bound. + Any unsupported nested value discards that source's **entire attrs + field** with a source-attributed warning, retaining sibling metadata and + valid attrs from the other source. Whole-field `attrs: null` is invalid and + falls back to valid sidecar attrs. Malformed YAML and duplicate mappings + detected by the parser keep their existing source-level failure behavior. + +`@rwdocs/core` / NAPI `renderPage()` exposes resolved attributes as optional +`meta.attrs`, omitted when empty. Built-in HTTP page responses, viewer, +navigation, search, Confluence, and rendered HTML do not expose or interpret +attrs. Integrations supply their own schema and presentation; attrs are exposed +metadata, **not a secret store or an authorization policy**. + +Rust retains numbers within serde_json's supported range, including the exact +bits of the finite `f64` chosen during source resolution through JSON byte +serialization/decoding. This does not promise exact decimal arithmetic. +JavaScript receives ordinary numbers, not BigInt or a string-number codec. Use +**strings for numeric identifiers requiring exact large integers**, especially beyond +`Number.MAX_SAFE_INTEGER` (`9007199254740991`). + +Attrs travel in the existing all-site manifest and structure cache, increasing +bytes, decoding/serialization work, and snapshot memory. The selected page also +incurs native-to-JavaScript conversion cost. There are no extra attrs objects or +fetches, nor attrs-specific render-cache fingerprints or unrelated-page HTML +invalidation. Existing source mtime changes can still invalidate an edited page. +Freshness follows the existing Site snapshot/load lifecycle: this adds no +polling, watcher, same-mtime recovery, or cross-process refresh guarantees. +See [embedding guidance](embedding.md#consuming-page-attrs). + +#### Attrs compatibility and rollout + +The flattened Document/S3 wire adds optional `attrs`, omitted when empty; +empty-attrs serialization is unchanged. Manifest `FORMAT_VERSION` remains **1**. +New readers accept old manifests; old readers ignore attrs and **lose them if +reserializing documents**. Upgrade readers before integrations rely on attrs +from published bundles. There is no migration or automatic deployment; mixed +versions do not provide semantic parity. Rollback requires disabling integration +consumption and coordinating reader/publisher changes, not deleting stored data. + +Adding `attrs: BTreeMap` to public `rw_meta::Meta` is a +**breaking (pre-1.0) Rust struct-literal change**. Existing literals must add +`attrs: Default::default()` or use `Meta::resolve`. Document syntax and optional +NAPI output are additive; this does not add an HTTP field or a CLI command. +Directly constructed Rust `Meta.attrs` is not a validated wrapper: callers must +keep values within the same transport nesting bound before publication or caching. +`rw-storage` enables serde_json's `float_roundtrip` feature for byte decoding; +Cargo feature unification also affects other JSON float decoding in the same +binary, not only attrs. + ### Migrating legacy metadata Earlier versions accepted `type` as an alias for `kind`. Rename that key to @@ -102,8 +181,9 @@ or migrated automatically. Adding `name: Option` to public `rw_meta::Meta` is a **pre-1.0 Rust struct-literal source break**: existing literals must add `name: None` (or use -`Meta::resolve` to parse declarations). HTTP/NAPI/viewer page metadata shapes -are unchanged; their existing section projections carry the effective name. +`Meta::resolve` to parse declarations). For the `name` feature, HTTP/NAPI/viewer +page metadata shapes are unchanged; their existing section projections carry +the effective name. The flattened Document/S3 wire adds optional `name`, omitted when absent; no-name serialization is unchanged. S3 manifest `FORMAT_VERSION` stays **1**. @@ -166,8 +246,8 @@ docs/guides/meta.yaml: sidecar `pages`: expected a list, found a string Warnings log at WARN level, so run `rw serve --verbose` (or set `RUST_LOG=warn`) to see them — the default verbosity hides them. -Diagnostics never reach HTTP responses or published bundles. Unknown keys -remain silently ignored. +Diagnostics never reach public responses or published bundles. Unknown top-level +metadata keys remain silently ignored; keys inside `attrs` are retained as data. ## Navigation ordering @@ -204,8 +284,8 @@ The page title is resolved in this order: ## Inheritance Metadata does not inherit from parent directories: `title`, `description`, -`kind`, `name`, and `pages` apply only to the page or directory that declares them, not -to anything beneath it. `namespace` is the one exception — it inherits down +`kind`, `name`, `attrs`, and `pages` apply only to the page or directory that +declares them, not to anything beneath it. `namespace` is the one exception — it inherits down the tree, as described above. ## Named sidecar files (`.meta.yaml`) diff --git a/packages/core/index.d.ts b/packages/core/index.d.ts index 24a508ab..b1160713 100644 --- a/packages/core/index.d.ts +++ b/packages/core/index.d.ts @@ -1,5 +1,6 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue } export declare class RwSite { getNavigation(sectionRef?: string | undefined | null): Promise listSections(): Promise> @@ -161,6 +162,7 @@ export interface PageMetaResponse { lastModified: string description?: string kind?: string + attrs?: Record sectionRef: string /** * Page path relative to its section root. Stable across whole-section diff --git a/packages/core/package.json b/packages/core/package.json index 7be6d084..aa47b877 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -10,6 +10,7 @@ "types": "index.d.ts", "napi": { "binaryName": "core", + "dtsHeader": "/* auto-generated by NAPI-RS */\n/* eslint-disable */\nexport type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }\n", "targets": [ "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", diff --git a/packages/core/test/helpers/attrs-s3-fixture.mjs b/packages/core/test/helpers/attrs-s3-fixture.mjs new file mode 100644 index 00000000..6ebe4acf --- /dev/null +++ b/packages/core/test/helpers/attrs-s3-fixture.mjs @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createServer } from "node:http"; +import { crc32 } from "node:zlib"; + +// Only the objects/operations used by the attrs addon test, not an S3 emulator. +export async function attrsS3Fixture(t, attrs) { + const objects = new Map(); + const requests = []; + const failures = []; + const prefix = "/attrs-test/docs/"; + const cacheKeys = new Set([ + "cache/site/structure", + "cache/pages/selected", + "cache/pages/unrelated", + ]); + const put = (key, value, cacheEtag) => { + const body = Buffer.isBuffer(value) ? value : Buffer.from(JSON.stringify(value)); + objects.set(key, { + body, + etag: `"${createHash("sha256").update(body).digest("hex")}"`, + cacheEtag, + }); + }; + const setAttrs = (next) => { + put("manifest.json", { + version: 1, + documents: [ + { + path: "selected", + title: "Selected", + has_content: true, + is_dir: false, + ...(Object.keys(next).length ? { attrs: next } : {}), + }, + { path: "unrelated", title: "Unrelated", has_content: true, is_dir: false }, + { + path: "virtual", + title: "Virtual", + has_content: false, + is_dir: true, + ...(Object.keys(next).length ? { attrs: next } : {}), + }, + ], + mtimes: { selected: 1700000000, unrelated: 1700000000, virtual: 1700000000 }, + }); + }; + setAttrs(attrs); + put("pages/selected.json", { content: "# Selected\n\nFixed body.\n" }); + put("pages/unrelated.json", { content: "# Unrelated\n\nOther fixed body.\n" }); + + const server = createServer(async (req, res) => { + try { + const url = new URL(req.url, "http://127.0.0.1"); + assert.ok(url.pathname.startsWith(prefix), `unexpected path: ${req.url}`); + const key = url.pathname.slice(prefix.length); + requests.push(`${req.method} ${key}`); + const operation = { GET: "GetObject", HEAD: "HeadObject", PUT: "PutObject" }[req.method]; + assert.ok(operation, `unsupported method: ${req.method}`); + for (const [name, value] of url.searchParams) { + assert.equal(name, "x-id", `unsupported query: ${req.url}`); + assert.equal(value, operation); + } + assert.match(req.headers.authorization ?? "", /Credential=rw-attrs-dummy\//); + if (req.method === "PUT") { + assert.ok(cacheKeys.has(key), `unsupported PUT: ${key}`); + const body = await decodeUpload(req); + // Invalid framing must never masquerade as an ordinary cache miss. + assert.ok(JSON.parse(body.toString()), "cache upload must be decoded JSON"); + const cacheEtag = req.headers["x-amz-meta-cache-etag"]; + assert.equal(typeof cacheEtag, "string"); + assert.ok(cacheEtag.length > 0); + put(key, body, cacheEtag); + res.writeHead(200, { ETag: objects.get(key).etag, "Content-Length": 0 }); + res.end(); + return; + } + assert.ok(objects.has(key) || cacheKeys.has(key), `unsupported GET: ${key}`); + if (req.method === "HEAD") assert.equal(key, "manifest.json"); + const object = objects.get(key); + if (!object) { + const body = "NoSuchKeyFixture cache miss"; + res.writeHead(404, { + "Content-Type": "application/xml", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); + return; + } + res.writeHead(200, { + ETag: object.etag, + "Content-Type": "application/octet-stream", + "Content-Length": object.body.length, + ...(object.cacheEtag ? { "x-amz-meta-cache-etag": object.cacheEtag } : {}), + }); + res.end(req.method === "HEAD" ? undefined : object.body); + } catch (error) { + failures.push(error.stack); + res.writeHead(400, { "Content-Length": 0 }); + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(async () => { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + server.closeAllConnections(); + }); + assert.deepEqual(failures, [], "unsupported S3 request or malformed upload"); + }); + return { + config: { + s3: { + bucket: "attrs-test", + entity: "docs", + region: "us-east-1", + endpoint: `http://127.0.0.1:${server.address().port}`, + accessKeyId: "rw-attrs-dummy", + secretAccessKey: "rw-attrs-dummy-secret", + }, + }, + setAttrs, + objects, + takeRequests() { + assert.deepEqual(failures, [], "unsupported S3 request or malformed upload"); + return requests.splice(0); + }, + }; +} + +async function decodeUpload(req) { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const wire = Buffer.concat(chunks); + // Node removes HTTP transfer-encoding, but NOT AWS's content-encoding. + assert.equal(req.headers["x-amz-sdk-checksum-algorithm"], "CRC32"); + let body = wire; + let checksum = req.headers["x-amz-checksum-crc32"]; + if (req.headers["content-encoding"] === "aws-chunked") { + assert.equal(req.headers["x-amz-trailer"], "x-amz-checksum-crc32"); + const decoded = []; + let offset = 0; + while (true) { + const end = wire.indexOf("\r\n", offset); + assert.ok(end >= offset, "missing AWS chunk length"); + const size = wire.subarray(offset, end).toString(); + assert.match(size, /^[0-9a-f]+$/i, "unsupported signed AWS chunk"); + offset = end + 2; + const length = Number.parseInt(size, 16); + if (length === 0) break; + assert.ok(offset + length + 2 <= wire.length, "truncated AWS chunk"); + decoded.push(wire.subarray(offset, offset + length)); + offset += length; + assert.equal(wire.subarray(offset, offset + 2).toString(), "\r\n"); + offset += 2; + } + const trailer = wire.subarray(offset).toString(); + const match = /^x-amz-checksum-crc32:([A-Za-z0-9+/=]+)\r\n\r\n$/.exec(trailer); + assert.ok(match, `unsupported AWS trailer: ${JSON.stringify(trailer)}`); + checksum = match[1]; + body = Buffer.concat(decoded); + assert.equal(body.length, Number(req.headers["x-amz-decoded-content-length"])); + } else { + assert.equal(req.headers["content-encoding"], undefined); + } + const expected = Buffer.alloc(4); + expected.writeUInt32BE(crc32(body)); + assert.equal(checksum, expected.toString("base64"), "AWS CRC32 must match decoded bytes"); + return body; +} diff --git a/packages/core/test/render-page-attrs-s3.test.mjs b/packages/core/test/render-page-attrs-s3.test.mjs new file mode 100644 index 00000000..66cf93f5 --- /dev/null +++ b/packages/core/test/render-page-attrs-s3.test.mjs @@ -0,0 +1,130 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { attrsS3Fixture } from "./helpers/attrs-s3-fixture.mjs"; + +const require = createRequire(import.meta.url); +const { createSite } = require("../index.js"); + +function assertAttrs(page, attrs) { + if (Object.keys(attrs).length === 0) { + assert.equal(Object.hasOwn(page.meta, "attrs"), false); + } else { + assert.deepEqual(page.meta.attrs, attrs); + } +} + +const initialLoad = ["GET manifest.json", "PUT cache/site/structure"]; +const coldPage = (page) => [ + `GET cache/pages/${page}`, + `GET pages/${page}.json`, + `PUT cache/pages/${page}`, +]; + +for (const [label, attrs] of [ + ["empty", {}], + [ + "populated", + { owner: "old", nested: { audiences: ["operators"], nullable: null }, enabled: true }, + ], +]) { + test(`S3 ${label} attrs use the same exact cold/warm requests and retain hits across reload`, async (t) => { + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + assert.deepEqual(fixture.takeRequests(), []); + const first = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), [...initialLoad, ...coldPage("selected")]); + assertAttrs(first, attrs); + const warm = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assert.deepEqual(warm, first); + const unrelated = await site.renderPage("unrelated"); + assert.deepEqual(fixture.takeRequests(), coldPage("unrelated")); + assertAttrs(unrelated, {}); + assert.deepEqual(await site.renderPage("unrelated"), unrelated); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/unrelated"]); + assertAttrs(await site.renderPage("virtual"), attrs); + assert.deepEqual(fixture.takeRequests(), []); + + assert.equal(await site.reload(false), false); + assert.deepEqual(fixture.takeRequests(), ["HEAD manifest.json"]); + // Keep one Site and fixed render inputs to isolate attrs-only refresh. + for (const [force, next] of [ + [false, { owner: "new" }], + [true, { owner: "forced", values: [null, 2] }], + [false, {}], + ]) { + const selectedCache = fixture.objects.get("cache/pages/selected"); + const unrelatedCache = fixture.objects.get("cache/pages/unrelated"); + const priorManifestEtag = fixture.objects.get("manifest.json").etag; + fixture.setAttrs(next); + assert.notEqual(fixture.objects.get("manifest.json").etag, priorManifestEtag); + assert.equal(await site.reload(force), true); + assert.deepEqual(fixture.takeRequests(), [ + ...(force ? [] : ["HEAD manifest.json"]), + "GET cache/site/structure", + "GET manifest.json", + "PUT cache/site/structure", + ]); + const refreshed = await site.renderPage("selected"); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assertAttrs(refreshed, next); + assert.equal(refreshed.content, first.content); + assert.deepEqual( + { ...refreshed, meta: { ...refreshed.meta, attrs: undefined } }, + { ...first, meta: { ...first.meta, attrs: undefined } }, + ); + assert.deepEqual(await site.renderPage("unrelated"), unrelated); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/unrelated"]); + assertAttrs(await site.renderPage("virtual"), next); + assert.deepEqual(fixture.takeRequests(), []); + assert.equal(fixture.objects.get("cache/pages/selected"), selectedCache); + assert.equal(fixture.objects.get("cache/pages/unrelated"), unrelatedCache); + assert.equal(await site.reload(false), false); + assert.deepEqual(fixture.takeRequests(), ["HEAD manifest.json"]); + } + }); + + test(`S3 ${label} metadata-only attrs need no page bundle or render-cache request even cold`, async (t) => { + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + const first = await site.renderPage("virtual"); + assert.deepEqual(fixture.takeRequests(), initialLoad); + assertAttrs(first, attrs); + assert.equal(first.content, "

Virtual

\n"); + assert.deepEqual(await site.renderPage("virtual"), first); + assert.deepEqual(fixture.takeRequests(), []); + }); +} + +// Catch extra Rust byte-decoder rounding, not ordinary JS large-integer limits. +test("finite attrs fractions agree across filesystem, S3 and structure-cache bytes", async (t) => { + const attrs = { value: 51.248178375505404, small: 0.1, negative: -0.125 }; + const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "rw-core-attrs-f64-")); + t.after(() => fs.rmSync(projectDir, { recursive: true, force: true })); + fs.writeFileSync(path.join(projectDir, "rw.toml"), ""); + fs.mkdirSync(path.join(projectDir, "docs")); + fs.writeFileSync( + path.join(projectDir, "docs", "selected.md"), + "---\nattrs: {value: 51.248178375505404, small: 0.1, negative: -0.125}\n---\n# Selected\n", + ); + assertAttrs(await createSite({ projectDir }).renderPage("selected"), attrs); + + const fixture = await attrsS3Fixture(t, attrs); + const site = createSite(fixture.config); + assertAttrs(await site.renderPage("selected"), attrs); + assert.deepEqual(fixture.takeRequests(), [...initialLoad, ...coldPage("selected")]); + + // Change only the cache token so reload decodes the original Rust-produced float bytes. + const structure = fixture.objects.get("cache/site/structure"); + assert.equal(structure.cacheEtag, "0"); + structure.cacheEtag = "1"; + assert.equal(await site.reload(true), true); + assert.deepEqual(fixture.takeRequests(), ["GET cache/site/structure"]); + assertAttrs(await site.renderPage("selected"), attrs); + assert.deepEqual(fixture.takeRequests(), ["GET cache/pages/selected"]); + assert.equal(fixture.objects.get("cache/site/structure"), structure); +}); diff --git a/packages/core/test/render-page-metadata.test.mjs b/packages/core/test/render-page-metadata.test.mjs index 52c8e97d..3be1bbe4 100644 --- a/packages/core/test/render-page-metadata.test.mjs +++ b/packages/core/test/render-page-metadata.test.mjs @@ -74,3 +74,105 @@ test("renderPage omits undeclared and internal metadata", async (t) => { assert.deepEqual(Object.keys(page).sort(), responseKeys); assert.deepEqual(Object.keys(page.meta).sort(), requiredMetaKeys); }); + +// Losing the selected page's canonical attrs, using assignment for object keys, +// or converting unsigned JSON numbers to strings must fail at the JS boundary. +const attrsYaml = [ + "attrs:", + " audiences: [operators]", + " owner: null", + " large: 18446744073709551615", + " signed: -9223372036854775808", + " fraction: 1.25", + " enabled: true", + " disabled: false", + ' "__proto__": {marker: root}', + " constructor: root-constructor", + " prototype: root-prototype", + ' "key\\u0000tail": kept', + " recursive:", + ' - [null, false, 3.5, "text\\u0000tail"]', + ' - "__proto__": {marker: nested}', + " constructor: nested-constructor", + " prototype: nested-prototype", + ' "key\\u0000tail": nested-kept', + " - {}", + " - []", +].join("\n"); + +for (const source of ["frontmatter", "metadata-only"]) { + test(`renderPage exposes recursive attrs from ${source} as ordinary JS values`, async (t) => { + const projectDir = project(t); + const dir = path.join(projectDir, "docs", "selected"); + fs.mkdirSync(dir); + const metadata = `title: Selected\nkind: domain\nname: selected-page\n${attrsYaml}\n`; + if (source === "frontmatter") { + fs.writeFileSync(path.join(dir, "index.md"), `---\n${metadata}---\n# Body\n`); + } else { + fs.writeFileSync(path.join(dir, "meta.yaml"), metadata); + } + + const page = await createSite({ projectDir }).renderPage("selected"); + + assert.ok(page.meta.attrs, "nonempty canonical attrs must be exposed"); + assert.deepEqual(page.meta.attrs.audiences, ["operators"]); + assert.equal(page.meta.attrs.owner, null); + assert.equal(typeof page.meta.attrs.large, "number"); + assert.equal(page.meta.attrs.large, Number("18446744073709551615")); + assert.equal(typeof page.meta.attrs.signed, "number"); + assert.equal(page.meta.attrs.signed, Number("-9223372036854775808")); + assert.equal(page.meta.attrs.fraction, 1.25); + assert.equal(page.meta.attrs.enabled, true); + assert.equal(page.meta.attrs.disabled, false); + assert.equal(Object.hasOwn(page.meta.attrs, "__proto__"), true); + assert.equal(Object.getPrototypeOf(page.meta.attrs), Object.prototype); + assert.deepEqual(page.meta.attrs.__proto__, { marker: "root" }); + assert.equal(page.meta.attrs["key\u0000tail"], "kept"); + assert.deepEqual(page.meta.attrs.recursive[0], [null, false, 3.5, "text\u0000tail"]); + const nested = page.meta.attrs.recursive[1]; + assert.equal(Object.hasOwn(nested, "__proto__"), true); + assert.equal(Object.getPrototypeOf(nested), Object.prototype); + assert.deepEqual(nested.__proto__, { marker: "nested" }); + assert.equal(nested["key\u0000tail"], "nested-kept"); + for (const [object, prefix] of [ + [page.meta.attrs, "root"], + [nested, "nested"], + ]) { + for (const key of ["constructor", "prototype"]) { + assert.equal(Object.hasOwn(object, key), true); + assert.equal(object[key], `${prefix}-${key}`); + } + const descriptor = Object.getOwnPropertyDescriptor(object, "__proto__"); + assert.equal(descriptor.writable, true); + assert.equal(descriptor.enumerable, true); + assert.equal(descriptor.configurable, true); + } + assert.deepEqual(page.meta.attrs.recursive[2], {}); + assert.equal(Object.getPrototypeOf(page.meta.attrs.recursive[2]), Object.prototype); + assert.deepEqual(page.meta.attrs.recursive[3], []); + assert.equal(page.meta.title, "Selected"); + assert.equal(page.meta.path, "/selected"); + assert.equal(page.meta.sourceFile, source === "frontmatter" ? "selected" : ""); + assert.equal(page.meta.sectionRef, "domain:default/selected-page"); + assert.equal(page.meta.subpath, ""); + assert.deepEqual(Object.keys(page).sort(), responseKeys); + assert.deepEqual(Object.keys(page.meta).sort(), ["attrs", "kind", ...requiredMetaKeys]); + }); +} + +test("renderPage omits empty attrs on filesystem and metadata-only pages", async (t) => { + const projectDir = project(t); + // Parent attrs must not leak into either empty child. + fs.writeFileSync( + path.join(projectDir, "docs", "index.md"), + "---\nattrs: {parent: true}\n---\nHome\n", + ); + fs.writeFileSync(path.join(projectDir, "docs", "empty.md"), "---\nattrs: {}\n---\nEmpty\n"); + fs.mkdirSync(path.join(projectDir, "docs", "virtual")); + fs.writeFileSync(path.join(projectDir, "docs", "virtual", "meta.yaml"), "attrs: {}\n"); + const site = createSite({ projectDir }); + for (const pagePath of ["empty", "virtual"]) { + const emptyPage = await site.renderPage(pagePath); + assert.equal(Object.hasOwn(emptyPage.meta, "attrs"), false); + } +});