trailview: bake the coarse stamps and the structure meshes (#3194)

builder/web/scripts/trailview-stamps.mjs

@@ -0,0 +1,62 @@
+// The stamp process of the `obc data` step `planner/trailview-stamps` (host/obc-trailview-features/src/stamps.rs).
+// All numbers are little-endian. It writes the stamp version first: a u32 length and its UTF-8. Then it reads
+// feature tiles until standard input ends, each a u8 zoom, u32 x, u32 y, u32 length and the tile body, and writes
+// for each the u32 count of its baked nodes, each a u8 level, u32 x, u32 y, u32 length and the node bytes.
+//
+//   node builder/web/scripts/trailview-stamps.mjs < tiles > nodes
+
+// The bundle module is TypeScript, which Node runs from 22.18 on.
+const [major, minor] = process.versions.node.split('.').map(Number);
+if (major < 22 || (major === 22 && minor < 18)) {
+    console.error(`trailview-stamps: Node ${process.versions.node} runs no TypeScript module; it needs 22.18 or newer`);
+    process.exit(1);
+}
+const { stampBundle } = await import('../vite/stamp-bundle.ts').catch(error => {
+    console.error(`trailview-stamps: ${error.message}; run \`npm ci --prefix builder/web\` first`);
+    process.exit(1);
+});
+const { code, version } = await stampBundle();
+// The stamp worker sets `self.onmessage`. Standard output holds the frames, so the stamp code logs to standard error.
+globalThis.self = globalThis;
+console.log = console.info = console.debug = console.error;
+const { bakeTile } = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`);
+
+const write = chunk => new Promise((resolve, reject) => process.stdout.write(chunk, error => (error ? reject(error) : resolve())));
+const u32 = value => {
+    const out = Buffer.alloc(4);
+    out.writeUInt32LE(value);
+    return out;
+};
+const place = (z, x, y, length) => {
+    const out = Buffer.alloc(13);
+    out.writeUInt8(z, 0);
+    out.writeUInt32LE(x, 1);
+    out.writeUInt32LE(y, 5);
+    out.writeUInt32LE(length, 9);
+    return out;
+};
+
+/** The next `n` bytes of standard input; null at its end where `last` allows it. */
+const input = process.stdin[Symbol.asyncIterator]();
+let held = Buffer.alloc(0);
+async function read(n, last = false) {
+    while (held.length < n) {
+        const { value, done } = await input.next();
+        if (done) {
+            if (last && !held.length) return null;
+            throw new Error(`standard input ends inside a tile: ${held.length} of ${n} bytes`);
+        }
+        held = Buffer.concat([held, value]);
+    }
+    const out = held.subarray(0, n);
+    held = held.subarray(n);
+    return out;
+}
+
+await write(Buffer.concat([u32(Buffer.byteLength(version)), Buffer.from(version)]));
+for (let head; (head = await read(13, true)); ) {
+    const tile = { level: head.readUInt8(0), x: head.readUInt32LE(1), y: head.readUInt32LE(5) };
+    const body = await read(head.readUInt32LE(9));
+    const nodes = await bakeTile(tile, body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength));
+    await write(Buffer.concat([u32(nodes.length), ...nodes.flatMap(([node, bytes]) => [place(node.level, node.x, node.y, bytes.byteLength), bytes])]));
+}

builder/web/src/lib/trailview/engine.ts

@@ -169,7 +169,7 @@ export class TrailView {
         this.pose = this.poseOf(view);
         this.extras = { date: view.date, time: view.time, weather: view.weather, debug: view.debug, plan: view.plan };
         const { features } = tiles;
-        this.terrain = new Terrain(this.frame, { fetch: tiles.terrain, decode: pageDecoder(signal) }, features, config.bounds, () => this.request(), signal, groundPlan, finest);
+        this.terrain = new Terrain(this.frame, { fetch: tiles.terrain, decode: pageDecoder(signal) }, features, config.bounds, () => this.request(), signal, groundPlan, finest, tiles.stamps);
         this.groundLook = new Ground(ground, () => this.request());
         this.houses = new Houses(this.frame, features, this.groundLook.rules, () => this.request(), signal);
         this.trees = new TreeModels(this.instances, () => this.request(), () => this.treesNear(), signal);

builder/web/src/lib/trailview/stream/terrain.ts

@@ -1,4 +1,5 @@
 import { PMTiles } from 'pmtiles';
+import STAMP_VERSION from 'virtual:stamp-version';
 import type { PlannerConfig } from '../../planner/config';
 import { decodedTiles, fetchTile, openSource, requestSignal, type Source } from '../../planner/layers/archive';
 import { worldPixel } from '../../planner/layers/mercator';
@@ -76,6 +77,7 @@ export function releaseTiles(config: PlannerConfig, stop = new AbortController()
 export interface EyeTiles {
     terrain: TerrainSource['fetch'];
     features: ReturnType<typeof featureTiles>;
+    stamps: ReturnType<typeof bakedNodes>;
     /** Settles when both tiles under the eye have arrived or failed. */
     arrived: Promise<unknown>;
 }
@@ -96,7 +98,7 @@ export function eyeTiles(config: PlannerConfig, lonLat: [number, number], stop:
         if (body) early = undefined;
         return body ?? fetch(z, x, y);
     };
-    return { terrain, features, arrived };
+    return { terrain, features, stamps: bakedNodes(config, stop), arrived };
 }
 
 /** The pages of tile bodies, decoded in workers until `stop` aborts. */
@@ -144,3 +146,27 @@ export function featureTiles(config: PlannerConfig, stop = new AbortController()
     }, body => body.buffer, FEATURE_BODIES, body => body.byteLength);
     return async (z, x, y) => (await tiles.tile(z, x, y))?.slice(0);
 }
+
+/**
+ * The nodes of a release's stamp layer: the bytes of node z/x/y as the stamp worker posts them (terrain/node-bytes.ts),
+ * undefined where the layer holds none. Undefined when the release has no stamp layer. A layer of another stamp
+ * version holds none. An abort of `stop` ends every request.
+ */
+export function bakedNodes(config: PlannerConfig, stop: AbortSignal): ((z: number, x: number, y: number) => Promise<Uint8Array | undefined>) | undefined {
+    const url = (config.layers as Record<string, string | undefined>)['trailview-stamps'];
+    if (!url) return undefined;
+    const open = async () => {
+        const opened = await openSource(url);
+        if (opened.metadata.stamp_version === STAMP_VERSION) return opened;
+        console.warn(`The TrailView stamp layer has stamp_version ${opened.metadata.stamp_version}, not ${STAMP_VERSION}. The stamp worker makes every node.`);
+        return undefined;
+    };
+    // A layer that fails to open holds nothing: the stamp worker makes every node.
+    const source = open().catch(() => undefined);
+    return async (z, x, y) => {
+        const opened = await source;
+        if (!opened || z < opened.minZoom || z > opened.maxZoom) return undefined;
+        const body = await opened.get(z, x, y, stop).catch(() => undefined);
+        return body && new Uint8Array(body);
+    };
+}

builder/web/src/lib/trailview/terrain/node-bytes.test.ts

@@ -0,0 +1,35 @@
+import { readFileSync } from 'node:fs';
+import { describe, expect, it } from 'vitest';
+import { decodeNode, encodeNode, unpackNode } from './node-bytes';
+import { OWNER, type Node } from './lod';
+
+const body = ({ level, x, y }: Node) => {
+    const bytes = readFileSync(new URL(`../../../../tests/browser/trailview/fixture/features/${level}/${x}/${y}.bin`, import.meta.url));
+    return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+};
+
+describe('node bytes', () => {
+    it('give back what the stamp worker posts for the coarse nodes and the owner nodes of a tile', async () => {
+        // The stamp worker's global.
+        Object.assign(globalThis, { self: globalThis });
+        const { bakeTile, stamps } = await import('./stamp-bake');
+        for (const tile of [{ level: 11, x: 1071, y: 721 }, { level: 13, x: 4288, y: 2887 }]) {
+            const baked = await bakeTile(tile, body(tile)), held = baked.filter(([node]) => node.level !== tile.level);
+            if (tile.level === 13) {
+                const mask = baked.find(([node]) => node.level === tile.level)![1];
+                expect(mask[0] | (mask[1] << 8)).toBe(held.reduce((bits, [node]) => bits | (1 << ((node.x & 3) + 4 * (node.y & 3))), 0));
+            }
+            expect(held.length).toBeGreaterThan(1);
+            for (const [node, bytes] of held) {
+                const posted = await stamps(tile, node, body(tile)), value = node.level === OWNER ? posted.meshes : posted;
+                expect(await unpackNode(bytes)).toStrictEqual(value);
+                expect(encodeNode(await unpackNode(bytes))).toStrictEqual(encodeNode(value));
+            }
+        }
+    }, 30_000);
+
+    it('refuse a value that JSON changes', () => {
+        for (const value of [{ a: NaN }, [-0], { a: Infinity }, { a: new Map() }, { a: new DataView(new ArrayBuffer(4)) }]) expect(() => encodeNode(value)).toThrow();
+        expect(decodeNode(encodeNode({ a: [1.5, null, true], b: new Float64Array([0.1, -2]), c: new ArrayBuffer(3) }))).toStrictEqual({ a: [1.5, null, true], b: new Float64Array([0.1, -2]), c: new ArrayBuffer(3) });
+    });
+});

builder/web/src/lib/trailview/terrain/node-bytes.ts

@@ -0,0 +1,112 @@
+// The bytes of what the stamp worker posts for a node: a baked node holds them, and the viewer reads them back.
+import { MeshoptDecoder } from 'meshoptimizer/decoder';
+
+/**
+ * The levels whose whole node the stamp layer holds, stamped with the ground plan. It holds the structure
+ * meshes of each owner node that has any, and at zoom 13 the u16 mask of the owner nodes that it holds.
+ */
+export const BAKED = { from: 9, to: 12 };
+
+/** The structure meshes of an owner node without a structure, as the worker posts them. */
+export const EMPTY_MESHES = { far: { structures: null, anchored: null }, near: { structures: null, anchored: null }, anchors: null, points: null };
+
+/**
+ * A mesh `{ vertices, indices, ...rest }` in the layer: `$mesh` and the rest. The distinct vertices `v` in the order
+ * of their first use, `w` of `s` bytes each, meshopt-coded; with `xy` their first two int32 words are int16 from
+ * that point. `r` holds, meshopt-coded, the distinct vertex of each of the `n` vertices (null when all differ),
+ * `i` the `ni` indices.
+ */
+export interface PackedMesh { n: number; w: number; s: number; xy: [number, number] | null; v: Uint8Array; r: Uint8Array | null; ni: number; i: Uint8Array }
+
+/** The typed arrays that the bytes hold, by their tags. */
+const ARRAYS = { Uint8Array, Uint16Array, Uint32Array, Int32Array, Float32Array, Float64Array };
+type Tag = keyof typeof ARRAYS | 'ArrayBuffer';
+/** A binary value in the JSON: its tag, its offset in the binary part and its bytes. */
+interface Slot { $: Tag; at: number; n: number }
+
+const align = (n: number) => (n + 7) & ~7;
+
+/**
+ * The bytes of `value`, plain objects, arrays, booleans, null, finite numbers, strings, ArrayBuffers and
+ * typed arrays: the length of the JSON as a little-endian u32, the JSON, then each binary value at an
+ * offset of 8 bytes. A value that JSON changes (NaN, ±Infinity, -0, a class instance) throws: decodeNode gives back `value` exactly.
+ */
+export function encodeNode(value: unknown): Uint8Array<ArrayBuffer> {
+    const parts: Uint8Array[] = [];
+    let end = 0;
+    const json = new TextEncoder().encode(JSON.stringify(value, (_, item: unknown) => {
+        if (typeof item === 'number' && (!Number.isFinite(item) || Object.is(item, -0))) throw new Error(`encodeNode: ${Object.is(item, -0) ? '-0' : item}`);
+        const tag = item instanceof ArrayBuffer ? 'ArrayBuffer' : ArrayBuffer.isView(item) ? item.constructor.name : undefined;
+        if (!tag && item !== null && typeof item === 'object' && !Array.isArray(item) && Object.getPrototypeOf(item) !== Object.prototype) throw new Error(`encodeNode: ${item.constructor.name}`);
+        if (!tag) return item;
+        if (tag !== 'ArrayBuffer' && !Object.hasOwn(ARRAYS, tag)) throw new Error(`encodeNode: ${tag}`);
+        const bytes = item instanceof ArrayBuffer ? new Uint8Array(item) : new Uint8Array((item as ArrayBufferView).buffer, (item as ArrayBufferView).byteOffset, (item as ArrayBufferView).byteLength);
+        const slot: Slot = { $: tag as Tag, at: end, n: bytes.byteLength };
+        parts.push(bytes);
+        end = align(end + bytes.byteLength);
+        return slot;
+    }));
+    const base = align(4 + json.byteLength), out = new Uint8Array(base + end);
+    new DataView(out.buffer).setUint32(0, json.byteLength, true);
+    out.set(json, 4);
+    let at = base;
+    for (const bytes of parts) {
+        out.set(bytes, at);
+        at += align(bytes.byteLength);
+    }
+    return out;
+}
+
+/** The value of bytes from encodeNode. Each binary value has an ArrayBuffer of its own. */
+export function decodeNode(bytes: Uint8Array): unknown {
+    const length = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(0, true), base = bytes.byteOffset + align(4 + length);
+    const json = new TextDecoder().decode(bytes.subarray(4, 4 + length));
+    return JSON.parse(json, (_, item) => {
+        if (item === null || typeof item !== 'object' || typeof item.$ !== 'string') return item;
+        const { $, at, n } = item as Slot, buffer = bytes.buffer.slice(base + at, base + at + n) as ArrayBuffer;
+        return $ === 'ArrayBuffer' ? buffer : new ARRAYS[$](buffer);
+    });
+}
+
+/** A sequence of u32 from meshopt's index sequence code. */
+function sequence(count: number, code: Uint8Array): Uint32Array<ArrayBuffer> {
+    const out = new Uint32Array(count);
+    MeshoptDecoder.decodeIndexSequence(new Uint8Array(out.buffer), count, 4, code);
+    return out;
+}
+
+/** The mesh that a PackedMesh holds: the bytes of its vertices and its indices as the worker made them. */
+function unpackMesh({ n, w, s, xy, v, r, ni, i }: PackedMesh): { vertices: ArrayBuffer; indices: Uint32Array<ArrayBuffer> } {
+    const distinct = new Uint8Array(w * s);
+    MeshoptDecoder.decodeVertexBuffer(distinct, w, s, v);
+    const stride = xy ? s + 4 : s, wide = xy ? new Uint8Array(w * stride) : distinct;
+    if (xy) {
+        const from = new DataView(distinct.buffer), to = new DataView(wide.buffer);
+        for (let k = 0; k < w; k++) {
+            to.setInt32(k * stride, xy[0] + from.getInt16(k * s, true), true);
+            to.setInt32(k * stride + 4, xy[1] + from.getInt16(k * s + 2, true), true);
+            wide.set(distinct.subarray(k * s + 4, (k + 1) * s), k * stride + 8);
+        }
+    }
+    let vertices = wide;
+    if (r) {
+        const of = sequence(n, r);
+        vertices = new Uint8Array(n * stride);
+        for (let k = 0; k < n; k++) vertices.set(wide.subarray(of[k] * stride, (of[k] + 1) * stride), k * stride);
+    }
+    return { vertices: vertices.buffer, indices: sequence(ni, i) };
+}
+
+/** The value of a baked node: decodeNode, with each packed mesh as the worker posts it. */
+export async function unpackNode(bytes: Uint8Array): Promise<unknown> {
+    await MeshoptDecoder.ready;
+    const value = decodeNode(bytes);
+    const walk = (item: unknown): unknown => {
+        if (item === null || typeof item !== 'object' || ArrayBuffer.isView(item) || item instanceof ArrayBuffer) return item;
+        if (Array.isArray(item)) return item.map(walk);
+        const { $mesh, ...rest } = item as { $mesh?: PackedMesh };
+        if ($mesh) return { ...unpackMesh($mesh), ...rest };
+        return Object.fromEntries(Object.entries(item).map(([key, next]) => [key, walk(next)]));
+    };
+    return walk(value);
+}