feat(search)!: make path the whole statement of what projection reads

packages/search-pipeline/test/multi-collection.integration.test.ts

@@ -31,12 +31,22 @@ const schema = searchSchema(
   {
     name: 'Dataset',
     class: DATASET,
-    fields: [{ name: 'title', kind: 'keyword', path: TITLE, array: true }],
+    fields: [
+      {
+        name: 'title',
+        kind: 'keyword',
+        path: TITLE,
+        array: true,
+        output: true,
+      },
+    ],
   },
   {
     name: 'Organization',
     class: ORGANIZATION,
-    fields: [{ name: 'name', kind: 'keyword', path: NAME, array: true }],
+    fields: [
+      { name: 'name', kind: 'keyword', path: NAME, array: true, output: true },
+    ],
   },
 );
 

packages/search-pipeline/test/search-stages.test.ts

@@ -22,7 +22,7 @@ const NAME = 'https://example.org/name';
 const schema = searchSchema({
   name: 'Person',
   class: PERSON,
-  fields: [{ name: 'name', kind: 'keyword', path: NAME }],
+  fields: [{ name: 'name', kind: 'keyword', path: NAME, output: true }],
 });
 const person = schema.get(PERSON) as SearchType;
 
@@ -115,7 +115,7 @@ describe('searchStages', () => {
     const lookalike: SearchType = {
       name: 'Person',
       class: PERSON,
-      fields: [{ name: 'name', kind: 'keyword', path: NAME }],
+      fields: [{ name: 'name', kind: 'keyword', path: NAME, output: true }],
     };
     const [stage] = searchStages({
       schema,

packages/search-typesense/src/collection-definition.ts

@@ -1,7 +1,11 @@
 import type { CollectionCreateSchema } from 'typesense';
 import type { CollectionFieldSchema } from 'typesense/lib/Typesense/Collection.js';
 import { type SearchField, type SearchType } from '@lde/search';
-import { displayFieldPattern, physicalFields } from '@lde/search/adapter';
+import {
+  displayFieldPattern,
+  isInternalField,
+  physicalFields,
+} from '@lde/search/adapter';
 import { deriveCollectionName } from './collection-name.js';
 
 /** Deployment-specific options the generic field model does not carry. */
@@ -46,6 +50,10 @@ export interface CollectionDefinitionOptions {
  * only the folded `*_search_${locale}`, facet/reference and `*_sort_${locale}`
  * companions are indexed. Keeping retrieval-only fields un-indexed is the lever
  * for holding a large index’s RAM down.
+ *
+ * {@link isInternalField Internal fields} (those declaring no role) are omitted
+ * entirely: they exist only as a projection-time reading device for the
+ * derives, so the collection stores nothing for them.
  */
 export function buildCollectionDefinition(
   searchType: SearchType,
@@ -73,6 +81,13 @@ function typesenseFields(
   defaultLocale: string | undefined,
   defaultSortingField: string | undefined,
 ): CollectionFieldSchema[] {
+  // An internal field (no role) is projected as a reading device for the
+  // derives and pruned before the writer, so the collection stores nothing for
+  // it: not stored, not indexed, no RAM. Same predicate the projection prunes
+  // by, so the index and the document cannot disagree.
+  if (isInternalField(field)) {
+    return [];
+  }
   const names = physicalFields(field);
   if (field.kind === 'text') {
     const locales = field.locales;

packages/search-typesense/test/blue-green-rebuild.test.ts

@@ -13,8 +13,8 @@ const datasetType: SearchType = {
   name: 'Dataset',
   class: 'https://example.org/Dataset',
   fields: [
-    { name: 'title', kind: 'keyword' },
-    { name: 'year', kind: 'integer' },
+    { name: 'title', kind: 'keyword', output: true },
+    { name: 'year', kind: 'integer', facetable: true },
   ],
 };
 

packages/search-typesense/test/collection-definition.test.ts

@@ -266,3 +266,56 @@ describe('und-locale text', () => {
     ]);
   });
 });
+
+describe('internal fields', () => {
+  it('omits an internal (zero-role) field of every kind from the collection', () => {
+    const collection = buildCollectionDefinition(
+      {
+        name: 'Doc',
+        class: 'urn:example:Doc',
+        fields: [
+          { name: 'token', path: 'urn:ex:token', kind: 'keyword' },
+          { name: 'ref', path: 'urn:ex:ref', kind: 'reference' },
+          { name: 'count', path: 'urn:ex:count', kind: 'integer' },
+          { name: 'score', path: 'urn:ex:score', kind: 'number' },
+          { name: 'flag', path: 'urn:ex:flag', kind: 'boolean' },
+          { name: 'note', path: 'urn:ex:note', kind: 'text', locales: ['nl'] },
+        ],
+      },
+      { name: 'docs' },
+    );
+    // Every field declares no role, so it is internal – a projection-time
+    // reading device, pruned before the writer. The collection stores nothing
+    // for any of them: not stored, not indexed, no RAM.
+    expect(collection.fields).toEqual([]);
+  });
+
+  it('keeps a field the moment it declares any role', () => {
+    const collection = buildCollectionDefinition(
+      {
+        name: 'Doc',
+        class: 'urn:example:Doc',
+        fields: [
+          { name: 'hidden', path: 'urn:ex:hidden', kind: 'keyword' },
+          {
+            name: 'shown',
+            path: 'urn:ex:shown',
+            kind: 'keyword',
+            facetable: true,
+          },
+        ],
+      },
+      { name: 'docs' },
+    );
+    // Only the field carrying a role reaches the collection.
+    expect(collection.fields).toEqual([
+      {
+        name: 'shown',
+        type: 'string',
+        facet: true,
+        sort: false,
+        optional: true,
+      },
+    ]);
+  });
+});

packages/search-typesense/test/in-place-rebuild.test.ts

@@ -12,7 +12,7 @@ const NAME = 'objects';
 const objectType: SearchType = {
   name: 'Object',
   class: 'https://example.org/Object',
-  fields: [{ name: 'title', kind: 'keyword' }],
+  fields: [{ name: 'title', kind: 'keyword', output: true }],
 };
 
 const datasetA = new Dataset({

packages/search-typesense/vite.config.ts

@@ -20,7 +20,7 @@ export default mergeConfig(
           // partial vitest run must never rewrite these – see AGENTS.md.
           functions: 98.65,
           lines: 98.83,
-          branches: 93.57,
+          branches: 93.61,
           statements: 98.85,
         },
       },

packages/search/README.md

@@ -58,8 +58,7 @@ a typed, deterministic function – easy to test, and swappable per deployment.
 Exports are stratified by audience:
 
 - **`@lde/search`** – the authoring surface: `defineSearchType`,
-  `searchSchema`, `projectRoots` (+ the IR readers `derive` functions use),
-  validation, and every model/query/result type.
+  `searchSchema`, `projectRoots`, validation, and every model/query/result type.
 - **`@lde/search/adapter`** – plumbing for engine adapters and API surfaces:
   `physicalFields`, the field selectors, `assertValidQuery`, the filter
   operators and storage codecs.
@@ -101,20 +100,19 @@ Two conventions hold across the whole family:
 ## Field model
 
 The mapping is data, not code. Each field declares its `kind`, the IR `path` to
-read (or a `derive` function for a **derived** field, computed from the framed
-node in declaration order – so it may read fields declared before it), and the
-capabilities it opts into. The physical field names a declaration fans out to
-(per-locale search/sort keys) come from
-`physicalFields`, the single convention projection, the collection definition and the
-query compiler all share.
+read (or a `derive` function for a **derived** field, computed from the document
+in declaration order – so it may read fields declared before it, never the
+graph), and the capabilities (**roles**) it opts into. `path` is therefore the
+complete statement of what the projection reads from the graph. A field that
+declares **no** role is an **internal field**: projected so a later `derive` can
+read it, then pruned before the writer and absent from the collection definition
+– not stored, not indexed, no RAM. The physical field names a declaration fans
+out to (per-locale search/sort keys) come from `physicalFields`, the single
+convention projection, the collection definition and the query compiler all
+share.
 
 ```ts
-import {
-  defineSearchType,
-  projectRoots,
-  irisOf,
-  searchSchema,
-} from '@lde/search';
+import { defineSearchType, projectRoots, searchSchema } from '@lde/search';
 
 const DATASET = defineSearchType({
   name: 'Dataset', // logical API name: names the GraphQL type, a REST path, …
@@ -141,12 +139,17 @@ const DATASET = defineSearchType({
     },
     // → size (int)
     { name: 'size', path: 'urn:dr:size', kind: 'integer', sortable: true },
-    // derived field (no path): computed from the framed node
+    // internal field (no role): projected as a reading device for the derive
+    // below, then pruned before the writer – absent from the collection too
+    { name: 'classes', path: 'urn:dr:class', kind: 'reference' },
+    // derived field (no path): computed from the document in declaration order,
+    // never from the graph – so `path` stays the whole statement of what is read
     {
       name: 'classCount',
       kind: 'integer',
       sortable: true,
-      derive: (node) => irisOf(node, 'urn:dr:class').length,
+      derive: (document) =>
+        (document.classes as string[] | undefined)?.length ?? 0,
     },
   ],
 });
@@ -225,8 +228,9 @@ each root’s subgraph is framed one at a time and its document yielded as
 produced, so beyond a subject index memory stays flat at scale (framing the
 whole graph at once is roughly O(N²)). Duplicate triples are collapsed first,
 because some SPARQL engines (e.g. QLever) do not deduplicate `CONSTRUCT` output.
-The IR carries no `@context`, so a `derive` function reading it sees full
-predicate IRIs with language tags preserved. `assertTypeInSchema` guards that
+Every predicate a value comes from is read through a field’s `path`; a `derive`
+computes only from the document projected so far, so `path` is the whole
+statement of what the projection reads. `assertTypeInSchema` guards that
 the passed `SearchType` is a member of the schema – the port’s own membership
 check – so no schema is ever forged to scope a projection to one type.
 

packages/search/src/adapter.ts

@@ -17,6 +17,7 @@ export {
   filterableFields,
   sortableFields,
   outputFields,
+  isInternalField,
   referenceFields,
   fieldNamed,
   labelFieldOf,

packages/search/src/index.ts

@@ -5,9 +5,9 @@
 // `@lde/search/testing`.
 
 // Projection: RDF CONSTRUCT quads → flat search documents, driven by the
-// unified SearchField/SearchType model. The IR readers (irisOf, …) are here
-// because `derive` functions are written against them.
-export { projectRoots, irisOf, literalsOf, firstLiteralOf } from './project.js';
+// unified SearchField/SearchType model. A `derive` reads the projected document,
+// never the graph, so the IR readers stay internal to projection.
+export { projectRoots } from './project.js';
 export type { SearchDocument } from './project.js';
 
 // Unified field model: one declaration drives projection, engine collection

packages/search/src/project.ts

@@ -8,6 +8,7 @@ import {
 import {
   assertTypeInSchema,
   displayFieldName,
+  isInternalField,
   isoToUnixSeconds,
   physicalFields,
   type KeywordField,
@@ -24,10 +25,14 @@ export type SearchDocument = { id: string } & Record<string, unknown>;
 /**
  * Project one framed JSON-LD node into a flat search document: apply each field
  * of the type in declaration order. A field with a `derive` function computes
- * its value from the node and the document as populated so far (so a derived
- * field may read fields declared before it). The physical field names a field
- * fans out to come from {@link physicalFields}, the single source shared with
- * the engine collection definition and the query compiler.
+ * its value from the document as populated so far (so a derived field may read
+ * fields declared before it), never from the graph – `path` is the complete
+ * statement of what the projection reads. {@link isInternalField Internal
+ * fields} (those declaring no role) are populated so a later derive can read
+ * them, then pruned before the document is returned: they must reach neither a
+ * writer nor the collection definition. The physical field names a field fans
+ * out to come from {@link physicalFields}, the single source shared with the
+ * engine collection definition and the query compiler.
  */
 export function projectDocument(
   node: FramedNode,
@@ -43,6 +48,14 @@ export function projectDocument(
   for (const field of searchType.fields) {
     applyField(document, node, field);
   }
+  // Prune internal fields: they were projected only so the derives above could
+  // read them (a structural memo shared across every derive that needs the
+  // value), and must not reach the writer or the collection definition.
+  for (const field of searchType.fields) {
+    if (isInternalField(field)) {
+      delete document[field.name];
+    }
+  }
   return document;
 }
 
@@ -82,7 +95,7 @@ function applyField(
   field: SearchField,
 ): void {
   if (field.derive !== undefined) {
-    const value = field.derive(node, document);
+    const value = field.derive(document);
     if (value !== undefined) {
       document[field.name] = value;
     }
@@ -209,7 +222,9 @@ function applyFacet(
   }
 }
 
-// --- Framed-IR readers (exported so derivations can read arbitrary paths) ---
+// --- Framed-IR readers: read a declared `path` off the framed node. Internal
+// to projection – a `derive` reads the projected document, never the node, so
+// `path` stays the whole statement of what the projection reads from the graph.
 
 /** A literal value with its (possibly empty) language tag. */
 interface LangValue {
@@ -223,20 +238,17 @@ function langValuesOf(node: FramedNode, path: string): LangValue[] {
     .filter((value): value is LangValue => value !== undefined);
 }
 
-export function literalsOf(node: FramedNode, path: string): string[] {
+function literalsOf(node: FramedNode, path: string): string[] {
   return valuesOf(node, path)
     .map(literalString)
     .filter((value): value is string => value !== undefined);
 }
 
-export function firstLiteralOf(
-  node: FramedNode,
-  path: string,
-): string | undefined {
+function firstLiteralOf(node: FramedNode, path: string): string | undefined {
   return literalsOf(node, path)[0];
 }
 
-export function irisOf(node: FramedNode, path: string): string[] {
+function irisOf(node: FramedNode, path: string): string[] {
   return valuesOf(node, path)
     .map(iriString)
     .filter((value): value is string => value !== undefined);