diff --git a/AGENTS.md b/AGENTS.md index 91278590..0024a2b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,11 +82,11 @@ Each package uses conditional exports with a `development` condition for local d - Test files use `.test.ts` suffix in `test/` directory - Fixtures in `test/fixtures/` - HTTP mocking with Nock -- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005–3006) +- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005–3006), `search-pipeline` extraction round-trip (3007) ### Key Dependencies -- RDF: `n3`, `sparqljs`, `jsonld` +- RDF: `n3`, `jsonld` - Query engines: `@comunica/query-sparql-file`, `ldkit` - CLI packages use Commander diff --git a/packages/search-pipeline/README.md b/packages/search-pipeline/README.md index 2ad6b74d..43e28907 100644 --- a/packages/search-pipeline/README.md +++ b/packages/search-pipeline/README.md @@ -21,7 +21,12 @@ The division of labour ([ADR 6](../../docs/decisions/0006-make-the-writer-transa **root-complete batch** into documents paired with their `SearchType` (`TypedSearchDocument`) – so projection happens inside the batch and memory is bounded by `batchSize` roots, not the dataset - ([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)); + ([ADR 13](../../docs/decisions/0013-project-inside-the-batch-per-root-type.md)). + Extraction is **generated from the schema**: `extractionQuery` mints a + CONSTRUCT that reads each field’s source `path` (a SPARQL property path) and + emits it under the field’s IR Alias (`urn:lde:‹Type›/‹field›`) – the same key + the projection reads back – so the reader and the projection cannot drift. A + stage without an explicit `readers` defaults to this generated reader; - `searchIndexWriter` is the pipeline’s single terminal: an engine-agnostic router that dispatches each document to the engine writer for **its** type’s collection ([ADR 9](../../docs/decisions/0009-route-a-whole-schema-projection-to-per-type-collections.md)). @@ -32,7 +37,7 @@ So a search pipeline is **one terminal, N stages**: `new Pipeline({ // One stage per root type: each selects its own roots (here by class – // `selectByClass` is a convenience for the object grain; a deployment writes // its own selector where the entry point is a domain fact) and extracts each - // root’s quads. `rootVariable` couples the selector’s projected variable to - // what the projection reads; it must not be `dataset` (reserved by the reader). + // root’s quads. Omitting `readers` defaults each stage to the Extraction + // CONSTRUCT generated from the schema (pass one only for a non-SPARQL source or + // to merge readers). `rootVariable` couples the selector’s projected variable + // to the CONSTRUCT’s free subject; it must not be `dataset` (reserved by the + // reader). stages: searchStages({ schema, types: [ @@ -92,13 +100,11 @@ const pipeline = new Pipeline({ searchType: dataset, rootVariable: 'root', itemSelector: selectByClass(dataset), - readers: new SparqlConstructReader({ query: '…' }), }, { searchType: organization, rootVariable: 'root', itemSelector: selectByClass(organization), - readers: new SparqlConstructReader({ query: '…' }), }, ], }), @@ -180,3 +186,37 @@ across a dataset or across datasets, so the same pipeline indexes a 15-quad catalog entry and a dataset of millions of objects. The one irreducible atom is a single root’s quads: unbounded for a pathological root, and nothing here changes that. + +## Extraction queries and non-deduplicating engines + +The generated extraction CONSTRUCTs are deliberately shaped to be well-formed for +**non-deduplicating** SPARQL engines such as +[QLever](https://github.com/ad-freiburg/qlever), which emit one copy of the +CONSTRUCT template per solution row rather than a deduplicated set. The generator +guarantees one output triple per genuine value: + +- **UNION per field, never conjunction.** Each field is its own `UNION` branch + (`?root ?value`), so two independent multi-valued fields never form a + cross-product – the failure mode that inflates a non-deduplicating engine’s + output by the product of their cardinalities (roughly 18-fold on a measured + QLever workload). +- **No projected-away template triple.** Every template triple binds its own + variables (`?root ?value`); there is no constant triple (e.g. + `?root a `) to be re-emitted once per solution. +- **Given roots.** Roots arrive as a `VALUES` set, not a pattern that fans out. +- **Inline references keep the discipline recursively:** a reference type’s + fields are UNION’d off the referent variable, so even a multi-hop nested + template never conjoins independent multi-valued fields – only the intermediate + link triple repeats, and only linearly. + +Because the queries are duplicate-free by construction, correctness and bounded +output volume do **not** depend on a client-side **post-processing deduplication +step** – buffering a whole result to deduplicate it would defeat the +batch-bounded streaming memory model above. The only residual duplication is +linear and comes from duplicate _input_ – a non-`DISTINCT` selector feeding a +multi-typed root more than once – which the streaming per-quad subject index +(`buildSubjectIndex`) collapses as a cheap backstop, not as a compensator for +query-shape inflation. Validated on a real QLever index over the full source +dump: raw CONSTRUCT output equalled the distinct set for a type with distinct +roots, and the projected documents were byte-identical to those from a +deduplicating engine. diff --git a/packages/search-pipeline/package.json b/packages/search-pipeline/package.json index 1b30333b..3228cd06 100644 --- a/packages/search-pipeline/package.json +++ b/packages/search-pipeline/package.json @@ -26,6 +26,9 @@ ], "dependencies": { "@lde/search": "^0.11.0", + "@traqula/generator-sparql-1-1": "^1.1.8", + "@traqula/parser-sparql-1-1": "^1.1.5", + "@traqula/rules-sparql-1-1": "^1.1.0", "tslib": "^2.3.0" }, "devDependencies": { diff --git a/packages/search-pipeline/src/extraction.ts b/packages/search-pipeline/src/extraction.ts new file mode 100644 index 00000000..9f97b9ec --- /dev/null +++ b/packages/search-pipeline/src/extraction.ts @@ -0,0 +1,189 @@ +import { + AstFactory, + type Pattern, + type PatternBgp, + type PatternGroup, + type QueryConstruct, + type QuerySelect, + type TermVariable, + type TripleNesting, +} from '@traqula/rules-sparql-1-1'; +import { Parser } from '@traqula/parser-sparql-1-1'; +import { Generator } from '@traqula/generator-sparql-1-1'; +import { + irAlias, + isInlineReference, + referenceTypeNamed, +} from '@lde/search/adapter'; +import type { SearchSchema, SearchType } from '@lde/search'; + +const factory = new AstFactory(); +const parser = new Parser(); +const generator = new Generator(); + +/** Options for {@link extractionQuery}. */ +export interface ExtractionOptions { + /** + * The variable the roots bind to. Left free in the generated query for the + * pipeline’s VALUES injection (`injectValues`), so it must match the stage’s + * `rootVariable` – the variable the item selector projects. Defaults to + * `root`, matching `selectByClass`. Must not be `dataset` (the SPARQL reader + * substitutes `?dataset` with the dataset IRI). + * @default 'root' + */ + readonly subjectVariable?: string; +} + +/** + * Generate a type’s **Extraction** CONSTRUCT from its {@link SearchType} + * declaration: the query whose output the projection frames and reads. Pure – + * `SearchType → QueryConstruct` – with no engine or deployment knowledge. + * + * - **template**: one `?root <`{@link irAlias IR Alias}`> ?value` triple per + * path-bearing field – IRIs only, a single subject. A property path cannot be + * a CONSTRUCT verb, so the flattened value must be minted onto its subject + * under the field’s alias; the projection reads it back under the same alias. + * - **WHERE**: one UNION branch per field (`?root ?value`), the field’s + * `path` embedded as a literal SPARQL property path (sequence / alternative / + * inverse). UNION per branch, never a conjunction – conjoining every path in + * one BGP cross-multiplies the multi-valued ones (~4× inflation). + * - **roots**: the subject variable is left free, to be bound by the pipeline’s + * `injectValues` (`VALUES ?root { … }`). Root *selection* is a separate, + * deployment concern (`selectByClass` / the stage’s `itemSelector`); this + * generator emits IR Aliases, the selector queries the source class. + * - **inline references** ({@link isInlineReference}): one CONSTRUCT with a + * nested template (`{ ?root <…/ref> ?r . ?r <…/field> ?v }`), recursing into + * the reference type to the schema’s declared depth. The referent-binding hop + * uses the source `path`; the emitted triples use the minted aliases. + * + * Wire the result into a `SparqlConstructReader` (see `searchStages`), which + * runs it per batch with the roots injected as VALUES. + */ +export function extractionQuery( + searchType: SearchType, + schema: SearchSchema, + options: ExtractionOptions = {}, +): QueryConstruct { + const subjectVariable = options.subjectVariable ?? 'root'; + const subject = factory.termVariable(subjectVariable, factory.gen()); + const built = buildFor(searchType, subject, schema, { next: 0 }); + if (built.branches.length === 0) { + throw new Error( + `Cannot generate an extraction CONSTRUCT for “${searchType.name}”: it declares no path-bearing field, so there is nothing to extract.`, + ); + } + const where = factory.patternGroup( + [factory.patternUnion(built.branches, factory.gen())], + factory.gen(), + ); + return factory.queryConstruct( + factory.gen(), + [], + factory.patternBgp(built.template, factory.gen()), + where, + {}, + factory.datasetClauses([], factory.gen()), + ); +} + +/** {@link extractionQuery}, serialised to a SPARQL string a reader can run. */ +export function extractionQueryString( + searchType: SearchType, + schema: SearchSchema, + options?: ExtractionOptions, +): string { + return generator.generate(extractionQuery(searchType, schema, options)); +} + +/** A per-query counter minting distinct value/referent variable names. */ +interface VariableCounter { + next: number; +} + +/** The template triples and WHERE branches a type contributes off `subject`. */ +interface Built { + readonly template: TripleNesting[]; + readonly branches: PatternGroup[]; +} + +/** + * Walk a type’s path-bearing fields off a subject variable, collecting the + * template triples (minted aliases) and one WHERE branch per field (source + * paths). Recurses into an inline reference’s type off a fresh referent + * variable, so the nested template keeps the `subject → referent → value` link + * in one CONSTRUCT. The recursion terminates because `searchSchema` rejects + * inline reference cycles. + */ +function buildFor( + searchType: SearchType, + subject: TermVariable, + schema: SearchSchema, + counter: VariableCounter, +): Built { + const template: TripleNesting[] = []; + const branches: PatternGroup[] = []; + for (const field of searchType.fields) { + if (field.path === undefined) { + continue; + } + const alias = factory.termNamed(factory.gen(), irAlias(searchType, field)); + const sourcePath = liftPath(field.path); + if (isInlineReference(field)) { + const referenceType = referenceTypeNamed(schema, field.ref.typeName); + if (referenceType === undefined) { + continue; + } + const referent = factory.termVariable( + `r${counter.next++}`, + factory.gen(), + ); + template.push(factory.triple(subject, alias, referent)); + const nested = buildFor(referenceType, referent, schema, counter); + template.push(...nested.template); + const patterns: Pattern[] = [ + factory.patternBgp( + [factory.triple(subject, sourcePath, referent)], + factory.gen(), + ), + ]; + // A single-branch union serialises to just its group, so only add the + // nested union when the reference type has fields of its own. + if (nested.branches.length > 0) { + patterns.push(factory.patternUnion(nested.branches, factory.gen())); + } + branches.push(factory.patternGroup(patterns, factory.gen())); + } else { + const value = factory.termVariable(`v${counter.next++}`, factory.gen()); + template.push(factory.triple(subject, alias, value)); + branches.push( + factory.patternGroup( + [ + factory.patternBgp( + [factory.triple(subject, sourcePath, value)], + factory.gen(), + ), + ], + factory.gen(), + ), + ); + } + } + return { template, branches }; +} + +/** + * Lift a field’s `path` – written in the SPARQL reader adapter’s grammar (a + * property path) – into a predicate AST node, by parsing it inside a throwaway + * query and taking the verb. A single IRI yields a plain named node; a + * sequence / alternative / inverse yields a path node. The WHERE consumes it; + * the CONSTRUCT template never does (a path cannot be a template verb). + */ +function liftPath(path: string): TripleNesting['predicate'] { + // Always a single-triple SELECT by construction, so the verb is at a fixed + // spot; a malformed `path` makes the parser itself throw, which is the right + // failure for invalid reader-adapter grammar. + const ast = parser.parse(`SELECT * WHERE { ?s ${path} ?o }`) as QuerySelect; + const bgp = ast.where.patterns[0] as PatternBgp; + const triple = bgp.triples[0] as TripleNesting; + return triple.predicate; +} diff --git a/packages/search-pipeline/src/index.ts b/packages/search-pipeline/src/index.ts index 0a611a18..7cb263a0 100644 --- a/packages/search-pipeline/src/index.ts +++ b/packages/search-pipeline/src/index.ts @@ -1,5 +1,7 @@ export { searchStages, selectByClass } from './search-stages.js'; export type { SearchStagesOptions, SearchStageType } from './search-stages.js'; +export { extractionQuery, extractionQueryString } from './extraction.js'; +export type { ExtractionOptions } from './extraction.js'; export { searchIndexWriter } from './search-index-writer.js'; export type { SearchIndexWriterOptions } from './search-index-writer.js'; export type { TypedSearchDocument } from './typed-search-document.js'; diff --git a/packages/search-pipeline/src/search-stages.ts b/packages/search-pipeline/src/search-stages.ts index c97ca585..56ad078d 100644 --- a/packages/search-pipeline/src/search-stages.ts +++ b/packages/search-pipeline/src/search-stages.ts @@ -1,10 +1,12 @@ import { + SparqlConstructReader, SparqlItemSelector, Stage, type ItemSelector, type StageReaders, } from '@lde/pipeline'; import { projectRoots, type RootType, type SearchSchema } from '@lde/search'; +import { extractionQueryString } from './extraction.js'; import type { TypedSearchDocument } from './typed-search-document.js'; /** One root type’s stage in a search pipeline. */ @@ -30,8 +32,16 @@ export interface SearchStageType { * convenience for the object grain, not a default. */ itemSelector: ItemSelector; - /** Reader(s) that extract each selected root’s quads. */ - readers: StageReaders; + /** + * Reader(s) that extract each selected root’s quads. Defaults to a single + * {@link SparqlConstructReader} running the **Extraction** CONSTRUCT generated + * from `searchType` ({@link extractionQuery}), with `rootVariable` as the free + * subject the batch’s roots bind to – the schema-derived reader the projection + * is guaranteed to agree with (they share the {@link irAlias} convention). + * Supply your own only to read from a non-SPARQL source, merge several + * readers, or attach transforms. + */ + readers?: StageReaders; /** * Roots (and so documents) per batch – the memory bound. Under a root-bound * selector it moves memory and request count, never output. @@ -88,9 +98,19 @@ export function searchStages( ); } const { rootVariable } = type; + // Default to the Extraction CONSTRUCT generated from the schema, its subject + // left free for the batch’s VALUES injection. The reader and the projection + // then agree by construction: both key off the same IR Aliases. + const readers = + type.readers ?? + new SparqlConstructReader({ + query: extractionQueryString(searchType, schema, { + subjectVariable: rootVariable, + }), + }); return new Stage({ name: searchType.name, - readers: type.readers, + readers, itemSelector: type.itemSelector, batchSize: type.batchSize, maxConcurrency: type.maxConcurrency, diff --git a/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts b/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts new file mode 100644 index 00000000..dd70c076 --- /dev/null +++ b/packages/search-pipeline/test/extraction-roundtrip.integration.test.ts @@ -0,0 +1,147 @@ +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { Dataset, Distribution } from '@lde/dataset'; +import type { DatasetWriter } from '@lde/pipeline'; +import { defineSearchType, searchSchema } from '@lde/search'; +import { + startSparqlEndpoint, + teardownSparqlEndpoint, +} from '@lde/local-sparql-endpoint'; +import { searchStages, selectByClass } from '../src/search-stages.js'; +import type { TypedSearchDocument } from '../src/typed-search-document.js'; + +const SCHEMA = 'https://schema.org/'; + +// A Drapo-shaped schema (see fixtures/drapo-sample.ttl): CreativeWork with two +// localized text fields and a labelOnly creator reference; Person resolves the +// creator labels. Paths are single predicates in the reader-adapter grammar. +const person = defineSearchType({ + name: 'Person', + class: `${SCHEMA}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 3 }, + }, + ], +}); + +const creativeWork = defineSearchType({ + name: 'CreativeWork', + class: `${SCHEMA}CreativeWork`, + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['nl', 'en'], + output: true, + searchable: { weight: 5 }, + sortable: true, + }, + { + name: 'description', + kind: 'text', + path: `<${SCHEMA}description>`, + locales: ['nl'], + output: true, + searchable: { weight: 2 }, + }, + { + name: 'creator', + kind: 'reference', + path: `<${SCHEMA}creator>`, + labelSource: 'Person', + facetable: true, + output: true, + ref: { typeName: 'Person', strategy: 'labelOnly' }, + }, + ], +}); + +const schema = searchSchema(creativeWork, person); + +describe('extraction round-trip: generate → read → frame → project', () => { + const port = 3007; + const distribution = Distribution.sparql( + new URL(`http://localhost:${port}/sparql`), + ); + const dataset = new Dataset({ + iri: new URL('http://example.org/dataset/drapo'), + distributions: [distribution], + }); + + beforeAll(async () => { + // Absolute path so the endpoint finds the fixture regardless of the cwd the + // runner spawns it from. + const fixture = fileURLToPath( + new URL('./fixtures/drapo-sample.ttl', import.meta.url), + ); + await startSparqlEndpoint(port, fixture); + }, 60_000); + + afterAll(async () => { + await teardownSparqlEndpoint(); + }); + + async function runStage(): Promise { + // No `readers`: the stage defaults to the generated Extraction CONSTRUCT, + // proving the schema-derived reader and the projection agree end to end + // against a real SPARQL engine, over roots selected by `selectByClass`. + const [stage] = searchStages({ + schema, + types: [ + { + searchType: creativeWork, + rootVariable: 'root', + itemSelector: selectByClass(creativeWork), + }, + ], + }); + const received: TypedSearchDocument[] = []; + const writer: DatasetWriter = { + write: async (_dataset, items) => { + for await (const item of items) { + received.push(item); + } + }, + }; + await stage.run(dataset, distribution, writer); + return received; + } + + it('projects each selected CreativeWork root into its search document', async () => { + const received = await runStage(); + + for (const item of received) { + expect(item.searchType).toBe(creativeWork); + } + const byId = Object.fromEntries( + received.map((item) => [item.document.id, item.document]), + ); + expect(Object.keys(byId).sort()).toEqual([ + 'https://ex/cw/1', + 'https://ex/cw/2', + ]); + + // The localized name flattened per locale, folded into the search field… + const first = byId['https://ex/cw/1']; + expect(first.name_nl).toBe('Het meisje met de parel'); + expect(first.name_en).toBe('Girl with a Pearl Earring'); + expect(first.name_search_nl).toBe('het meisje met de parel'); + expect(first.description_nl).toBe('Een schilderij van Johannes Vermeer.'); + // …and the labelOnly creator carried as its bare IRI (label resolved at + // query time from the Person collection, not here). + expect(first.creator).toEqual(['https://ex/p/1']); + + // A root with only a name still projects, with the optional fields absent. + const second = byId['https://ex/cw/2']; + expect(second.name_nl).toBe('De nachtwacht'); + expect(second).not.toHaveProperty('creator'); + expect(second).not.toHaveProperty('description_nl'); + }); +}); diff --git a/packages/search-pipeline/test/extraction.test.ts b/packages/search-pipeline/test/extraction.test.ts new file mode 100644 index 00000000..d47583ca --- /dev/null +++ b/packages/search-pipeline/test/extraction.test.ts @@ -0,0 +1,359 @@ +import { describe, expect, it } from 'vitest'; +import { Parser } from '@traqula/parser-sparql-1-1'; +import { + AstFactory, + type Path, + type PatternGroup, + type QueryConstruct, + type TripleNesting, +} from '@traqula/rules-sparql-1-1'; +import { defineSearchType, searchSchema } from '@lde/search'; +import { irAlias, referenceTypeNamed } from '@lde/search/adapter'; +import type { SearchSchema, SearchType } from '@lde/search'; +import { extractionQuery, extractionQueryString } from '../src/extraction.js'; + +const SCHEMA = 'https://schema.org/'; +const factory = new AstFactory(); + +// A Drapo-shaped SCHEMA-AP-NDE schema: CreativeWork (two localized text fields +// and a labelOnly creator reference) resolving Person for its creator labels. +// Every path is a single predicate – the shape the Drapo dump exercises. +const person = defineSearchType({ + name: 'Person', + class: `${SCHEMA}Person`, + fields: [ + { + name: 'label', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['nl', 'und'], + output: true, + searchable: { weight: 3 }, + }, + ], +}); + +const creativeWork = defineSearchType({ + name: 'CreativeWork', + class: `${SCHEMA}CreativeWork`, + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['nl', 'und'], + output: true, + searchable: { weight: 5 }, + }, + { + name: 'description', + kind: 'text', + path: `<${SCHEMA}description>`, + locales: ['nl', 'und'], + output: true, + searchable: { weight: 2 }, + }, + { + name: 'creator', + kind: 'reference', + path: `<${SCHEMA}creator>`, + labelSource: 'Person', + facetable: true, + output: true, + ref: { typeName: 'Person', strategy: 'labelOnly' }, + }, + ], +}); + +const drapoSchema = searchSchema(creativeWork, person); + +/** The template triples, each narrowed from the broader BGP element type. */ +function templateTriples(query: QueryConstruct): TripleNesting[] { + return query.template.triples.map((triple) => { + if (!factory.isTriple(triple)) { + throw new Error('expected a plain triple in the CONSTRUCT template'); + } + return triple; + }); +} + +/** The predicate IRIs (the minted IR Aliases) a CONSTRUCT template emits. */ +function templatePredicates(query: QueryConstruct): string[] { + return templateTriples(query).map((triple) => { + const predicate = triple.predicate; + if (!factory.isTermNamed(predicate)) { + throw new Error('a CONSTRUCT template verb must be a plain IRI'); + } + return predicate.value; + }); +} + +/** The WHERE’s top-level UNION branches (one group per field). */ +function unionBranches(query: QueryConstruct): PatternGroup[] { + const [union] = query.where.patterns; + if (union === undefined || !factory.isPatternUnion(union)) { + throw new Error('expected a UNION at the top of the WHERE'); + } + return union.patterns; +} + +describe('extractionQuery', () => { + it('mints one IR-Alias template triple per path-bearing field, single subject', () => { + const query = extractionQuery(creativeWork, drapoSchema); + + // Template: IRIs only, one triple per field, all off the same ?root subject. + expect(templatePredicates(query)).toEqual([ + irAlias(creativeWork, creativeWork.fields[0]), + irAlias(creativeWork, creativeWork.fields[1]), + irAlias(creativeWork, creativeWork.fields[2]), + ]); + for (const triple of templateTriples(query)) { + expect(triple.subject).toMatchObject({ + subType: 'variable', + value: 'root', + }); + } + }); + + it('reads each field via its source path in a UNION branch, not a conjunction', () => { + // One branch per field – never one BGP conjoining all paths (that + // cross-product is what inflates a multi-valued result ~4×). + expect( + unionBranches(extractionQuery(creativeWork, drapoSchema)), + ).toHaveLength(3); + }); + + it('embeds a multi-hop path as a SPARQL property path (sequence)', () => { + const withPathField = defineSearchType({ + name: 'CreativeWork', + class: `${SCHEMA}CreativeWork`, + fields: [ + { + name: 'publisherName', + kind: 'text', + // A qualified two-hop value no single predicate can address. + path: `<${SCHEMA}publisher>/<${SCHEMA}name>`, + locales: ['und'], + output: true, + }, + ], + }); + const query = extractionQuery(withPathField, searchSchema(withPathField)); + + const [group] = unionBranches(query); + const [bgp] = group.patterns; + if (bgp === undefined || !factory.isPatternBgp(bgp)) { + throw new Error('expected a bgp'); + } + const triple = bgp.triples[0]; + if (!factory.isTriple(triple)) { + throw new Error('expected a triple'); + } + const path = triple.predicate as Path; + expect(path).toMatchObject({ type: 'path', subType: '/' }); + // …while the template verb stays a plain minted IRI (a path cannot be one). + expect(templatePredicates(query)).toEqual([ + irAlias(withPathField, withPathField.fields[0]), + ]); + }); + + it('leaves the root subject free for the pipeline VALUES injection', () => { + const query = extractionQuery(creativeWork, drapoSchema, { + subjectVariable: 'item', + }); + for (const triple of templateTriples(query)) { + expect(triple.subject).toMatchObject({ + subType: 'variable', + value: 'item', + }); + } + // Defaults to `root`, matching selectByClass’s default binding. + expect(extractionQueryString(creativeWork, drapoSchema)).toContain('?root'); + }); + + it('stringifies to a runnable CONSTRUCT the SPARQL parser round-trips', () => { + const query = extractionQueryString(creativeWork, drapoSchema); + expect(query).toContain('CONSTRUCT'); + expect(query).toContain('UNION'); + const reparsed = new Parser().parse(query); + expect(reparsed).toMatchObject({ type: 'query', subType: 'construct' }); + }); + + it('throws for a type with no path-bearing field – nothing to extract', () => { + const empty: SearchType = { + name: 'Empty', + class: 'urn:x:Empty', + fields: [{ name: 'computed', kind: 'keyword', derive: () => 'x' }], + }; + expect(() => extractionQuery(empty, searchSchema(empty))).toThrow( + /no path-bearing field/, + ); + }); +}); + +describe('inline references (nested template)', () => { + // Synthetic coverage only: no current consumer (Drapo is labelOnly) exercises + // an inline reference, so the nested-template shape is pinned here rather than + // at scale. + const registration = defineSearchType({ + name: 'Registration', + fields: [ + { name: 'datePosted', kind: 'date', path: `<${SCHEMA}datePosted>` }, + ], + }); + const dataset = defineSearchType({ + name: 'Dataset', + class: `${SCHEMA}Dataset`, + fields: [ + { + name: 'registration', + kind: 'reference', + array: true, + path: `<${SCHEMA}subjectOf>`, + ref: { typeName: 'Registration', strategy: 'inline' }, + }, + ], + }); + const inlineSchema = searchSchema(dataset, registration); + + it('emits one CONSTRUCT with a nested template linking ?root → ?referent → ?value', () => { + const query = extractionQuery(dataset, inlineSchema); + // The reference hop and the referent’s field are both minted, off the two + // subjects of the nested template. + expect(templatePredicates(query)).toEqual([ + irAlias(dataset, dataset.fields[0]), + irAlias(registration, registration.fields[0]), + ]); + const [refTriple, valueTriple] = templateTriples(query); + // ?root <…/registration> ?r ; ?r <…/datePosted> ?v – the link is preserved. + expect(refTriple.subject).toMatchObject({ value: 'root' }); + expect(valueTriple.subject).toEqual(refTriple.object); + }); + + it('binds the referent even when the reference type has no path-bearing field', () => { + // A reference type whose only field is derived reaches the graph for nothing + // of its own, but the reference hop is still emitted so a later derive can + // read the referent’s @id: the branch binds ?r without a nested union. + const marker = defineSearchType({ + name: 'Marker', + fields: [{ name: 'present', kind: 'boolean', derive: () => true }], + }); + const withMarker = defineSearchType({ + name: 'Dataset', + class: `${SCHEMA}Dataset`, + fields: [ + { + name: 'marker', + kind: 'reference', + path: `<${SCHEMA}subjectOf>`, + ref: { typeName: 'Marker', strategy: 'inline' }, + }, + ], + }); + const query = extractionQuery(withMarker, searchSchema(withMarker, marker)); + // Only the reference hop is minted (the derived field reads no path). + expect(templatePredicates(query)).toEqual([ + irAlias(withMarker, withMarker.fields[0]), + ]); + // The branch is just `{ ?root ?r }` – one BGP, no nested union. + const [group] = unionBranches(query); + expect(group.patterns).toHaveLength(1); + expect(group.patterns[0]).toMatchObject({ subType: 'bgp' }); + }); + + it('silently omits an inline reference the given schema does not declare', () => { + // Generated against a foreign schema that omits the referent type – the same + // graceful degradation the projection makes. The resolvable field is still + // extracted; the unresolvable inline reference contributes nothing. + const withName = defineSearchType({ + name: 'Dataset', + class: `${SCHEMA}Dataset`, + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + }, + { + name: 'registration', + kind: 'reference', + array: true, + path: `<${SCHEMA}subjectOf>`, + ref: { typeName: 'Registration', strategy: 'inline' }, + }, + ], + }); + const foreignSchema = searchSchema({ + name: 'Other', + class: 'urn:x:Other', + fields: [], + }); + const query = extractionQuery(withName, foreignSchema); + expect(templatePredicates(query)).toEqual([ + irAlias(withName, withName.fields[0]), + ]); + }); +}); + +describe('extraction ⟷ projection contract', () => { + // The drift guard: every IR Alias the generator mints is one the projection + // reads, and vice versa. Both derive from the same rule – a path-bearing field, + // recursing inline referents through their reference type – so a change to + // either walk that drops or adds a field breaks this test. + function projectionReads( + searchType: SearchType, + schema: SearchSchema, + ): Set { + const aliases = new Set(); + for (const field of searchType.fields) { + if (field.path === undefined) { + continue; + } + aliases.add(irAlias(searchType, field)); + if (field.kind === 'reference' && field.ref?.strategy === 'inline') { + const referenceType = referenceTypeNamed(schema, field.ref.typeName); + if (referenceType !== undefined) { + for (const alias of projectionReads(referenceType, schema)) { + aliases.add(alias); + } + } + } + } + return aliases; + } + + it('mints exactly the aliases the projection reads, for a labelOnly schema', () => { + const minted = new Set( + templatePredicates(extractionQuery(creativeWork, drapoSchema)), + ); + expect(minted).toEqual(projectionReads(creativeWork, drapoSchema)); + }); + + it('mints exactly the aliases the projection reads, through an inline reference', () => { + const registration = defineSearchType({ + name: 'Registration', + fields: [ + { name: 'datePosted', kind: 'date', path: `<${SCHEMA}datePosted>` }, + ], + }); + const dataset = defineSearchType({ + name: 'Dataset', + class: `${SCHEMA}Dataset`, + fields: [ + { + name: 'registration', + kind: 'reference', + array: true, + path: `<${SCHEMA}subjectOf>`, + ref: { typeName: 'Registration', strategy: 'inline' }, + }, + ], + }); + const schema = searchSchema(dataset, registration); + const minted = new Set( + templatePredicates(extractionQuery(dataset, schema)), + ); + expect(minted).toEqual(projectionReads(dataset, schema)); + }); +}); diff --git a/packages/search-pipeline/test/fixtures/drapo-sample.ttl b/packages/search-pipeline/test/fixtures/drapo-sample.ttl new file mode 100644 index 00000000..474445dd --- /dev/null +++ b/packages/search-pipeline/test/fixtures/drapo-sample.ttl @@ -0,0 +1,16 @@ +@prefix schema: . + +# A Drapo-shaped SCHEMA-AP-NDE sample: CreativeWork roots typed in the source +# (so selectByClass selects them), each with localized name/description and a +# labelOnly creator reference resolving a typed Person. + + a schema:CreativeWork ; + schema:name "Het meisje met de parel"@nl, "Girl with a Pearl Earring"@en ; + schema:description "Een schilderij van Johannes Vermeer."@nl ; + schema:creator . + + a schema:CreativeWork ; + schema:name "De nachtwacht"@nl . + + a schema:Person ; + schema:name "Johannes Vermeer" . diff --git a/packages/search-pipeline/test/multi-collection.integration.test.ts b/packages/search-pipeline/test/multi-collection.integration.test.ts index 1cf3ceab..8e2af3da 100644 --- a/packages/search-pipeline/test/multi-collection.integration.test.ts +++ b/packages/search-pipeline/test/multi-collection.integration.test.ts @@ -10,6 +10,7 @@ import { type RootType, type SearchDocument, } from '@lde/search'; +import { irAlias } from '@lde/search/adapter'; import { BlueGreenRebuild } from '@lde/search-typesense'; import { searchIndexWriter } from '../src/search-index-writer.js'; import type { TypedSearchDocument } from '../src/typed-search-document.js'; @@ -53,6 +54,11 @@ const schema = searchSchema( const datasetType = schema.get(DATASET) as RootType; const organizationType = schema.get(ORGANIZATION) as RootType; +// The Extraction CONSTRUCT emits values under the field’s IR Alias; the framed +// quads fed to the projection are keyed by it, not by the source path. +const TITLE_ALIAS = irAlias(datasetType, datasetType.fields[0]); +const NAME_ALIAS = irAlias(organizationType, organizationType.fields[0]); + const COLLECTION: Record = { [DATASET]: 'datasets', [ORGANIZATION]: 'organizations', @@ -67,13 +73,21 @@ const dataset = new Dataset({ function mixedQuads(): Quad[] { return [ quad(namedNode('https://ex/d/1'), namedNode(RDF_TYPE), namedNode(DATASET)), - quad(namedNode('https://ex/d/1'), namedNode(TITLE), literal('Verhaal')), + quad( + namedNode('https://ex/d/1'), + namedNode(TITLE_ALIAS), + literal('Verhaal'), + ), quad( namedNode('https://ex/o/1'), namedNode(RDF_TYPE), namedNode(ORGANIZATION), ), - quad(namedNode('https://ex/o/1'), namedNode(NAME), literal('Rijksmuseum')), + quad( + namedNode('https://ex/o/1'), + namedNode(NAME_ALIAS), + literal('Rijksmuseum'), + ), ]; } diff --git a/packages/search-pipeline/test/search-stages.test.ts b/packages/search-pipeline/test/search-stages.test.ts index 27350024..e1502246 100644 --- a/packages/search-pipeline/test/search-stages.test.ts +++ b/packages/search-pipeline/test/search-stages.test.ts @@ -11,6 +11,7 @@ import { type VariableBindings, } from '@lde/pipeline'; import { projectRoots, searchSchema, type RootType } from '@lde/search'; +import { irAlias } from '@lde/search/adapter'; import { searchStages, selectByClass } from '../src/search-stages.js'; import type { TypedSearchDocument } from '../src/typed-search-document.js'; @@ -26,6 +27,11 @@ const schema = searchSchema({ }); const person = schema.get(PERSON) as RootType; +// The stub reader stands in for the Extraction CONSTRUCT, so it emits each +// value under the field’s IR Alias – the key the projection reads – not the +// source path. +const NAME_ALIAS = irAlias(person, person.fields[0]); + const dataset = new Dataset({ iri: new URL('http://example.org/dataset/1'), distributions: [], @@ -54,7 +60,7 @@ const nameReader: Reader = { for (const binding of options?.bindings ?? []) { const root = binding.root.value; quads.push( - quad(namedNode(root), namedNode(NAME), literal(`Name ${root}`)), + quad(namedNode(root), namedNode(NAME_ALIAS), literal(`Name ${root}`)), ); } return Promise.resolve(stream(quads)); diff --git a/packages/search-pipeline/tsconfig.lib.json b/packages/search-pipeline/tsconfig.lib.json index 3cc0126f..e26f1181 100644 --- a/packages/search-pipeline/tsconfig.lib.json +++ b/packages/search-pipeline/tsconfig.lib.json @@ -10,6 +10,9 @@ }, "include": ["src/**/*.ts"], "references": [ + { + "path": "../local-sparql-endpoint/tsconfig.lib.json" + }, { "path": "../search/tsconfig.lib.json" }, diff --git a/packages/search/src/adapter.ts b/packages/search/src/adapter.ts index 0a123e3d..6fe080ed 100644 --- a/packages/search/src/adapter.ts +++ b/packages/search/src/adapter.ts @@ -9,6 +9,7 @@ export { assertTypeInSchema, physicalFields, + irAlias, displayFieldName, displayFieldPattern, displayLangOf, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index e5f46320..de6f6532 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -9,6 +9,7 @@ import { assertTypeInSchema, displayFieldName, inlineFramingDepth, + irAlias, isInternalField, isInlineReference, isoToUnixSeconds, @@ -108,7 +109,7 @@ function projectFields( } const document: SearchDocument = { id }; for (const field of searchType.fields) { - applyField(document, node, field, schema); + applyField(document, node, field, searchType, schema); } return document; } @@ -148,6 +149,7 @@ function applyField( document: SearchDocument, node: FramedNode, field: SearchField, + searchType: SearchType, schema: SearchSchema | undefined, ): void { if (field.derive !== undefined) { @@ -157,42 +159,48 @@ function applyField( } return; } - const path = field.path; - if (path === undefined) { + if (field.path === undefined) { // Neither path nor derive: populated outside the projection, if at all. return; } + // The framed node is keyed by the {@link irAlias IR Alias} the extraction + // CONSTRUCT minted, not by the source `path`: `path` states what to read from + // the graph (the reader adapter’s grammar), the alias is what the reader + // emitted it under. Minting against `searchType` – a root field against the + // root type, an inline referent’s field against its reference type – is what + // lets one subject be a root of two types without their fields colliding. + const alias = irAlias(searchType, field); if (isInlineReference(field)) { // An inline reference is a nested structure, not a bare IRI: it can only be // projected with the schema that declares its reference type. Without one, // project nothing rather than fall through and emit the referent IRIs under // the field name (the wrong shape). if (schema !== undefined) { - applyInlineReference(document, node, path, field, schema); + applyInlineReference(document, node, alias, field, schema); } return; } switch (field.kind) { case 'text': - return applyText(document, langValuesOf(node, path), field); + return applyText(document, langValuesOf(node, alias), field); case 'keyword': - return applyFacet(document, literalsOf(node, path), field); + return applyFacet(document, literalsOf(node, alias), field); case 'reference': - return applyFacet(document, irisOf(node, path), field); + return applyFacet(document, irisOf(node, alias), field); case 'integer': return setNumber( document, field.name, - toInteger(firstLiteralOf(node, path)), + toInteger(firstLiteralOf(node, alias)), ); case 'number': return setNumber( document, field.name, - toNumber(firstLiteralOf(node, path)), + toNumber(firstLiteralOf(node, alias)), ); case 'date': { - const literal = firstLiteralOf(node, path); + const literal = firstLiteralOf(node, alias); return setNumber( document, field.name, @@ -201,7 +209,7 @@ function applyField( } case 'boolean': { // The xsd:boolean lexical space: true/false/1/0. - const literal = firstLiteralOf(node, path); + const literal = firstLiteralOf(node, alias); if (literal !== undefined) { document[field.name] = literal === 'true' || literal === '1'; } @@ -289,12 +297,13 @@ function applyFacet( } /** - * Project an inline reference: the referent node(s) embedded under `path` are - * each projected through the reference’s {@link ReferenceType} (its fields’ - * paths relative to the referent node) and attached under the field’s name – a - * nested {@link SearchDocument} for a single reference, an array for an - * `array` one. The referent is projected in full – internal fields included – - * so the declaring type’s (or the reference type’s own) derives can read them; + * Project an inline reference: the referent node(s) embedded under the field’s + * {@link irAlias IR Alias} are each projected through the reference’s + * {@link ReferenceType} (whose own fields read their own aliases, minted against + * the reference type) and attached under the field’s name – a nested + * {@link SearchDocument} for a single reference, an array for an `array` one. + * The referent is projected in full – internal fields included – so the + * declaring type’s (or the reference type’s own) derives can read them; * {@link pruneInternalFields} then removes the internal fields from a *surfaced* * referent and deletes an internal inline reference whole. Recurses through * `schema`, so an inline reference may itself carry further inline references to @@ -304,7 +313,7 @@ function applyFacet( function applyInlineReference( document: SearchDocument, node: FramedNode, - path: string, + alias: string, field: ReferenceField & { readonly ref: { readonly typeName: string } }, schema: SearchSchema, ): void { @@ -315,7 +324,7 @@ function applyInlineReference( if (referenceType === undefined) { return; } - const referents = valuesOf(node, path) + const referents = valuesOf(node, alias) .filter(isObject) .filter((referent) => typeof referent['@id'] === 'string') .map((referent) => projectFields(referent, referenceType, schema)); @@ -325,9 +334,11 @@ function applyInlineReference( document[field.name] = field.array === true ? referents : referents[0]; } -// --- Framed-IR readers: read a declared `path` off the framed node. Internal -// to projection – a `derive` reads the projected document, never the node, so -// `path` stays the whole statement of what the projection reads from the graph. +// --- Framed-IR readers: read a field’s value off the framed node by its +// {@link irAlias IR Alias} key. Internal to projection – a `derive` reads the +// projected document, never the node, so `path` stays the whole statement of +// what the projection reads from the graph, and the alias the whole statement of +// what the reader emitted it under. /** A literal value with its (possibly empty) language tag. */ interface LangValue { @@ -335,30 +346,30 @@ interface LangValue { readonly lang: string; } -function langValuesOf(node: FramedNode, path: string): LangValue[] { - return valuesOf(node, path) +function langValuesOf(node: FramedNode, key: string): LangValue[] { + return valuesOf(node, key) .map(toLangValue) .filter((value): value is LangValue => value !== undefined); } -function literalsOf(node: FramedNode, path: string): string[] { - return valuesOf(node, path) +function literalsOf(node: FramedNode, key: string): string[] { + return valuesOf(node, key) .map(literalString) .filter((value): value is string => value !== undefined); } -function firstLiteralOf(node: FramedNode, path: string): string | undefined { - return literalsOf(node, path)[0]; +function firstLiteralOf(node: FramedNode, key: string): string | undefined { + return literalsOf(node, key)[0]; } -function irisOf(node: FramedNode, path: string): string[] { - return valuesOf(node, path) +function irisOf(node: FramedNode, key: string): string[] { + return valuesOf(node, key) .map(iriString) .filter((value): value is string => value !== undefined); } -function valuesOf(node: FramedNode, path: string): unknown[] { - const value = node[path]; +function valuesOf(node: FramedNode, key: string): unknown[] { + const value = node[key]; if (value === undefined) { return []; } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 5342b84b..e56f2ce0 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -919,6 +919,27 @@ export function fieldNamed( return searchType.fields.find((field) => field.name === name); } +/** + * The **IR Alias** predicate for a field: `urn:lde:‹SearchType.name›/‹field.name›`. + * The extraction CONSTRUCT emits a field’s value under this minted predicate, and + * the {@link projectDocument projection} reads it back under the same key – the + * two sides agree by calling this one function rather than by a hand-written + * convention that can drift (exactly the argument {@link physicalFields}’ JSDoc + * makes for the physical fanout). + * + * A property path cannot be a CONSTRUCT template verb, so flattening a multi-hop + * value onto its subject must mint a predicate for it; that predicate is a + * mechanical function of the field name, never authored by hand and never a + * public vocabulary. Field names are unique per type and restricted to + * `[A-Za-z_][A-Za-z0-9_]*` ({@link validateSearchType}), so the alias needs no + * escaping; it is qualified by the **type** name because one subject can be a + * root of two types (`frame-by-type`), which must not collide on a shared field + * name. + */ +export function irAlias(searchType: SearchType, field: SearchField): string { + return `urn:lde:${searchType.name}/${field.name}`; +} + /** * Whether a facet on this field returns fixed range bins (a histogram) rather * than one bucket per distinct value: it declares non-empty diff --git a/packages/search/test/project.test.ts b/packages/search/test/project.test.ts index 918fc4c3..ee286eb5 100644 --- a/packages/search/test/project.test.ts +++ b/packages/search/test/project.test.ts @@ -17,19 +17,26 @@ const DR = 'urn:dr:'; const IANA = 'https://www.iana.org/assignments/media-types/'; const DATASET = dcat.Dataset.value; +// The extraction CONSTRUCT emits each value under its field’s IR Alias +// (`urn:lde:‹Type›/‹field›`), and the projection reads it back under that key – +// never under the source `path`. Framed nodes here are therefore keyed by the +// alias, which is a function of the declaring type name and the field name. +const alias = (type: string, field: string) => `urn:lde:${type}/${field}`; +const dsKey = (field: string) => alias('Dataset', field); + const node = { '@id': 'https://ex/d/1', - [dcterms.title.value]: [ + [dsKey('title')]: [ { '@language': 'nl', '@value': 'Titel' }, { '@language': 'en', '@value': 'Title' }, ], - [dcterms.publisher.value]: { '@id': 'https://ex/o/1' }, - [`${DR}publisherName`]: { '@language': 'nl', '@value': 'Erfgoed' }, - [dcat.keyword.value]: [{ '@language': 'nl', '@value': 'Erfgoed' }], - [`${DR}format`]: [`${IANA}text/turtle`], - [`${DR}class`]: [{ '@id': 'http://schema.org/Person' }], - [`${DR}datePosted`]: { '@value': '2024-01-01T00:00:00.000Z' }, - [`${DR}size`]: { '@type': xsd.integer.value, '@value': '1234' }, + [dsKey('publisher')]: { '@id': 'https://ex/o/1' }, + [dsKey('publisherName')]: { '@language': 'nl', '@value': 'Erfgoed' }, + [dsKey('keyword')]: [{ '@language': 'nl', '@value': 'Erfgoed' }], + [dsKey('format')]: [`${IANA}text/turtle`], + [dsKey('class')]: [{ '@id': 'http://schema.org/Person' }], + [dsKey('date_posted')]: { '@value': '2024-01-01T00:00:00.000Z' }, + [dsKey('size')]: { '@type': xsd.integer.value, '@value': '1234' }, }; const fields: SearchField[] = [ @@ -126,10 +133,10 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/3', - [`${DR}size`]: { '@value': 42 }, - [dcterms.language.value]: { '@value': true }, - [dcat.keyword.value]: 'bareString', - [`${DR}class`]: 'http://example.org/BareClass', + [dsKey('size')]: { '@value': 42 }, + [dsKey('language')]: { '@value': true }, + [dsKey('keyword')]: 'bareString', + [dsKey('class')]: 'http://example.org/BareClass', }, { name: 'Dataset', @@ -170,7 +177,7 @@ describe('projectDocument', () => { it('projects a number field as a float (not truncated like integer)', () => { const document = projectDocument( - { '@id': 'https://ex/d/12', [`${DR}size`]: { '@value': '1234.5' } }, + { '@id': 'https://ex/d/12', [dsKey('size')]: { '@value': '1234.5' } }, { name: 'Dataset', class: DATASET, @@ -192,7 +199,7 @@ describe('projectDocument', () => { }; const project = (value: unknown): SearchDocument => projectDocument( - { '@id': 'https://ex/d/5', [`${DR}iiif`]: { '@value': value } }, + { '@id': 'https://ex/d/5', [dsKey('iiif')]: { '@value': value } }, withBoolean, ); @@ -207,7 +214,7 @@ describe('projectDocument', () => { it('folds the transformed values (not the raw ones) for a facet search field', () => { const document = projectDocument( - { '@id': 'https://ex/d/4', [`${DR}format`]: [`${IANA}text/turtle`] }, + { '@id': 'https://ex/d/4', [dsKey('format')]: [`${IANA}text/turtle`] }, { name: 'Dataset', class: DATASET, @@ -230,7 +237,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/2', - [dcterms.title.value]: { '@language': 'nl', '@value': 'Solo' }, + [dsKey('title')]: { '@language': 'nl', '@value': 'Solo' }, }, { name: 'Dataset', class: DATASET, fields }, ); @@ -253,7 +260,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/6', - [dcterms.title.value]: { '@language': 'fr', '@value': 'Bonjour' }, + [dsKey('title')]: { '@language': 'fr', '@value': 'Bonjour' }, }, { name: 'Dataset', class: DATASET, fields }, ); @@ -273,7 +280,7 @@ describe('projectDocument', () => { { '@id': 'https://ex/d/6b', // Non-conformant `pt_BR` (underscore instead of hyphen) – dirty data. - [dcterms.title.value]: { '@language': 'pt_BR', '@value': 'Mapa' }, + [dsKey('title')]: { '@language': 'pt_BR', '@value': 'Mapa' }, }, { name: 'Dataset', class: DATASET, fields }, ); @@ -288,7 +295,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/7', - [dcterms.title.value]: { '@value': 'Naamloos' }, + [dsKey('title')]: { '@value': 'Naamloos' }, }, { name: 'Dataset', class: DATASET, fields }, ); @@ -304,7 +311,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/10', - [dcterms.title.value]: { '@language': 'nl', '@value': 'Verhalen' }, + [dsKey('title')]: { '@language': 'nl', '@value': 'Verhalen' }, }, { name: 'Dataset', @@ -331,7 +338,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/10b', - [dcterms.title.value]: [ + [dsKey('title')]: [ { '@language': 'nl', '@value': 'Verhalen' }, { '@language': 'fr', '@value': 'Récits' }, ], @@ -364,7 +371,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/8', - [dcterms.title.value]: [ + [dsKey('title')]: [ { '@language': 'nl', '@value': 'Titel' }, { '@language': 'nl', '@value': 'Ondertitel' }, ], @@ -380,7 +387,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/11', - [dcterms.title.value]: { '@language': 'nl', '@value': 'Titel' }, + [dsKey('title')]: { '@language': 'nl', '@value': 'Titel' }, }, { name: 'Dataset', @@ -427,11 +434,11 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/internal', - [`${DR}token`]: ['tok'], - [`${DR}ref`]: { '@id': 'https://ex/o/9' }, - [`${DR}count`]: { '@value': '7' }, - [`${DR}score`]: { '@value': '1.5' }, - [`${DR}flag`]: { '@value': 'true' }, + [dsKey('token')]: ['tok'], + [dsKey('ref')]: { '@id': 'https://ex/o/9' }, + [dsKey('count')]: { '@value': '7' }, + [dsKey('score')]: { '@value': '1.5' }, + [dsKey('flag')]: { '@value': 'true' }, }, { name: 'Dataset', @@ -455,7 +462,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/reading-device', - [`${DR}class`]: [ + [dsKey('classes')]: [ { '@id': 'http://schema.org/Person' }, { '@id': 'http://schema.org/Place' }, ], @@ -532,18 +539,22 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/reg', - [`${DR}registration`]: [ + [dsKey('registration')]: [ { '@id': 'https://ex/r/1', - 'https://schema.org/dateRead': { '@value': '2024-01-01T00:00:00Z' }, - 'https://schema.org/datePosted': { + [alias('Registration', 'dateRead')]: { + '@value': '2024-01-01T00:00:00Z', + }, + [alias('Registration', 'datePosted')]: { '@value': '2024-02-01T00:00:00Z', }, }, { '@id': 'https://ex/r/2', - 'https://schema.org/dateRead': { '@value': '2024-06-01T00:00:00Z' }, - 'https://schema.org/datePosted': { + [alias('Registration', 'dateRead')]: { + '@value': '2024-06-01T00:00:00Z', + }, + [alias('Registration', 'datePosted')]: { '@value': '2024-07-01T00:00:00Z', }, }, @@ -612,7 +623,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/noschema', - [`${DR}registration`]: { '@id': 'https://ex/r/1' }, + [dsKey('registration')]: { '@id': 'https://ex/r/1' }, }, dataset, ); @@ -644,7 +655,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/foreign', - [`${DR}registration`]: { '@id': 'https://ex/r/9' }, + [dsKey('registration')]: { '@id': 'https://ex/r/9' }, }, dataset, foreignSchema, @@ -684,9 +695,9 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/api', - [`${DR}creator`]: { + [dsKey('creator')]: { '@id': 'https://ex/c/1', - 'https://schema.org/name': { '@language': 'nl', '@value': 'Naam' }, + [alias('Creator', 'label')]: { '@language': 'nl', '@value': 'Naam' }, }, }, dataset, @@ -754,11 +765,14 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/prune', - [`${DR}creator`]: [ + [dsKey('creator')]: [ { '@id': 'https://ex/c/2', - 'https://schema.org/name': { '@language': 'nl', '@value': 'Naam' }, - 'https://schema.org/alternateName': 'Alt', + [alias('Creator', 'label')]: { + '@language': 'nl', + '@value': 'Naam', + }, + [alias('Creator', 'rawSort')]: 'Alt', }, ], }, @@ -781,7 +795,14 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/13', - [dcterms.title.value]: [ + [dsKey('title')]: [ + { '@language': 'nl', '@value': 'Café' }, + 'Untagged subtitle', + ], + // `note` reads dcterms:title too, so the extraction emits the value under + // its own alias as well – each field gets its own IR Alias, even when two + // share a source path. + [dsKey('note')]: [ { '@language': 'nl', '@value': 'Café' }, 'Untagged subtitle', ], @@ -833,12 +854,12 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/12', - [dcterms.title.value]: [ + [dsKey('title')]: [ { '@language': 'nl', '@value': 'Titel' }, { '@language': 'nl', '@value': { nested: true } }, ], - [dcat.keyword.value]: [{ '@value': { nested: true } }, 'kaart'], - [dcterms.publisher.value]: [{ nested: true }, { '@id': 'https://o/1' }], + [dsKey('keyword')]: [{ '@value': { nested: true } }, 'kaart'], + [dsKey('publisher')]: [{ nested: true }, { '@id': 'https://o/1' }], }, { name: 'Dataset', @@ -874,7 +895,7 @@ describe('projectDocument', () => { it('throws when the framed node has no @id', () => { expect(() => projectDocument( - { [dcterms.title.value]: { '@value': 'No id' } }, + { [dsKey('title')]: { '@value': 'No id' } }, { name: 'Dataset', class: DATASET, fields }, ), ).toThrow(/without an @id/); @@ -886,7 +907,7 @@ describe('projectDocument', () => { const document = projectDocument( { '@id': 'https://ex/d/9', - [dcterms.title.value]: { '@language': 'nl', '@value': 'Titel' }, + [dsKey('title')]: { '@language': 'nl', '@value': 'Titel' }, }, { name: 'Dataset', @@ -912,8 +933,8 @@ describe('projectRoots', () => { it('projects exactly the given roots, without any rdf:type', async () => { // No type triples: the roots are supplied by the caller (the selector). const quads = new Parser({ format: 'N-Triples' }).parse(` - <${dcterms.title.value}> "Titel"@nl . - <${dcterms.title.value}> "Andere"@nl . + <${dsKey('title')}> "Titel"@nl . + <${dsKey('title')}> "Andere"@nl . `); const documents: SearchDocument[] = []; @@ -938,7 +959,7 @@ describe('projectRoots', () => { it('frames a repeated root once (a non-DISTINCT selector may yield duplicates)', async () => { const quads = new Parser({ format: 'N-Triples' }).parse( - ` <${dcterms.title.value}> "Titel"@nl .`, + ` <${dsKey('title')}> "Titel"@nl .`, ); const documents: SearchDocument[] = []; @@ -959,7 +980,7 @@ describe('projectRoots', () => { it('yields a bare document, not paired with a searchType', async () => { const quads = new Parser({ format: 'N-Triples' }).parse( - ` <${dcterms.title.value}> "Titel"@nl .`, + ` <${dsKey('title')}> "Titel"@nl .`, ); const documents: SearchDocument[] = []; @@ -980,8 +1001,8 @@ describe('projectRoots', () => { it('frames only the given roots, ignoring other subjects in the quads', async () => { const quads = new Parser({ format: 'N-Triples' }).parse(` - <${dcterms.title.value}> "Een"@nl . - <${dcterms.title.value}> "Twee"@nl . + <${dsKey('title')}> "Een"@nl . + <${dsKey('title')}> "Twee"@nl . `); const ids: string[] = []; diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index 996a9181..735917cf 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -9,6 +9,7 @@ import { fieldNamed, filterableFields, inlineFramingDepth, + irAlias, isoToUnixSeconds, isRangeFacet, outputFields, @@ -159,6 +160,33 @@ describe('physicalFields', () => { }); }); +describe('irAlias', () => { + const dataset: SearchType = { + name: 'Dataset', + class: DATASET, + fields: [{ name: 'publisherName', kind: 'text', locales: ['nl'] }], + }; + const person: SearchType = { + name: 'Person', + fields: [{ name: 'label', kind: 'text', locales: ['nl'] }], + }; + + it('mints urn:lde:‹Type›/‹field› from the type and field names', () => { + expect(irAlias(dataset, dataset.fields[0])).toBe( + 'urn:lde:Dataset/publisherName', + ); + }); + + it('qualifies by type name, so one subject can be a root of two types', () => { + // Same field name, different declaring type → distinct aliases. + const datasetLabel = { name: 'label', kind: 'text', locales: ['nl'] }; + expect(irAlias(dataset, datasetLabel as SearchField)).not.toBe( + irAlias(person, person.fields[0]), + ); + expect(irAlias(person, person.fields[0])).toBe('urn:lde:Person/label'); + }); +}); + describe('display field helpers', () => { const label: TextField = { name: 'label',