From 0803f1b4b46a8007f6b757680a7926643ebb608f Mon Sep 17 00:00:00 2001 From: Aileen Booker Date: Wed, 19 Aug 2026 17:55:36 +0400 Subject: [PATCH] Added the deep fragmenter record-building interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapters currently coordinate projection, grouping, size limits, and error handling themselves, which is why records can exceed Algolia's free-plan ceiling, useful Ghost fields are hard to add safely, and empty content vanishes from the index. Owning that policy behind one synchronous createAlgoliaRecords seam lets the CLI and Netlify slices adopt validated projection, deterministic 9,999-byte packing, fallback records, and whole-batch preflight errors without re-implementing any of it — while both deprecated wrappers keep their exact behavior so nothing changes for consumers until those follow-up releases. Refs #220 --- CONTEXT.md | 24 + packages/algolia-fragmenter/README.md | 56 +- .../lib/create-algolia-records.d.mts | 16 + .../lib/create-algolia-records.d.mts.map | 1 + .../lib/create-algolia-records.mjs | 40 + .../lib/create-algolia-records.mjs.map | 1 + packages/algolia-fragmenter/lib/errors.d.mts | 44 + .../algolia-fragmenter/lib/errors.d.mts.map | 1 + packages/algolia-fragmenter/lib/errors.mjs | 23 + .../algolia-fragmenter/lib/errors.mjs.map | 1 + .../algolia-fragmenter/lib/grouping.d.mts | 18 + .../algolia-fragmenter/lib/grouping.d.mts.map | 1 + packages/algolia-fragmenter/lib/grouping.mjs | 27 + .../algolia-fragmenter/lib/grouping.mjs.map | 1 + packages/algolia-fragmenter/lib/index.d.mts | 10 +- .../algolia-fragmenter/lib/index.d.mts.map | 2 +- packages/algolia-fragmenter/lib/index.mjs | 44 +- packages/algolia-fragmenter/lib/index.mjs.map | 2 +- packages/algolia-fragmenter/lib/policy.d.mts | 47 + .../algolia-fragmenter/lib/policy.d.mts.map | 1 + packages/algolia-fragmenter/lib/policy.mjs | 271 +++ .../algolia-fragmenter/lib/policy.mjs.map | 1 + .../algolia-fragmenter/lib/projection.d.mts | 27 + .../lib/projection.d.mts.map | 1 + .../algolia-fragmenter/lib/projection.mjs | 225 +++ .../algolia-fragmenter/lib/projection.mjs.map | 1 + packages/algolia-fragmenter/lib/records.d.mts | 15 + .../algolia-fragmenter/lib/records.d.mts.map | 1 + packages/algolia-fragmenter/lib/records.mjs | 133 ++ .../algolia-fragmenter/lib/records.mjs.map | 1 + .../src/create-algolia-records.mts | 48 + packages/algolia-fragmenter/src/errors.mts | 78 + packages/algolia-fragmenter/src/grouping.mts | 46 + packages/algolia-fragmenter/src/index.mts | 98 +- packages/algolia-fragmenter/src/policy.mts | 504 ++++++ .../algolia-fragmenter/src/projection.mts | 370 ++++ packages/algolia-fragmenter/src/records.mts | 216 +++ .../test/create-algolia-records.test.mts | 1518 +++++++++++++++++ .../test/package.acceptance.test.mts | 145 +- vitest.config.mjs | 2 +- 40 files changed, 3957 insertions(+), 104 deletions(-) create mode 100644 packages/algolia-fragmenter/lib/create-algolia-records.d.mts create mode 100644 packages/algolia-fragmenter/lib/create-algolia-records.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/create-algolia-records.mjs create mode 100644 packages/algolia-fragmenter/lib/create-algolia-records.mjs.map create mode 100644 packages/algolia-fragmenter/lib/errors.d.mts create mode 100644 packages/algolia-fragmenter/lib/errors.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/errors.mjs create mode 100644 packages/algolia-fragmenter/lib/errors.mjs.map create mode 100644 packages/algolia-fragmenter/lib/grouping.d.mts create mode 100644 packages/algolia-fragmenter/lib/grouping.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/grouping.mjs create mode 100644 packages/algolia-fragmenter/lib/grouping.mjs.map create mode 100644 packages/algolia-fragmenter/lib/policy.d.mts create mode 100644 packages/algolia-fragmenter/lib/policy.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/policy.mjs create mode 100644 packages/algolia-fragmenter/lib/policy.mjs.map create mode 100644 packages/algolia-fragmenter/lib/projection.d.mts create mode 100644 packages/algolia-fragmenter/lib/projection.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/projection.mjs create mode 100644 packages/algolia-fragmenter/lib/projection.mjs.map create mode 100644 packages/algolia-fragmenter/lib/records.d.mts create mode 100644 packages/algolia-fragmenter/lib/records.d.mts.map create mode 100644 packages/algolia-fragmenter/lib/records.mjs create mode 100644 packages/algolia-fragmenter/lib/records.mjs.map create mode 100644 packages/algolia-fragmenter/src/create-algolia-records.mts create mode 100644 packages/algolia-fragmenter/src/errors.mts create mode 100644 packages/algolia-fragmenter/src/grouping.mts create mode 100644 packages/algolia-fragmenter/src/policy.mts create mode 100644 packages/algolia-fragmenter/src/projection.mts create mode 100644 packages/algolia-fragmenter/src/records.mts create mode 100644 packages/algolia-fragmenter/test/create-algolia-records.test.mts diff --git a/CONTEXT.md b/CONTEXT.md index 25d28684..b8f41468 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -28,6 +28,10 @@ _Avoid_: Custom field, pass-through field An allowlisted public Ghost source field or package-owned compatibility projection that projection configuration may include, omit, or expose under a validated alias. Enabled optional fields are repeated in every Algolia record derived from that Ghost content. _Avoid_: Custom field, arbitrary field +**Ranking sibling**: +An additional custom-ranking value carried beside the package-owned heading and position values, sourced from an allowlisted numeric or boolean Ghost field under a validated alias. +_Avoid_: Custom ranking field, ranking attribute, sort field + **Extraction fragment**: An ordered emitted unit of searchable rendered meaning with searchable fragment HTML, preserved source text, heading context, and a fragment source. _Avoid_: Chunk, paragraph record @@ -44,6 +48,10 @@ _Avoid_: Outer HTML, raw attribute value A stable description of whether an extraction fragment came from element content or an attribute, including whether an element was selected as ordinary content or as a card-heading fallback. _Avoid_: Parser node, candidate ID, card adapter +**Anchor group**: +The ordered extraction fragments of one Ghost content item that share the same anchor, kept in first-seen anchor order. It is the unit that fixes an Algolia record's deep link, heading context, and identifier. +_Avoid_: Heading group, section, chunk + **HTML extractor**: The component that converts rendered HTML into ordered extraction fragments. _Avoid_: Fragmenter, transformer @@ -56,6 +64,22 @@ _Avoid_: HTML extractor The final indexed object containing projected Ghost fields, grouped extracted content, and ranking metadata. _Avoid_: Extraction fragment, Algolia post +**Fallback record**: +The single Algolia record emitted for Ghost content that produces no extraction fragments. It carries the Ghost content projection with empty fragment content and the headingless rank. +_Avoid_: Empty record, placeholder record, stub + +**Continuation record**: +An Algolia record carrying the later whole extraction fragments of one anchor group that did not fit within the record byte ceiling. It repeats the same projection and deep link under a stable suffixed object ID. +_Avoid_: Split record, overflow record, record page + +**Record byte ceiling**: +The largest compact UTF-8 byte size allowed for one complete Algolia record, chosen so output stays valid on Algolia's smallest plan. +_Avoid_: Size limit, 10 KB limit, character count + +**Preflight**: +The offline check that validates caller policy, Ghost content, and every complete Algolia record before any Algolia request. Failing preflight produces no records at all. +_Avoid_: Dry run (for this record check; the term still belongs to release tooling), validation pass, sanity check + **Ghost-rendered fixture**: An immutable Content API response produced by Ghost from controlled source content and retained as deterministic test evidence. _Avoid_: Mock response, live fixture diff --git a/packages/algolia-fragmenter/README.md b/packages/algolia-fragmenter/README.md index d3feaf91..4d33e198 100644 --- a/packages/algolia-fragmenter/README.md +++ b/packages/algolia-fragmenter/README.md @@ -16,7 +16,59 @@ pnpm add @tryghost/algolia-fragmenter ## Usage -Convert Ghost Content API posts, then reduce the resulting records into fragments: +`createAlgoliaRecords` turns Ghost content into complete final Algolia records in one synchronous call. It owns projection, HTML extraction, heading-anchor grouping, fallback records, deep links, identifiers, ranking metadata, record-size handling, and validation: + +```js +import {createAlgoliaRecords} from '@tryghost/algolia-fragmenter'; + +const records = createAlgoliaRecords(posts); +``` + +Every record contains the package-owned fields `objectID`, `slug`, `url`, `html`, `title`, `headings`, `anchor`, and `customRanking` with its `position` and `heading` values. Ghost content that produces no extraction fragments emits one fallback record with empty `html`, no anchor, and the headingless rank. The required Ghost input fields are `id`, `slug`, `url`, `title`, and `html`. + +### Options + +```js +const records = createAlgoliaRecords(posts, { + ignoreSlugs: ['secret-page'], + contentProjection: { + fields: ['image', 'tags', {source: 'reading_time', as: 'readingMinutes'}], + customRanking: [{source: 'featured', as: 'isFeatured'}] + } +}); +``` + +- `ignoreSlugs` excludes content by slug before the rest of its fields are validated. +- `contentProjection.fields` is the complete optional field set; it replaces the default set rather than patching it, and `[]` selects no optional fields. Without `contentProjection`, the optional fields are `image`, `tags`, `authors`, and `excerpt`. +- `contentProjection.customRanking` adds ranking siblings beside the package-owned `position` and `heading` values. Each sibling needs a validated alias. + +The optional source allowlist is `image`, `tags`, `authors`, `excerpt`, `custom_excerpt`, `feature_image_alt`, `feature_image_caption`, `canonical_url`, `featured`, `visibility`, `created_at`, `updated_at`, `published_at`, and `reading_time`. Ranking siblings may only be sourced from `featured` and `reading_time`. A field may be aliased with `{source, as}`, where `as` matches `^[A-Za-z][A-Za-z0-9_]*$` and changes only the output key. + +Enabled optional fields are repeated in every record derived from the same Ghost content. A missing scalar becomes `null`, missing `tags` or `authors` become `[]`, and meaningful `false`, `0`, and empty-string values are preserved. `image` reads Ghost's `feature_image`, and `tags` and `authors` keep the `{name, slug}` shape. + +### Record size + +Every complete record stays within 9,999 compact UTF-8 bytes. Whole extraction fragments are packed greedily and never truncated: the first record of an anchor group keeps `_` and continuations add `_`. An indivisible fragment, or required metadata that leaves no room for one, fails instead of being shortened. + +### Errors + +`createAlgoliaRecords` validates the whole batch and returns no records when any deterministic problem exists — it never returns a partial array. It throws one `FragmenterError` whose `code` is `INVALID_POLICY`, `INVALID_GHOST_CONTENT`, or `RECORD_TOO_LARGE`, and whose `issues` array lists every issue in input order: + +```js +import {createAlgoliaRecords, FragmenterError} from '@tryghost/algolia-fragmenter'; + +try { + createAlgoliaRecords(posts, options); +} catch (error) { + if (error instanceof FragmenterError) { + console.error(error.code, error.issues); + } +} +``` + +Policy issues carry the configuration `path` that must change. Content issues add the batch `index`, the Ghost content id, and the expected type. Size issues add the record's `objectID`, the anchor and source position when available, the measured `bytes`, the 9,999-byte `limit`, and the `excess`. + +### Deprecated wrappers ```js import {fragmentTransformer, transformToAlgoliaObject} from '@tryghost/algolia-fragmenter'; @@ -27,7 +79,7 @@ const fragments = records.reduce(fragmentTransformer, []); `transformToAlgoliaObject` accepts an optional array of post slugs to exclude as its second argument. `fragmentTransformer` is designed to be passed directly to `Array#reduce`. -Both operations are deprecated compatibility wrappers. They remain available with their existing output while a deeper record-building API is introduced separately. +Both operations are deprecated compatibility wrappers. They keep their existing output, do not receive the projection policy, and do not apply the record-size behaviour. New callers should use `createAlgoliaRecords`. This package is ESM-only and requires Node.js 24 or later. diff --git a/packages/algolia-fragmenter/lib/create-algolia-records.d.mts b/packages/algolia-fragmenter/lib/create-algolia-records.d.mts new file mode 100644 index 00000000..5f232ba1 --- /dev/null +++ b/packages/algolia-fragmenter/lib/create-algolia-records.d.mts @@ -0,0 +1,16 @@ +import { type CreateAlgoliaRecordsOptions } from './policy.mjs'; +import { type GhostContent } from './projection.mjs'; +import { type AlgoliaRecord } from './records.mjs'; +/** + * Turns Ghost content into complete final Algolia records: projection, HTML extraction, legacy + * anchor grouping, fallback records, deep links, identifiers, ranking metadata, and + * deterministic record-size handling. + * + * The whole batch is validated before any record is returned. A deterministic policy, Ghost + * content, or record-size problem throws one {@link FragmenterError} carrying every issue in + * input order; a partial batch is never returned. + * + * @throws {FragmenterError} `INVALID_POLICY`, `INVALID_GHOST_CONTENT`, or `RECORD_TOO_LARGE`. + */ +export declare const createAlgoliaRecords: (ghostContent: readonly GhostContent[], options?: CreateAlgoliaRecordsOptions) => readonly AlgoliaRecord[]; +//# sourceMappingURL=create-algolia-records.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/create-algolia-records.d.mts.map b/packages/algolia-fragmenter/lib/create-algolia-records.d.mts.map new file mode 100644 index 00000000..f06eb3a2 --- /dev/null +++ b/packages/algolia-fragmenter/lib/create-algolia-records.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-algolia-records.d.mts","sourceRoot":"","sources":["../src/create-algolia-records.mts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,2BAA2B,EAAC,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAsB,KAAK,YAAY,EAAC,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAuB,KAAK,aAAa,EAAC,MAAM,eAAe,CAAC;AAEvE;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,iBACf,SAAS,YAAY,EAAE,YAC3B,2BAA2B,KACtC,SAAS,aAAa,EAyBxB,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/create-algolia-records.mjs b/packages/algolia-fragmenter/lib/create-algolia-records.mjs new file mode 100644 index 00000000..55e744d4 --- /dev/null +++ b/packages/algolia-fragmenter/lib/create-algolia-records.mjs @@ -0,0 +1,40 @@ +import { extract } from '@tryghost/algolia-html-extractor'; +import { FragmenterError } from './errors.mjs'; +import { groupFragmentsByAnchor } from './grouping.mjs'; +import { resolvePolicy } from './policy.mjs'; +import { prepareGhostContent } from './projection.mjs'; +import { createContentRecords } from './records.mjs'; +/** + * Turns Ghost content into complete final Algolia records: projection, HTML extraction, legacy + * anchor grouping, fallback records, deep links, identifiers, ranking metadata, and + * deterministic record-size handling. + * + * The whole batch is validated before any record is returned. A deterministic policy, Ghost + * content, or record-size problem throws one {@link FragmenterError} carrying every issue in + * input order; a partial batch is never returned. + * + * @throws {FragmenterError} `INVALID_POLICY`, `INVALID_GHOST_CONTENT`, or `RECORD_TOO_LARGE`. + */ +export const createAlgoliaRecords = (ghostContent, options) => { + const policy = resolvePolicy(options); + if (!policy.ok) { + throw new FragmenterError('INVALID_POLICY', policy.issues); + } + const prepared = prepareGhostContent(ghostContent, policy.policy); + if (!prepared.ok) { + throw new FragmenterError('INVALID_GHOST_CONTENT', prepared.issues); + } + const records = []; + const issues = []; + for (const content of prepared.contents) { + const groups = groupFragmentsByAnchor(extract(content.html)); + const contentRecords = createContentRecords(content, groups); + records.push(...contentRecords.records); + issues.push(...contentRecords.issues); + } + if (issues.length > 0) { + throw new FragmenterError('RECORD_TOO_LARGE', issues); + } + return records; +}; +//# sourceMappingURL=create-algolia-records.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/create-algolia-records.mjs.map b/packages/algolia-fragmenter/lib/create-algolia-records.mjs.map new file mode 100644 index 00000000..08d6082a --- /dev/null +++ b/packages/algolia-fragmenter/lib/create-algolia-records.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"create-algolia-records.mjs","sourceRoot":"","sources":["../src/create-algolia-records.mts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,kCAAkC,CAAC;AAEzD,OAAO,EAAC,eAAe,EAAuB,MAAM,cAAc,CAAC;AACnE,OAAO,EAAC,sBAAsB,EAAC,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAC,aAAa,EAAmC,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAC,mBAAmB,EAAoB,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAC,oBAAoB,EAAqB,MAAM,eAAe,CAAC;AAEvE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,YAAqC,EACrC,OAAqC,EACb,EAAE;IAC1B,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,eAAe,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,eAAe,CAAC,uBAAuB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,cAAc,GAAG,oBAAoB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC","sourcesContent":["import {extract} from '@tryghost/algolia-html-extractor';\n\nimport {FragmenterError, type RecordSizeIssue} from './errors.mjs';\nimport {groupFragmentsByAnchor} from './grouping.mjs';\nimport {resolvePolicy, type CreateAlgoliaRecordsOptions} from './policy.mjs';\nimport {prepareGhostContent, type GhostContent} from './projection.mjs';\nimport {createContentRecords, type AlgoliaRecord} from './records.mjs';\n\n/**\n * Turns Ghost content into complete final Algolia records: projection, HTML extraction, legacy\n * anchor grouping, fallback records, deep links, identifiers, ranking metadata, and\n * deterministic record-size handling.\n *\n * The whole batch is validated before any record is returned. A deterministic policy, Ghost\n * content, or record-size problem throws one {@link FragmenterError} carrying every issue in\n * input order; a partial batch is never returned.\n *\n * @throws {FragmenterError} `INVALID_POLICY`, `INVALID_GHOST_CONTENT`, or `RECORD_TOO_LARGE`.\n */\nexport const createAlgoliaRecords = (\n ghostContent: readonly GhostContent[],\n options?: CreateAlgoliaRecordsOptions\n): readonly AlgoliaRecord[] => {\n const policy = resolvePolicy(options);\n if (!policy.ok) {\n throw new FragmenterError('INVALID_POLICY', policy.issues);\n }\n\n const prepared = prepareGhostContent(ghostContent, policy.policy);\n if (!prepared.ok) {\n throw new FragmenterError('INVALID_GHOST_CONTENT', prepared.issues);\n }\n\n const records: AlgoliaRecord[] = [];\n const issues: RecordSizeIssue[] = [];\n for (const content of prepared.contents) {\n const groups = groupFragmentsByAnchor(extract(content.html));\n const contentRecords = createContentRecords(content, groups);\n records.push(...contentRecords.records);\n issues.push(...contentRecords.issues);\n }\n\n if (issues.length > 0) {\n throw new FragmenterError('RECORD_TOO_LARGE', issues);\n }\n\n return records;\n};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/errors.d.mts b/packages/algolia-fragmenter/lib/errors.d.mts new file mode 100644 index 00000000..fd74625c --- /dev/null +++ b/packages/algolia-fragmenter/lib/errors.d.mts @@ -0,0 +1,44 @@ +export type FragmenterErrorCode = 'INVALID_POLICY' | 'INVALID_GHOST_CONTENT' | 'RECORD_TOO_LARGE'; +export type PolicyIssueReason = 'invalid-shape' | 'unknown-property' | 'unknown-source' | 'repeated-source' | 'repeated-output' | 'invalid-alias' | 'protected-collision' | 'container-collision' | 'canonical-collision' | 'reserved-collision'; +export type PolicyIssue = Readonly<{ + kind: 'policy'; + reason: PolicyIssueReason; + path: string; + message: string; +}>; +export type GhostContentIssueReason = 'invalid-shape' | 'missing' | 'wrong-type'; +export type ExpectedValueType = 'string' | 'number' | 'boolean' | 'object' | 'array'; +export type GhostContentIssue = Readonly<{ + kind: 'content'; + reason: GhostContentIssueReason; + path: string; + index: number | null; + contentId: string | null; + expected: ExpectedValueType; + message: string; +}>; +export type RecordSizeIssue = Readonly<{ + kind: 'size'; + reason: 'record-too-large'; + path: string; + index: number; + contentId: string; + objectID: string; + anchor: string | null; + position: number | null; + bytes: number; + limit: number; + excess: number; + message: string; +}>; +export type FragmenterIssue = PolicyIssue | GhostContentIssue | RecordSizeIssue; +/** + * The single public error for every deterministic policy, Ghost content, or record size + * problem found while building Algolia records. It never carries a partial record batch. + */ +export declare class FragmenterError extends Error { + readonly code: FragmenterErrorCode; + readonly issues: readonly FragmenterIssue[]; + constructor(code: FragmenterErrorCode, issues: readonly FragmenterIssue[]); +} +//# sourceMappingURL=errors.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/errors.d.mts.map b/packages/algolia-fragmenter/lib/errors.d.mts.map new file mode 100644 index 00000000..bda2520a --- /dev/null +++ b/packages/algolia-fragmenter/lib/errors.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"errors.d.mts","sourceRoot":"","sources":["../src/errors.mts"],"names":[],"mappings":"AAAA,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,GAAG,uBAAuB,GAAG,kBAAkB,CAAC;AAElG,MAAM,MAAM,iBAAiB,GACvB,eAAe,GACf,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,iBAAiB,GACjB,eAAe,GACf,qBAAqB,GACrB,qBAAqB,GACrB,qBAAqB,GACrB,oBAAoB,CAAC;AAE3B,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,iBAAiB,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC,CAAC;AAEH,MAAM,MAAM,uBAAuB,GAAG,eAAe,GAAG,SAAS,GAAG,YAAY,CAAC;AAEjF,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;AAErF,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC;IACrC,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,uBAAuB,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,kBAAkB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACnB,CAAC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,iBAAiB,GAAG,eAAe,CAAC;AAahF;;;GAGG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACtC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAE5C,YAAY,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,SAAS,eAAe,EAAE,EAKxE;CACJ"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/errors.mjs b/packages/algolia-fragmenter/lib/errors.mjs new file mode 100644 index 00000000..50bd8521 --- /dev/null +++ b/packages/algolia-fragmenter/lib/errors.mjs @@ -0,0 +1,23 @@ +const MESSAGE_ISSUE_LIMIT = 5; +const describeIssues = (code, issues) => { + const listed = issues.slice(0, MESSAGE_ISSUE_LIMIT).map(issue => issue.message); + const remaining = issues.length - listed.length; + const suffix = remaining > 0 ? `; and ${remaining} more` : ''; + const count = `${issues.length} issue${issues.length === 1 ? '' : 's'}`; + return `${code}: ${count}. ${listed.join('; ')}${suffix}`; +}; +/** + * The single public error for every deterministic policy, Ghost content, or record size + * problem found while building Algolia records. It never carries a partial record batch. + */ +export class FragmenterError extends Error { + code; + issues; + constructor(code, issues) { + super(describeIssues(code, issues)); + this.name = 'FragmenterError'; + this.code = code; + this.issues = Object.freeze([...issues]); + } +} +//# sourceMappingURL=errors.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/errors.mjs.map b/packages/algolia-fragmenter/lib/errors.mjs.map new file mode 100644 index 00000000..70882131 --- /dev/null +++ b/packages/algolia-fragmenter/lib/errors.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"errors.mjs","sourceRoot":"","sources":["../src/errors.mts"],"names":[],"mappings":"AAoDA,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,MAAM,cAAc,GAAG,CAAC,IAAyB,EAAE,MAAkC,EAAU,EAAE;IAC7F,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAChD,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,SAAS,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,MAAM,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,SAAS,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAExE,OAAO,GAAG,IAAI,KAAK,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC;AAC9D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC7B,IAAI,CAAsB;IAC1B,MAAM,CAA6B;IAE5C,YAAY,IAAyB,EAAE,MAAkC;QACrE,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IAC7C,CAAC;CACJ","sourcesContent":["export type FragmenterErrorCode = 'INVALID_POLICY' | 'INVALID_GHOST_CONTENT' | 'RECORD_TOO_LARGE';\n\nexport type PolicyIssueReason =\n | 'invalid-shape'\n | 'unknown-property'\n | 'unknown-source'\n | 'repeated-source'\n | 'repeated-output'\n | 'invalid-alias'\n | 'protected-collision'\n | 'container-collision'\n | 'canonical-collision'\n | 'reserved-collision';\n\nexport type PolicyIssue = Readonly<{\n kind: 'policy';\n reason: PolicyIssueReason;\n path: string;\n message: string;\n}>;\n\nexport type GhostContentIssueReason = 'invalid-shape' | 'missing' | 'wrong-type';\n\nexport type ExpectedValueType = 'string' | 'number' | 'boolean' | 'object' | 'array';\n\nexport type GhostContentIssue = Readonly<{\n kind: 'content';\n reason: GhostContentIssueReason;\n path: string;\n index: number | null;\n contentId: string | null;\n expected: ExpectedValueType;\n message: string;\n}>;\n\nexport type RecordSizeIssue = Readonly<{\n kind: 'size';\n reason: 'record-too-large';\n path: string;\n index: number;\n contentId: string;\n objectID: string;\n anchor: string | null;\n position: number | null;\n bytes: number;\n limit: number;\n excess: number;\n message: string;\n}>;\n\nexport type FragmenterIssue = PolicyIssue | GhostContentIssue | RecordSizeIssue;\n\nconst MESSAGE_ISSUE_LIMIT = 5;\n\nconst describeIssues = (code: FragmenterErrorCode, issues: readonly FragmenterIssue[]): string => {\n const listed = issues.slice(0, MESSAGE_ISSUE_LIMIT).map(issue => issue.message);\n const remaining = issues.length - listed.length;\n const suffix = remaining > 0 ? `; and ${remaining} more` : '';\n const count = `${issues.length} issue${issues.length === 1 ? '' : 's'}`;\n\n return `${code}: ${count}. ${listed.join('; ')}${suffix}`;\n};\n\n/**\n * The single public error for every deterministic policy, Ghost content, or record size\n * problem found while building Algolia records. It never carries a partial record batch.\n */\nexport class FragmenterError extends Error {\n readonly code: FragmenterErrorCode;\n readonly issues: readonly FragmenterIssue[];\n\n constructor(code: FragmenterErrorCode, issues: readonly FragmenterIssue[]) {\n super(describeIssues(code, issues));\n this.name = 'FragmenterError';\n this.code = code;\n this.issues = Object.freeze([...issues]);\n }\n}\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/grouping.d.mts b/packages/algolia-fragmenter/lib/grouping.d.mts new file mode 100644 index 00000000..211848b6 --- /dev/null +++ b/packages/algolia-fragmenter/lib/grouping.d.mts @@ -0,0 +1,18 @@ +import type { ExtractionFragment } from '@tryghost/algolia-html-extractor'; +export type NonEmptyFragments = readonly [ExtractionFragment, ...ExtractionFragment[]]; +export type FragmentGroup = Readonly<{ + anchor: string | null; + fragments: NonEmptyFragments; +}>; +/** + * Collects extraction fragments into first-seen anchor groups. Non-adjacent fragments that + * repeat an anchor join the existing group, which is the legacy grouping rule shared by the + * deprecated wrappers and the deep record interface. + */ +export declare const groupFragmentsByAnchor: (fragments: readonly ExtractionFragment[]) => readonly FragmentGroup[]; +/** + * Merges the fragments of one record. The first fragment contributes its markup verbatim; + * every later preformatted fragment contributes its text only. + */ +export declare const mergeRecordHtml: (fragments: readonly ExtractionFragment[]) => string; +//# sourceMappingURL=grouping.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/grouping.d.mts.map b/packages/algolia-fragmenter/lib/grouping.d.mts.map new file mode 100644 index 00000000..86dea892 --- /dev/null +++ b/packages/algolia-fragmenter/lib/grouping.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"grouping.d.mts","sourceRoot":"","sources":["../src/grouping.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,kCAAkC,CAAC;AAEzE,MAAM,MAAM,iBAAiB,GAAG,SAAS,CAAC,kBAAkB,EAAE,GAAG,kBAAkB,EAAE,CAAC,CAAC;AAEvF,MAAM,MAAM,aAAa,GAAG,QAAQ,CAAC;IACjC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,iBAAiB,CAAC;CAChC,CAAC,CAAC;AAOH;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,cACpB,SAAS,kBAAkB,EAAE,KACzC,SAAS,aAAa,EAYxB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,cAAe,SAAS,kBAAkB,EAAE,KAAG,MAM1E,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/grouping.mjs b/packages/algolia-fragmenter/lib/grouping.mjs new file mode 100644 index 00000000..17f6e95d --- /dev/null +++ b/packages/algolia-fragmenter/lib/grouping.mjs @@ -0,0 +1,27 @@ +/** + * Collects extraction fragments into first-seen anchor groups. Non-adjacent fragments that + * repeat an anchor join the existing group, which is the legacy grouping rule shared by the + * deprecated wrappers and the deep record interface. + */ +export const groupFragmentsByAnchor = (fragments) => { + const groups = []; + for (const fragment of fragments) { + const existingGroup = groups.find(group => group.anchor === fragment.anchor); + if (existingGroup === undefined) { + groups.push({ anchor: fragment.anchor, fragments: [fragment] }); + continue; + } + existingGroup.fragments.push(fragment); + } + return groups; +}; +/** + * Merges the fragments of one record. The first fragment contributes its markup verbatim; + * every later preformatted fragment contributes its text only. + */ +export const mergeRecordHtml = (fragments) => { + return fragments + .map((fragment, index) => index > 0 && fragment.sourceTag === 'pre' ? ` ${fragment.text}` : fragment.html) + .join(''); +}; +//# sourceMappingURL=grouping.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/grouping.mjs.map b/packages/algolia-fragmenter/lib/grouping.mjs.map new file mode 100644 index 00000000..6177736b --- /dev/null +++ b/packages/algolia-fragmenter/lib/grouping.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"grouping.mjs","sourceRoot":"","sources":["../src/grouping.mts"],"names":[],"mappings":"AAcA;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAClC,SAAwC,EAChB,EAAE;IAC1B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAC,CAAC,CAAC;YAC9D,SAAS;QACb,CAAC;QACD,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,SAAwC,EAAU,EAAE;IAChF,OAAO,SAAS;SACX,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CACrB,KAAK,GAAG,CAAC,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAClF;SACA,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC,CAAC","sourcesContent":["import type {ExtractionFragment} from '@tryghost/algolia-html-extractor';\n\nexport type NonEmptyFragments = readonly [ExtractionFragment, ...ExtractionFragment[]];\n\nexport type FragmentGroup = Readonly<{\n anchor: string | null;\n fragments: NonEmptyFragments;\n}>;\n\ntype MutableFragmentGroup = {\n anchor: string | null;\n fragments: [ExtractionFragment, ...ExtractionFragment[]];\n};\n\n/**\n * Collects extraction fragments into first-seen anchor groups. Non-adjacent fragments that\n * repeat an anchor join the existing group, which is the legacy grouping rule shared by the\n * deprecated wrappers and the deep record interface.\n */\nexport const groupFragmentsByAnchor = (\n fragments: readonly ExtractionFragment[]\n): readonly FragmentGroup[] => {\n const groups: MutableFragmentGroup[] = [];\n for (const fragment of fragments) {\n const existingGroup = groups.find(group => group.anchor === fragment.anchor);\n if (existingGroup === undefined) {\n groups.push({anchor: fragment.anchor, fragments: [fragment]});\n continue;\n }\n existingGroup.fragments.push(fragment);\n }\n\n return groups;\n};\n\n/**\n * Merges the fragments of one record. The first fragment contributes its markup verbatim;\n * every later preformatted fragment contributes its text only.\n */\nexport const mergeRecordHtml = (fragments: readonly ExtractionFragment[]): string => {\n return fragments\n .map((fragment, index) =>\n index > 0 && fragment.sourceTag === 'pre' ? ` ${fragment.text}` : fragment.html\n )\n .join('');\n};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/index.d.mts b/packages/algolia-fragmenter/lib/index.d.mts index ce8750c9..d88df5e6 100644 --- a/packages/algolia-fragmenter/lib/index.d.mts +++ b/packages/algolia-fragmenter/lib/index.d.mts @@ -1,5 +1,11 @@ -export type GhostContent = Readonly>; -export type AlgoliaRecord = Record; +import type { GhostContent } from './projection.mjs'; +import type { AlgoliaRecord } from './records.mjs'; +export { createAlgoliaRecords } from './create-algolia-records.mjs'; +export { FragmenterError } from './errors.mjs'; +export type { ExpectedValueType, FragmenterErrorCode, FragmenterIssue, GhostContentIssue, GhostContentIssueReason, PolicyIssue, PolicyIssueReason, RecordSizeIssue } from './errors.mjs'; +export type { ContentProjection, CreateAlgoliaRecordsOptions, OptionalProjectionSource, ProjectionField, RankingField, RankingSource } from './policy.mjs'; +export type { GhostContent } from './projection.mjs'; +export type { AlgoliaRecord } from './records.mjs'; /** * @deprecated Retained for compatibility while the deep record-building API is introduced. */ diff --git a/packages/algolia-fragmenter/lib/index.d.mts.map b/packages/algolia-fragmenter/lib/index.d.mts.map index 566ef681..efdf6ae0 100644 --- a/packages/algolia-fragmenter/lib/index.d.mts.map +++ b/packages/algolia-fragmenter/lib/index.d.mts.map @@ -1 +1 @@ -{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAEA,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAoEpD;;GAEG;AACH,eAAO,MAAM,mBAAmB,sBACT,aAAa,EAAE,gBACpB,aAAa,KAC5B,aAAa,EASf,CAAC;AAqBF;;GAEG;AACH,eAAO,MAAM,wBAAwB,UAC1B,SAAS,YAAY,EAAE,gBAChB,SAAS,MAAM,EAAE,KAChC,aAAa,EAqBf,CAAC"} \ No newline at end of file +{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,eAAe,CAAC;AAEjD,OAAO,EAAC,oBAAoB,EAAC,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAC,eAAe,EAAC,MAAM,cAAc,CAAC;AAC7C,YAAY,EACR,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,EACX,iBAAiB,EACjB,eAAe,EAClB,MAAM,cAAc,CAAC;AACtB,YAAY,EACR,iBAAiB,EACjB,2BAA2B,EAC3B,wBAAwB,EACxB,eAAe,EACf,YAAY,EACZ,aAAa,EAChB,MAAM,cAAc,CAAC;AACtB,YAAY,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AACnD,YAAY,EAAC,aAAa,EAAC,MAAM,eAAe,CAAC;AA+BjD;;GAEG;AACH,eAAO,MAAM,mBAAmB,sBACT,aAAa,EAAE,gBACpB,aAAa,KAC5B,aAAa,EAKf,CAAC;AAqBF;;GAEG;AACH,eAAO,MAAM,wBAAwB,UAC1B,SAAS,YAAY,EAAE,gBAChB,SAAS,MAAM,EAAE,KAChC,aAAa,EAqBf,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/index.mjs b/packages/algolia-fragmenter/lib/index.mjs index 4552b2f9..d14cee47 100644 --- a/packages/algolia-fragmenter/lib/index.mjs +++ b/packages/algolia-fragmenter/lib/index.mjs @@ -1,31 +1,19 @@ import { extract } from '@tryghost/algolia-html-extractor'; -const createLegacyFragment = (fragment) => ({ - html: fragment.html, - content: fragment.text, - headings: [...fragment.headingPath], - anchor: fragment.anchor, - sourceTag: fragment.sourceTag, - customRanking: { - position: fragment.position, - heading: fragment.headingRank - } -}); -const reduceFragmentsUnderHeadings = (groups, fragment) => { - const existingGroup = groups.find(group => group.anchor === fragment.anchor); - if (existingGroup === undefined) { - groups.push(fragment); - return groups; - } - existingGroup.html += fragment.sourceTag === 'pre' ? ` ${fragment.content}` : fragment.html; - existingGroup.content += ` ${fragment.content}`; - return groups; -}; -const toAlgoliaRecord = (ghostContent, fragment, index) => { - const { content: _content, sourceTag: _sourceTag, ...groupedFragment } = fragment; - const url = fragment.anchor === null ? ghostContent.url : `${ghostContent.url}#${fragment.anchor}`; +import { groupFragmentsByAnchor, mergeRecordHtml } from './grouping.mjs'; +export { createAlgoliaRecords } from './create-algolia-records.mjs'; +export { FragmenterError } from './errors.mjs'; +const toAlgoliaRecord = (ghostContent, group, index) => { + const [first] = group.fragments; + const url = group.anchor === null ? ghostContent.url : `${ghostContent.url}#${group.anchor}`; return { ...ghostContent, - ...groupedFragment, + html: mergeRecordHtml(group.fragments), + headings: [...first.headingPath], + anchor: group.anchor, + customRanking: { + position: first.position, + heading: first.headingRank + }, url, objectID: `${ghostContent.objectID}_${index}` }; @@ -34,10 +22,8 @@ const toAlgoliaRecord = (ghostContent, fragment, index) => { * @deprecated Retained for compatibility while the deep record-building API is introduced. */ export const fragmentTransformer = (recordAccumulator, ghostContent) => { - const groupedFragments = extract(ghostContent.html) - .map(createLegacyFragment) - .reduce(reduceFragmentsUnderHeadings, []); - const records = groupedFragments.map((fragment, index) => toAlgoliaRecord(ghostContent, fragment, index)); + const groups = groupFragmentsByAnchor(extract(ghostContent.html)); + const records = groups.map((group, index) => toAlgoliaRecord(ghostContent, group, index)); return [...recordAccumulator, ...records]; }; const projectLegacyRelations = (value, fieldName) => { diff --git a/packages/algolia-fragmenter/lib/index.mjs.map b/packages/algolia-fragmenter/lib/index.mjs.map index 0b00f7db..1ed631ac 100644 --- a/packages/algolia-fragmenter/lib/index.mjs.map +++ b/packages/algolia-fragmenter/lib/index.mjs.map @@ -1 +1 @@ -{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAA0C,MAAM,kCAAkC,CAAC;AA4BlG,MAAM,oBAAoB,GAAG,CAAC,QAA4C,EAAkB,EAAE,CAAC,CAAC;IAC5F,IAAI,EAAE,QAAQ,CAAC,IAAI;IACnB,OAAO,EAAE,QAAQ,CAAC,IAAI;IACtB,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC;IACnC,MAAM,EAAE,QAAQ,CAAC,MAAM;IACvB,SAAS,EAAE,QAAQ,CAAC,SAAS;IAC7B,aAAa,EAAE;QACX,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,OAAO,EAAE,QAAQ,CAAC,WAAW;KAChC;CACJ,CAAC,CAAC;AAEH,MAAM,4BAA4B,GAAG,CACjC,MAAwB,EACxB,QAAwB,EACR,EAAE;IAClB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,aAAa,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5F,aAAa,CAAC,OAAO,IAAI,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACpB,YAA2B,EAC3B,QAAwB,EACxB,KAAa,EACA,EAAE;IACf,MAAM,EAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,eAAe,EAAC,GAAG,QAAQ,CAAC;IAChF,MAAM,GAAG,GACL,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;IAE3F,OAAO;QACH,GAAG,YAAY;QACf,GAAG,eAAe;QAClB,GAAG;QACH,QAAQ,EAAE,GAAG,YAAY,CAAC,QAAQ,IAAI,KAAK,EAAE;KAChD,CAAC;AACN,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAC/B,iBAAkC,EAClC,YAA2B,EACZ,EAAE;IACjB,MAAM,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC,IAAc,CAAC;SACxD,GAAG,CAAC,oBAAoB,CAAC;SACzB,MAAM,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CACrD,eAAe,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjD,CAAC;IAEF,OAAO,CAAC,GAAG,iBAAiB,EAAE,GAAG,OAAO,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAC3B,KAAc,EACd,SAA6B,EACQ,EAAE;IACvC,MAAM,SAAS,GAAG,KAA6D,CAAC;IAChF,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,SAAS,CAAC,QAAQ,SAAS,4BAA4B,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,SAAS,GAA0C,EAAE,CAAC;IAC5D,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;QACzB,SAAS,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC;AACrB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACpC,KAA8B,EAC9B,WAA+B,EAChB,EAAE;IACjB,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,SAAS;QACb,CAAC;QAED,cAAc,CAAC,IAAI,CAAC;YAChB,QAAQ,EAAE,IAAI,CAAC,EAAE;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC/C,OAAO,EAAE,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC,CAAC;AAEF,eAAe,EAAC,mBAAmB,EAAE,wBAAwB,EAAC,CAAC","sourcesContent":["import {extract, type ExtractedTagName, type HeadingRank} from '@tryghost/algolia-html-extractor';\n\nexport type GhostContent = Readonly>;\n\nexport type AlgoliaRecord = Record;\n\ntype GhostRelation = Readonly>;\n\ntype LegacyRelationCollection = {\n length: number;\n forEach(callback: (relation: GhostRelation) => void): void;\n};\n\ntype GroupedFragment = {\n html: string;\n headings: string[];\n anchor: string | null;\n customRanking: {\n position: number;\n heading: HeadingRank;\n };\n};\n\ntype LegacyFragment = GroupedFragment & {\n content: string;\n sourceTag: ExtractedTagName;\n};\n\nconst createLegacyFragment = (fragment: ReturnType[number]): LegacyFragment => ({\n html: fragment.html,\n content: fragment.text,\n headings: [...fragment.headingPath],\n anchor: fragment.anchor,\n sourceTag: fragment.sourceTag,\n customRanking: {\n position: fragment.position,\n heading: fragment.headingRank\n }\n});\n\nconst reduceFragmentsUnderHeadings = (\n groups: LegacyFragment[],\n fragment: LegacyFragment\n): LegacyFragment[] => {\n const existingGroup = groups.find(group => group.anchor === fragment.anchor);\n if (existingGroup === undefined) {\n groups.push(fragment);\n return groups;\n }\n\n existingGroup.html += fragment.sourceTag === 'pre' ? ` ${fragment.content}` : fragment.html;\n existingGroup.content += ` ${fragment.content}`;\n return groups;\n};\n\nconst toAlgoliaRecord = (\n ghostContent: AlgoliaRecord,\n fragment: LegacyFragment,\n index: number\n): AlgoliaRecord => {\n const {content: _content, sourceTag: _sourceTag, ...groupedFragment} = fragment;\n const url =\n fragment.anchor === null ? ghostContent.url : `${ghostContent.url}#${fragment.anchor}`;\n\n return {\n ...ghostContent,\n ...groupedFragment,\n url,\n objectID: `${ghostContent.objectID}_${index}`\n };\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const fragmentTransformer = (\n recordAccumulator: AlgoliaRecord[],\n ghostContent: AlgoliaRecord\n): AlgoliaRecord[] => {\n const groupedFragments = extract(ghostContent.html as string)\n .map(createLegacyFragment)\n .reduce(reduceFragmentsUnderHeadings, []);\n const records = groupedFragments.map((fragment, index) =>\n toAlgoliaRecord(ghostContent, fragment, index)\n );\n\n return [...recordAccumulator, ...records];\n};\n\nconst projectLegacyRelations = (\n value: unknown,\n fieldName: 'tags' | 'authors'\n): Array<{name: unknown; slug: unknown}> => {\n const relations = value as Partial | null | undefined;\n if (!relations?.length) {\n return [];\n }\n if (typeof relations.forEach !== 'function') {\n throw new TypeError(`post.${fieldName}.forEach is not a function`);\n }\n\n const projected: Array<{name: unknown; slug: unknown}> = [];\n relations.forEach(relation => {\n projected.push({name: relation.name, slug: relation.slug});\n });\n return projected;\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const transformToAlgoliaObject = (\n posts: readonly GhostContent[],\n ignoreSlugs?: readonly string[]\n): AlgoliaRecord[] => {\n const algoliaObjects: AlgoliaRecord[] = [];\n\n for (const post of posts) {\n if (ignoreSlugs?.some(slug => slug === post.slug)) {\n continue;\n }\n\n algoliaObjects.push({\n objectID: post.id,\n slug: post.slug,\n url: post.url,\n html: post.html,\n image: post.feature_image,\n title: post.title,\n tags: projectLegacyRelations(post.tags, 'tags'),\n authors: projectLegacyRelations(post.authors, 'authors')\n });\n }\n\n return algoliaObjects;\n};\n\nexport default {fragmentTransformer, transformToAlgoliaObject};\n"]} \ No newline at end of file +{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../src/index.mts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,kCAAkC,CAAC;AAEzD,OAAO,EAAC,sBAAsB,EAAE,eAAe,EAAqB,MAAM,gBAAgB,CAAC;AAI3F,OAAO,EAAC,oBAAoB,EAAC,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAC,eAAe,EAAC,MAAM,cAAc,CAAC;AA6B7C,MAAM,eAAe,GAAG,CACpB,YAA2B,EAC3B,KAAoB,EACpB,KAAa,EACA,EAAE;IACf,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;IAE7F,OAAO;QACH,GAAG,YAAY;QACf,IAAI,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC;QACtC,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC;QAChC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,aAAa,EAAE;YACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,WAAW;SAC7B;QACD,GAAG;QACH,QAAQ,EAAE,GAAG,YAAY,CAAC,QAAQ,IAAI,KAAK,EAAE;KAChD,CAAC;AACN,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAC/B,iBAAkC,EAClC,YAA2B,EACZ,EAAE;IACjB,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,YAAY,CAAC,IAAc,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAE1F,OAAO,CAAC,GAAG,iBAAiB,EAAE,GAAG,OAAO,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAC3B,KAAc,EACd,SAA6B,EACQ,EAAE;IACvC,MAAM,SAAS,GAAG,KAA6D,CAAC;IAChF,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,SAAS,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,SAAS,CAAC,QAAQ,SAAS,4BAA4B,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,SAAS,GAA0C,EAAE,CAAC;IAC5D,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;QACzB,SAAS,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IACH,OAAO,SAAS,CAAC;AACrB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACpC,KAA8B,EAC9B,WAA+B,EAChB,EAAE;IACjB,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,SAAS;QACb,CAAC;QAED,cAAc,CAAC,IAAI,CAAC;YAChB,QAAQ,EAAE,IAAI,CAAC,EAAE;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC/C,OAAO,EAAE,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC,CAAC;AAEF,eAAe,EAAC,mBAAmB,EAAE,wBAAwB,EAAC,CAAC","sourcesContent":["import {extract} from '@tryghost/algolia-html-extractor';\n\nimport {groupFragmentsByAnchor, mergeRecordHtml, type FragmentGroup} from './grouping.mjs';\nimport type {GhostContent} from './projection.mjs';\nimport type {AlgoliaRecord} from './records.mjs';\n\nexport {createAlgoliaRecords} from './create-algolia-records.mjs';\nexport {FragmenterError} from './errors.mjs';\nexport type {\n ExpectedValueType,\n FragmenterErrorCode,\n FragmenterIssue,\n GhostContentIssue,\n GhostContentIssueReason,\n PolicyIssue,\n PolicyIssueReason,\n RecordSizeIssue\n} from './errors.mjs';\nexport type {\n ContentProjection,\n CreateAlgoliaRecordsOptions,\n OptionalProjectionSource,\n ProjectionField,\n RankingField,\n RankingSource\n} from './policy.mjs';\nexport type {GhostContent} from './projection.mjs';\nexport type {AlgoliaRecord} from './records.mjs';\n\ntype GhostRelation = Readonly>;\n\ntype LegacyRelationCollection = {\n length: number;\n forEach(callback: (relation: GhostRelation) => void): void;\n};\n\nconst toAlgoliaRecord = (\n ghostContent: AlgoliaRecord,\n group: FragmentGroup,\n index: number\n): AlgoliaRecord => {\n const [first] = group.fragments;\n const url = group.anchor === null ? ghostContent.url : `${ghostContent.url}#${group.anchor}`;\n\n return {\n ...ghostContent,\n html: mergeRecordHtml(group.fragments),\n headings: [...first.headingPath],\n anchor: group.anchor,\n customRanking: {\n position: first.position,\n heading: first.headingRank\n },\n url,\n objectID: `${ghostContent.objectID}_${index}`\n };\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const fragmentTransformer = (\n recordAccumulator: AlgoliaRecord[],\n ghostContent: AlgoliaRecord\n): AlgoliaRecord[] => {\n const groups = groupFragmentsByAnchor(extract(ghostContent.html as string));\n const records = groups.map((group, index) => toAlgoliaRecord(ghostContent, group, index));\n\n return [...recordAccumulator, ...records];\n};\n\nconst projectLegacyRelations = (\n value: unknown,\n fieldName: 'tags' | 'authors'\n): Array<{name: unknown; slug: unknown}> => {\n const relations = value as Partial | null | undefined;\n if (!relations?.length) {\n return [];\n }\n if (typeof relations.forEach !== 'function') {\n throw new TypeError(`post.${fieldName}.forEach is not a function`);\n }\n\n const projected: Array<{name: unknown; slug: unknown}> = [];\n relations.forEach(relation => {\n projected.push({name: relation.name, slug: relation.slug});\n });\n return projected;\n};\n\n/**\n * @deprecated Retained for compatibility while the deep record-building API is introduced.\n */\nexport const transformToAlgoliaObject = (\n posts: readonly GhostContent[],\n ignoreSlugs?: readonly string[]\n): AlgoliaRecord[] => {\n const algoliaObjects: AlgoliaRecord[] = [];\n\n for (const post of posts) {\n if (ignoreSlugs?.some(slug => slug === post.slug)) {\n continue;\n }\n\n algoliaObjects.push({\n objectID: post.id,\n slug: post.slug,\n url: post.url,\n html: post.html,\n image: post.feature_image,\n title: post.title,\n tags: projectLegacyRelations(post.tags, 'tags'),\n authors: projectLegacyRelations(post.authors, 'authors')\n });\n }\n\n return algoliaObjects;\n};\n\nexport default {fragmentTransformer, transformToAlgoliaObject};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/policy.d.mts b/packages/algolia-fragmenter/lib/policy.d.mts new file mode 100644 index 00000000..d82680f7 --- /dev/null +++ b/packages/algolia-fragmenter/lib/policy.d.mts @@ -0,0 +1,47 @@ +import type { PolicyIssue } from './errors.mjs'; +export type OptionalProjectionSource = 'image' | 'tags' | 'authors' | 'excerpt' | 'custom_excerpt' | 'feature_image_alt' | 'feature_image_caption' | 'canonical_url' | 'featured' | 'visibility' | 'created_at' | 'updated_at' | 'published_at' | 'reading_time'; +export type ProjectionField = OptionalProjectionSource | Readonly<{ + source: OptionalProjectionSource; + as: string; +}>; +export type RankingSource = 'featured' | 'reading_time'; +export type RankingField = Readonly<{ + source: RankingSource; + as: string; +}>; +export type ContentProjection = Readonly<{ + fields: readonly ProjectionField[]; + customRanking?: readonly RankingField[]; +}>; +export type CreateAlgoliaRecordsOptions = Readonly<{ + ignoreSlugs?: readonly string[]; + contentProjection?: ContentProjection; +}>; +export type ResolvedProjectionField = Readonly<{ + source: OptionalProjectionSource; + outputKey: string; +}>; +export type ResolvedRankingField = Readonly<{ + source: RankingSource; + outputKey: string; +}>; +export type ResolvedPolicy = Readonly<{ + ignoreSlugs: readonly string[]; + fields: readonly ResolvedProjectionField[]; + rankingFields: readonly ResolvedRankingField[]; +}>; +export type PolicyResolution = Readonly<{ + ok: true; + policy: ResolvedPolicy; +}> | Readonly<{ + ok: false; + issues: readonly PolicyIssue[]; +}>; +export declare const isPlainObject: (value: unknown) => value is Record; +/** + * Validates caller options before any Ghost content is inspected and returns the policy the + * projection, ranking, and record stages read. Every policy issue is collected in declaration + * order rather than stopping at the first one. + */ +export declare const resolvePolicy: (options: unknown) => PolicyResolution; +//# sourceMappingURL=policy.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/policy.d.mts.map b/packages/algolia-fragmenter/lib/policy.d.mts.map new file mode 100644 index 00000000..0345561a --- /dev/null +++ b/packages/algolia-fragmenter/lib/policy.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"policy.d.mts","sourceRoot":"","sources":["../src/policy.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAoB,MAAM,cAAc,CAAC;AAEjE,MAAM,MAAM,wBAAwB,GAC9B,OAAO,GACP,MAAM,GACN,SAAS,GACT,SAAS,GACT,gBAAgB,GAChB,mBAAmB,GACnB,uBAAuB,GACvB,eAAe,GACf,UAAU,GACV,YAAY,GACZ,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,cAAc,CAAC;AAErB,MAAM,MAAM,eAAe,GACrB,wBAAwB,GACxB,QAAQ,CAAC;IACL,MAAM,EAAE,wBAAwB,CAAC;IACjC,EAAE,EAAE,MAAM,CAAC;CACd,CAAC,CAAC;AAET,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,cAAc,CAAC;AAExD,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC;IAChC,MAAM,EAAE,aAAa,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;CACd,CAAC,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC;IACrC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IACnC,aAAa,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAC3C,CAAC,CAAC;AAEH,MAAM,MAAM,2BAA2B,GAAG,QAAQ,CAAC;IAC/C,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACzC,CAAC,CAAC;AAEH,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAC3C,MAAM,EAAE,wBAAwB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACrB,CAAC,CAAC;AAEH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,CAAC;IACxC,MAAM,EAAE,aAAa,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACrB,CAAC,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC;IAClC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B,MAAM,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAC3C,aAAa,EAAE,SAAS,oBAAoB,EAAE,CAAC;CAClD,CAAC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GACtB,QAAQ,CAAC;IAAC,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,cAAc,CAAA;CAAC,CAAC,GAC5C,QAAQ,CAAC;IAAC,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CAAC,CAAC,CAAC;AAiF5D,eAAO,MAAM,aAAa,UAAW,OAAO,KAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAE7E,CAAC;AAgVF;;;;GAIG;AACH,eAAO,MAAM,aAAa,YAAa,OAAO,KAAG,gBAmBhD,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/policy.mjs b/packages/algolia-fragmenter/lib/policy.mjs new file mode 100644 index 00000000..29e78956 --- /dev/null +++ b/packages/algolia-fragmenter/lib/policy.mjs @@ -0,0 +1,271 @@ +/** + * Every canonical allowlist name. The `Record` keeps this set exhaustive: a source added to + * `OptionalProjectionSource` without a name here fails to compile. + */ +const CANONICAL_SOURCE_NAMES = { + image: true, + tags: true, + authors: true, + excerpt: true, + custom_excerpt: true, + feature_image_alt: true, + feature_image_caption: true, + canonical_url: true, + featured: true, + visibility: true, + created_at: true, + updated_at: true, + published_at: true, + reading_time: true +}; +const RANKING_SOURCE_NAMES = { + featured: true, + reading_time: true +}; +const DEFAULT_PROJECTION_SOURCES = [ + 'image', + 'tags', + 'authors', + 'excerpt' +]; +const PROTECTED_RECORD_FIELDS = [ + 'objectID', + 'slug', + 'url', + 'title', + 'html', + 'headings', + 'anchor' +]; +const PROTECTED_RANKING_FIELDS = ['heading', 'position']; +const PROTECTED_RANKING_OUTPUT_NAMES = [ + ...PROTECTED_RECORD_FIELDS, + ...PROTECTED_RANKING_FIELDS +]; +const ALGOLIA_RESERVED_NAMES = [ + '_highlightResult', + '_snippetResult', + '_rankingInfo', + '_distinctSeqID', + 'distinctSeqId', + '_tags', + '_geoloc' +]; +const RANKING_CONTAINER = 'customRanking'; +const ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/u; +const OPTIONS_PROPERTIES = ['ignoreSlugs', 'contentProjection']; +const CONTENT_PROJECTION_PROPERTIES = ['fields', 'customRanking']; +const PROJECTION_FIELD_PROPERTIES = ['source', 'as']; +export const isPlainObject = (value) => { + return typeof value === 'object' && value !== null && !Array.isArray(value); +}; +const isProjectionSource = (value) => { + return Object.hasOwn(CANONICAL_SOURCE_NAMES, value); +}; +const isRankingSource = (value) => { + return Object.hasOwn(RANKING_SOURCE_NAMES, value); +}; +const createIssue = (reason, path, message) => ({ + kind: 'policy', + reason, + path, + message +}); +const invalidShape = (path, expectedShape) => createIssue('invalid-shape', path, `${path}: expected ${expectedShape}.`); +const unknownProperty = (path, name) => createIssue('unknown-property', path, `${path}: unknown property "${name}".`); +const unknownSource = (path, value, kind) => createIssue('unknown-source', path, `${path}: "${value}" is not an allowed ${kind} source.`); +const repeatedSource = (path, source, kind) => createIssue('repeated-source', path, `${path}: ${kind} source "${source}" is configured more than once.`); +const invalidAlias = (path, name) => createIssue('invalid-alias', path, `${path}: alias "${name}" must match ^[A-Za-z][A-Za-z0-9_]*$.`); +const findUnknownProperties = (value, allowed) => { + return Object.keys(value).filter(key => !allowed.includes(key)); +}; +const collectUnknownProperties = (value, allowed, path, issues) => { + for (const key of findUnknownProperties(value, allowed)) { + issues.push(unknownProperty(path, key)); + } +}; +/** + * Output names live in one policy-wide namespace shared by projection fields and ranking + * siblings, so the checks below run in a fixed order for every configured output name. + */ +const findOutputCollision = (outputName, source, path, protectedNames, usedOutputNames) => { + if (protectedNames.includes(outputName)) { + const owner = PROTECTED_RANKING_FIELDS.includes(outputName) ? 'ranking' : 'record'; + return createIssue('protected-collision', path, `${path}: output name "${outputName}" is a protected ${owner} field.`); + } + if (outputName === RANKING_CONTAINER) { + return createIssue('container-collision', path, `${path}: output name "customRanking" is the package-owned ranking container.`); + } + if (ALGOLIA_RESERVED_NAMES.includes(outputName)) { + return createIssue('reserved-collision', path, `${path}: output name "${outputName}" is reserved by Algolia.`); + } + if (outputName !== source && isProjectionSource(outputName)) { + return createIssue('canonical-collision', path, `${path}: output name "${outputName}" impersonates a canonical allowlist field.`); + } + if (usedOutputNames.has(outputName)) { + return createIssue('repeated-output', path, `${path}: output name "${outputName}" is produced more than once.`); + } + return null; +}; +const readAliasedFieldShape = (entry, path, expectedShape) => { + if (!isPlainObject(entry)) { + return { ok: false, issue: invalidShape(path, expectedShape) }; + } + const [unknownKey] = findUnknownProperties(entry, PROJECTION_FIELD_PROPERTIES); + if (unknownKey !== undefined) { + return { ok: false, issue: unknownProperty(path, unknownKey) }; + } + if (typeof entry.source !== 'string' || typeof entry.as !== 'string') { + return { ok: false, issue: invalidShape(path, 'a {source, as} object of strings') }; + } + return { ok: true, value: { source: entry.source, alias: entry.as } }; +}; +const readProjectionFieldShape = (entry, path) => { + if (typeof entry === 'string') { + return { ok: true, value: { source: entry, alias: null } }; + } + return readAliasedFieldShape(entry, path, 'a projection source name or a {source, as} object'); +}; +const resolveProjectionField = (entry, path, usedSources, usedOutputNames) => { + const shape = readProjectionFieldShape(entry, path); + if (!shape.ok) { + return shape; + } + const { source, alias } = shape.value; + if (!isProjectionSource(source)) { + return { ok: false, issue: unknownSource(path, source, 'projection') }; + } + if (usedSources.has(source)) { + return { ok: false, issue: repeatedSource(path, source, 'projection') }; + } + const outputPath = alias === null ? path : `${path}.as`; + if (alias !== null && !ALIAS_PATTERN.test(alias)) { + return { ok: false, issue: invalidAlias(outputPath, alias) }; + } + const outputKey = alias ?? source; + const collision = findOutputCollision(outputKey, source, outputPath, PROTECTED_RECORD_FIELDS, usedOutputNames); + if (collision !== null) { + return { ok: false, issue: collision }; + } + usedSources.add(source); + usedOutputNames.add(outputKey); + return { ok: true, value: { source, outputKey } }; +}; +const resolveRankingField = (entry, path, usedSources, usedOutputNames) => { + const shape = readAliasedFieldShape(entry, path, 'a {source, as} object'); + if (!shape.ok) { + return shape; + } + const { source, alias } = shape.value; + if (!isRankingSource(source)) { + return { ok: false, issue: unknownSource(path, source, 'ranking') }; + } + if (usedSources.has(source)) { + return { ok: false, issue: repeatedSource(path, source, 'ranking') }; + } + const outputPath = `${path}.as`; + if (!ALIAS_PATTERN.test(alias)) { + return { ok: false, issue: invalidAlias(outputPath, alias) }; + } + const collision = findOutputCollision(alias, source, outputPath, PROTECTED_RANKING_OUTPUT_NAMES, usedOutputNames); + if (collision !== null) { + return { ok: false, issue: collision }; + } + usedSources.add(source); + usedOutputNames.add(alias); + return { ok: true, value: { source, outputKey: alias } }; +}; +/** + * Resolves one configured list. Sources are unique per list, while output names are checked + * against the policy-wide namespace the caller owns. + */ +const resolveEntries = (entries, listPath, usedOutputNames, issues, resolveEntry) => { + const usedSources = new Set(); + const fields = []; + for (const [index, entry] of entries.entries()) { + const resolved = resolveEntry(entry, `${listPath}[${index}]`, usedSources, usedOutputNames); + if (!resolved.ok) { + issues.push(resolved.issue); + continue; + } + fields.push(resolved.value); + } + return fields; +}; +const resolveFields = (value, usedOutputNames, issues) => { + if (!Array.isArray(value)) { + issues.push(invalidShape('contentProjection.fields', 'an array of projection fields')); + return []; + } + return resolveEntries(value, 'contentProjection.fields', usedOutputNames, issues, resolveProjectionField); +}; +const resolveRankingFields = (value, usedOutputNames, issues) => { + if (value === undefined) { + return []; + } + if (!Array.isArray(value)) { + issues.push(invalidShape('contentProjection.customRanking', 'an array of ranking fields')); + return []; + } + return resolveEntries(value, 'contentProjection.customRanking', usedOutputNames, issues, resolveRankingField); +}; +const createDefaultProjection = () => ({ + fields: DEFAULT_PROJECTION_SOURCES.map(source => ({ source, outputKey: source })), + rankingFields: [] +}); +const resolveContentProjection = (value, issues) => { + if (value === undefined) { + return createDefaultProjection(); + } + if (!isPlainObject(value)) { + issues.push(invalidShape('contentProjection', 'an object')); + return { fields: [], rankingFields: [] }; + } + collectUnknownProperties(value, CONTENT_PROJECTION_PROPERTIES, 'contentProjection', issues); + const usedOutputNames = new Set(); + return { + fields: resolveFields(value.fields, usedOutputNames, issues), + rankingFields: resolveRankingFields(value.customRanking, usedOutputNames, issues) + }; +}; +const resolveIgnoreSlugs = (value, issues) => { + if (value === undefined) { + return []; + } + if (!Array.isArray(value)) { + issues.push(invalidShape('ignoreSlugs', 'an array of strings')); + return []; + } + const slugs = []; + for (const [index, entry] of value.entries()) { + if (typeof entry !== 'string') { + issues.push(invalidShape(`ignoreSlugs[${index}]`, 'a string')); + continue; + } + slugs.push(entry); + } + return slugs; +}; +/** + * Validates caller options before any Ghost content is inspected and returns the policy the + * projection, ranking, and record stages read. Every policy issue is collected in declaration + * order rather than stopping at the first one. + */ +export const resolvePolicy = (options) => { + if (options === undefined) { + const projection = createDefaultProjection(); + return { ok: true, policy: { ignoreSlugs: [], ...projection } }; + } + if (!isPlainObject(options)) { + return { ok: false, issues: [invalidShape('options', 'an object')] }; + } + const issues = []; + collectUnknownProperties(options, OPTIONS_PROPERTIES, 'options', issues); + const ignoreSlugs = resolveIgnoreSlugs(options.ignoreSlugs, issues); + const projection = resolveContentProjection(options.contentProjection, issues); + if (issues.length > 0) { + return { ok: false, issues }; + } + return { ok: true, policy: { ignoreSlugs, ...projection } }; +}; +//# sourceMappingURL=policy.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/policy.mjs.map b/packages/algolia-fragmenter/lib/policy.mjs.map new file mode 100644 index 00000000..ba59705c --- /dev/null +++ b/packages/algolia-fragmenter/lib/policy.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"policy.mjs","sourceRoot":"","sources":["../src/policy.mts"],"names":[],"mappings":"AA2EA;;;GAGG;AACH,MAAM,sBAAsB,GAAqD;IAC7E,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,IAAI;IACvB,qBAAqB,EAAE,IAAI;IAC3B,aAAa,EAAE,IAAI;IACnB,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,IAAI;IAChB,YAAY,EAAE,IAAI;IAClB,YAAY,EAAE,IAAI;CACrB,CAAC;AAEF,MAAM,oBAAoB,GAA0C;IAChE,QAAQ,EAAE,IAAI;IACd,YAAY,EAAE,IAAI;CACrB,CAAC;AAEF,MAAM,0BAA0B,GAAwC;IACpE,OAAO;IACP,MAAM;IACN,SAAS;IACT,SAAS;CACZ,CAAC;AAEF,MAAM,uBAAuB,GAAsB;IAC/C,UAAU;IACV,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,UAAU;IACV,QAAQ;CACX,CAAC;AAEF,MAAM,wBAAwB,GAAsB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,8BAA8B,GAAsB;IACtD,GAAG,uBAAuB;IAC1B,GAAG,wBAAwB;CAC9B,CAAC;AAEF,MAAM,sBAAsB,GAAsB;IAC9C,kBAAkB;IAClB,gBAAgB;IAChB,cAAc;IACd,gBAAgB;IAChB,eAAe;IACf,OAAO;IACP,SAAS;CACZ,CAAC;AAEF,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAC1C,MAAM,aAAa,GAAG,0BAA0B,CAAC;AACjD,MAAM,kBAAkB,GAAsB,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC;AACnF,MAAM,6BAA6B,GAAsB,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;AACrF,MAAM,2BAA2B,GAAsB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AAExE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAc,EAAoC,EAAE;IAC9E,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAChF,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAa,EAAqC,EAAE;IAC5E,OAAO,MAAM,CAAC,MAAM,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,KAAa,EAA0B,EAAE;IAC9D,OAAO,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,MAAyB,EAAE,IAAY,EAAE,OAAe,EAAe,EAAE,CAAC,CAAC;IAC5F,IAAI,EAAE,QAAQ;IACd,MAAM;IACN,IAAI;IACJ,OAAO;CACV,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,aAAqB,EAAe,EAAE,CACtE,WAAW,CAAC,eAAe,EAAE,IAAI,EAAE,GAAG,IAAI,cAAc,aAAa,GAAG,CAAC,CAAC;AAE9E,MAAM,eAAe,GAAG,CAAC,IAAY,EAAE,IAAY,EAAe,EAAE,CAChE,WAAW,CAAC,kBAAkB,EAAE,IAAI,EAAE,GAAG,IAAI,uBAAuB,IAAI,IAAI,CAAC,CAAC;AAElF,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,IAAY,EAAe,EAAE,CAC7E,WAAW,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,IAAI,MAAM,KAAK,uBAAuB,IAAI,UAAU,CAAC,CAAC;AAEjG,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,MAAc,EAAE,IAAY,EAAe,EAAE,CAC/E,WAAW,CACP,iBAAiB,EACjB,IAAI,EACJ,GAAG,IAAI,KAAK,IAAI,YAAY,MAAM,iCAAiC,CACtE,CAAC;AAEN,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAY,EAAe,EAAE,CAC7D,WAAW,CACP,eAAe,EACf,IAAI,EACJ,GAAG,IAAI,YAAY,IAAI,uCAAuC,CACjE,CAAC;AAEN,MAAM,qBAAqB,GAAG,CAC1B,KAAwC,EACxC,OAA0B,EACT,EAAE;IACnB,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC7B,KAAwC,EACxC,OAA0B,EAC1B,IAAY,EACZ,MAAqB,EACjB,EAAE;IACN,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IAC5C,CAAC;AACL,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,mBAAmB,GAAG,CACxB,UAAkB,EAClB,MAAc,EACd,IAAY,EACZ,cAAiC,EACjC,eAAoC,EAClB,EAAE;IACpB,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,wBAAwB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnF,OAAO,WAAW,CACd,qBAAqB,EACrB,IAAI,EACJ,GAAG,IAAI,kBAAkB,UAAU,oBAAoB,KAAK,SAAS,CACxE,CAAC;IACN,CAAC;IACD,IAAI,UAAU,KAAK,iBAAiB,EAAE,CAAC;QACnC,OAAO,WAAW,CACd,qBAAqB,EACrB,IAAI,EACJ,GAAG,IAAI,uEAAuE,CACjF,CAAC;IACN,CAAC;IACD,IAAI,sBAAsB,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9C,OAAO,WAAW,CACd,oBAAoB,EACpB,IAAI,EACJ,GAAG,IAAI,kBAAkB,UAAU,2BAA2B,CACjE,CAAC;IACN,CAAC;IACD,IAAI,UAAU,KAAK,MAAM,IAAI,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1D,OAAO,WAAW,CACd,qBAAqB,EACrB,IAAI,EACJ,GAAG,IAAI,kBAAkB,UAAU,6CAA6C,CACnF,CAAC;IACN,CAAC;IACD,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,OAAO,WAAW,CACd,iBAAiB,EACjB,IAAI,EACJ,GAAG,IAAI,kBAAkB,UAAU,+BAA+B,CACrE,CAAC;IACN,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAC1B,KAAc,EACd,IAAY,EACZ,aAAqB,EACQ,EAAE;IAC/B,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC,EAAC,CAAC;IACjE,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,GAAG,qBAAqB,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;IAC/E,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,EAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QACnE,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,IAAI,EAAE,kCAAkC,CAAC,EAAC,CAAC;IACtF,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAC,EAAC,CAAC;AACtE,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAC7B,KAAc,EACd,IAAY,EACoB,EAAE;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAC,EAAC,CAAC;IAC3D,CAAC;IAED,OAAO,qBAAqB,CAAC,KAAK,EAAE,IAAI,EAAE,mDAAmD,CAAC,CAAC;AACnG,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAC3B,KAAc,EACd,IAAY,EACZ,WAAwB,EACxB,eAA4B,EACO,EAAE;IACrC,MAAM,KAAK,GAAG,wBAAwB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,EAAC,MAAM,EAAE,KAAK,EAAC,GAAG,KAAK,CAAC,KAAK,CAAC;IACpC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,EAAC,CAAC;IACzE,CAAC;IACD,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,EAAC,CAAC;IAC1E,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC;IACxD,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,EAAC,CAAC;IAC/D,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,IAAI,MAAM,CAAC;IAClC,MAAM,SAAS,GAAG,mBAAmB,CACjC,SAAS,EACT,MAAM,EACN,UAAU,EACV,uBAAuB,EACvB,eAAe,CAClB,CAAC;IACF,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACrB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAC,CAAC;IACzC,CAAC;IAED,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,SAAS,EAAC,EAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CACxB,KAAc,EACd,IAAY,EACZ,WAAwB,EACxB,eAA4B,EACI,EAAE;IAClC,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,EAAE,uBAAuB,CAAC,CAAC;IAC1E,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,EAAC,MAAM,EAAE,KAAK,EAAC,GAAG,KAAK,CAAC,KAAK,CAAC;IACpC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAC,CAAC;IACtE,CAAC;IACD,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAC,CAAC;IACvE,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,IAAI,KAAK,CAAC;IAChC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,EAAC,CAAC;IAC/D,CAAC;IAED,MAAM,SAAS,GAAG,mBAAmB,CACjC,KAAK,EACL,MAAM,EACN,UAAU,EACV,8BAA8B,EAC9B,eAAe,CAClB,CAAC;IACF,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACrB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAC,CAAC;IACzC,CAAC;IAED,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAC,EAAC,CAAC;AACzD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,cAAc,GAAG,CACnB,OAA2B,EAC3B,QAAgB,EAChB,eAA4B,EAC5B,MAAqB,EACrB,YAKsB,EACN,EAAE;IAClB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,EAAE,GAAG,QAAQ,IAAI,KAAK,GAAG,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC;QAC5F,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,SAAS;QACb,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAClB,KAAc,EACd,eAA4B,EAC5B,MAAqB,EACa,EAAE;IACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,0BAA0B,EAAE,+BAA+B,CAAC,CAAC,CAAC;QACvF,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,cAAc,CACjB,KAAK,EACL,0BAA0B,EAC1B,eAAe,EACf,MAAM,EACN,sBAAsB,CACzB,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CACzB,KAAc,EACd,eAA4B,EAC5B,MAAqB,EACU,EAAE;IACjC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,iCAAiC,EAAE,4BAA4B,CAAC,CAAC,CAAC;QAC3F,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,cAAc,CACjB,KAAK,EACL,iCAAiC,EACjC,eAAe,EACf,MAAM,EACN,mBAAmB,CACtB,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,GAAuB,EAAE,CAAC,CAAC;IACvD,MAAM,EAAE,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAC,CAAC,CAAC;IAC/E,aAAa,EAAE,EAAE;CACpB,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,CAAC,KAAc,EAAE,MAAqB,EAAsB,EAAE;IAC3F,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,uBAAuB,EAAE,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC,CAAC;QAC5D,OAAO,EAAC,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAC,CAAC;IAC3C,CAAC;IAED,wBAAwB,CAAC,KAAK,EAAE,6BAA6B,EAAE,mBAAmB,EAAE,MAAM,CAAC,CAAC;IAC5F,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,OAAO;QACH,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAAC;QAC5D,aAAa,EAAE,oBAAoB,CAAC,KAAK,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC;KACpF,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAc,EAAE,MAAqB,EAAqB,EAAE;IACpF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAChE,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,KAAK,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;YAC/D,SAAS;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAgB,EAAoB,EAAE;IAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,uBAAuB,EAAE,CAAC;QAC7C,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAC,WAAW,EAAE,EAAE,EAAE,GAAG,UAAU,EAAC,EAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,EAAC,CAAC;IACvE,CAAC;IAED,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,wBAAwB,CAAC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,wBAAwB,CAAC,OAAO,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAE/E,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAC,CAAC;IAC/B,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAC,WAAW,EAAE,GAAG,UAAU,EAAC,EAAC,CAAC;AAC5D,CAAC,CAAC","sourcesContent":["import type {PolicyIssue, PolicyIssueReason} from './errors.mjs';\n\nexport type OptionalProjectionSource =\n | 'image'\n | 'tags'\n | 'authors'\n | 'excerpt'\n | 'custom_excerpt'\n | 'feature_image_alt'\n | 'feature_image_caption'\n | 'canonical_url'\n | 'featured'\n | 'visibility'\n | 'created_at'\n | 'updated_at'\n | 'published_at'\n | 'reading_time';\n\nexport type ProjectionField =\n | OptionalProjectionSource\n | Readonly<{\n source: OptionalProjectionSource;\n as: string;\n }>;\n\nexport type RankingSource = 'featured' | 'reading_time';\n\nexport type RankingField = Readonly<{\n source: RankingSource;\n as: string;\n}>;\n\nexport type ContentProjection = Readonly<{\n fields: readonly ProjectionField[];\n customRanking?: readonly RankingField[];\n}>;\n\nexport type CreateAlgoliaRecordsOptions = Readonly<{\n ignoreSlugs?: readonly string[];\n contentProjection?: ContentProjection;\n}>;\n\nexport type ResolvedProjectionField = Readonly<{\n source: OptionalProjectionSource;\n outputKey: string;\n}>;\n\nexport type ResolvedRankingField = Readonly<{\n source: RankingSource;\n outputKey: string;\n}>;\n\nexport type ResolvedPolicy = Readonly<{\n ignoreSlugs: readonly string[];\n fields: readonly ResolvedProjectionField[];\n rankingFields: readonly ResolvedRankingField[];\n}>;\n\nexport type PolicyResolution =\n | Readonly<{ok: true; policy: ResolvedPolicy}>\n | Readonly<{ok: false; issues: readonly PolicyIssue[]}>;\n\ntype AliasedFieldShape = Readonly<{source: string; alias: string}>;\n\ntype ProjectionFieldShape = Readonly<{source: string; alias: string | null}>;\n\ntype Resolution =\n | Readonly<{ok: true; value: Value}>\n | Readonly<{ok: false; issue: PolicyIssue}>;\n\ntype ResolvedProjection = Readonly<{\n fields: readonly ResolvedProjectionField[];\n rankingFields: readonly ResolvedRankingField[];\n}>;\n\n/**\n * Every canonical allowlist name. The `Record` keeps this set exhaustive: a source added to\n * `OptionalProjectionSource` without a name here fails to compile.\n */\nconst CANONICAL_SOURCE_NAMES: Readonly> = {\n image: true,\n tags: true,\n authors: true,\n excerpt: true,\n custom_excerpt: true,\n feature_image_alt: true,\n feature_image_caption: true,\n canonical_url: true,\n featured: true,\n visibility: true,\n created_at: true,\n updated_at: true,\n published_at: true,\n reading_time: true\n};\n\nconst RANKING_SOURCE_NAMES: Readonly> = {\n featured: true,\n reading_time: true\n};\n\nconst DEFAULT_PROJECTION_SOURCES: readonly OptionalProjectionSource[] = [\n 'image',\n 'tags',\n 'authors',\n 'excerpt'\n];\n\nconst PROTECTED_RECORD_FIELDS: readonly string[] = [\n 'objectID',\n 'slug',\n 'url',\n 'title',\n 'html',\n 'headings',\n 'anchor'\n];\n\nconst PROTECTED_RANKING_FIELDS: readonly string[] = ['heading', 'position'];\n\nconst PROTECTED_RANKING_OUTPUT_NAMES: readonly string[] = [\n ...PROTECTED_RECORD_FIELDS,\n ...PROTECTED_RANKING_FIELDS\n];\n\nconst ALGOLIA_RESERVED_NAMES: readonly string[] = [\n '_highlightResult',\n '_snippetResult',\n '_rankingInfo',\n '_distinctSeqID',\n 'distinctSeqId',\n '_tags',\n '_geoloc'\n];\n\nconst RANKING_CONTAINER = 'customRanking';\nconst ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/u;\nconst OPTIONS_PROPERTIES: readonly string[] = ['ignoreSlugs', 'contentProjection'];\nconst CONTENT_PROJECTION_PROPERTIES: readonly string[] = ['fields', 'customRanking'];\nconst PROJECTION_FIELD_PROPERTIES: readonly string[] = ['source', 'as'];\n\nexport const isPlainObject = (value: unknown): value is Record => {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n};\n\nconst isProjectionSource = (value: string): value is OptionalProjectionSource => {\n return Object.hasOwn(CANONICAL_SOURCE_NAMES, value);\n};\n\nconst isRankingSource = (value: string): value is RankingSource => {\n return Object.hasOwn(RANKING_SOURCE_NAMES, value);\n};\n\nconst createIssue = (reason: PolicyIssueReason, path: string, message: string): PolicyIssue => ({\n kind: 'policy',\n reason,\n path,\n message\n});\n\nconst invalidShape = (path: string, expectedShape: string): PolicyIssue =>\n createIssue('invalid-shape', path, `${path}: expected ${expectedShape}.`);\n\nconst unknownProperty = (path: string, name: string): PolicyIssue =>\n createIssue('unknown-property', path, `${path}: unknown property \"${name}\".`);\n\nconst unknownSource = (path: string, value: string, kind: string): PolicyIssue =>\n createIssue('unknown-source', path, `${path}: \"${value}\" is not an allowed ${kind} source.`);\n\nconst repeatedSource = (path: string, source: string, kind: string): PolicyIssue =>\n createIssue(\n 'repeated-source',\n path,\n `${path}: ${kind} source \"${source}\" is configured more than once.`\n );\n\nconst invalidAlias = (path: string, name: string): PolicyIssue =>\n createIssue(\n 'invalid-alias',\n path,\n `${path}: alias \"${name}\" must match ^[A-Za-z][A-Za-z0-9_]*$.`\n );\n\nconst findUnknownProperties = (\n value: Readonly>,\n allowed: readonly string[]\n): readonly string[] => {\n return Object.keys(value).filter(key => !allowed.includes(key));\n};\n\nconst collectUnknownProperties = (\n value: Readonly>,\n allowed: readonly string[],\n path: string,\n issues: PolicyIssue[]\n): void => {\n for (const key of findUnknownProperties(value, allowed)) {\n issues.push(unknownProperty(path, key));\n }\n};\n\n/**\n * Output names live in one policy-wide namespace shared by projection fields and ranking\n * siblings, so the checks below run in a fixed order for every configured output name.\n */\nconst findOutputCollision = (\n outputName: string,\n source: string,\n path: string,\n protectedNames: readonly string[],\n usedOutputNames: ReadonlySet\n): PolicyIssue | null => {\n if (protectedNames.includes(outputName)) {\n const owner = PROTECTED_RANKING_FIELDS.includes(outputName) ? 'ranking' : 'record';\n return createIssue(\n 'protected-collision',\n path,\n `${path}: output name \"${outputName}\" is a protected ${owner} field.`\n );\n }\n if (outputName === RANKING_CONTAINER) {\n return createIssue(\n 'container-collision',\n path,\n `${path}: output name \"customRanking\" is the package-owned ranking container.`\n );\n }\n if (ALGOLIA_RESERVED_NAMES.includes(outputName)) {\n return createIssue(\n 'reserved-collision',\n path,\n `${path}: output name \"${outputName}\" is reserved by Algolia.`\n );\n }\n if (outputName !== source && isProjectionSource(outputName)) {\n return createIssue(\n 'canonical-collision',\n path,\n `${path}: output name \"${outputName}\" impersonates a canonical allowlist field.`\n );\n }\n if (usedOutputNames.has(outputName)) {\n return createIssue(\n 'repeated-output',\n path,\n `${path}: output name \"${outputName}\" is produced more than once.`\n );\n }\n\n return null;\n};\n\nconst readAliasedFieldShape = (\n entry: unknown,\n path: string,\n expectedShape: string\n): Resolution => {\n if (!isPlainObject(entry)) {\n return {ok: false, issue: invalidShape(path, expectedShape)};\n }\n\n const [unknownKey] = findUnknownProperties(entry, PROJECTION_FIELD_PROPERTIES);\n if (unknownKey !== undefined) {\n return {ok: false, issue: unknownProperty(path, unknownKey)};\n }\n if (typeof entry.source !== 'string' || typeof entry.as !== 'string') {\n return {ok: false, issue: invalidShape(path, 'a {source, as} object of strings')};\n }\n\n return {ok: true, value: {source: entry.source, alias: entry.as}};\n};\n\nconst readProjectionFieldShape = (\n entry: unknown,\n path: string\n): Resolution => {\n if (typeof entry === 'string') {\n return {ok: true, value: {source: entry, alias: null}};\n }\n\n return readAliasedFieldShape(entry, path, 'a projection source name or a {source, as} object');\n};\n\nconst resolveProjectionField = (\n entry: unknown,\n path: string,\n usedSources: Set,\n usedOutputNames: Set\n): Resolution => {\n const shape = readProjectionFieldShape(entry, path);\n if (!shape.ok) {\n return shape;\n }\n\n const {source, alias} = shape.value;\n if (!isProjectionSource(source)) {\n return {ok: false, issue: unknownSource(path, source, 'projection')};\n }\n if (usedSources.has(source)) {\n return {ok: false, issue: repeatedSource(path, source, 'projection')};\n }\n\n const outputPath = alias === null ? path : `${path}.as`;\n if (alias !== null && !ALIAS_PATTERN.test(alias)) {\n return {ok: false, issue: invalidAlias(outputPath, alias)};\n }\n\n const outputKey = alias ?? source;\n const collision = findOutputCollision(\n outputKey,\n source,\n outputPath,\n PROTECTED_RECORD_FIELDS,\n usedOutputNames\n );\n if (collision !== null) {\n return {ok: false, issue: collision};\n }\n\n usedSources.add(source);\n usedOutputNames.add(outputKey);\n return {ok: true, value: {source, outputKey}};\n};\n\nconst resolveRankingField = (\n entry: unknown,\n path: string,\n usedSources: Set,\n usedOutputNames: Set\n): Resolution => {\n const shape = readAliasedFieldShape(entry, path, 'a {source, as} object');\n if (!shape.ok) {\n return shape;\n }\n\n const {source, alias} = shape.value;\n if (!isRankingSource(source)) {\n return {ok: false, issue: unknownSource(path, source, 'ranking')};\n }\n if (usedSources.has(source)) {\n return {ok: false, issue: repeatedSource(path, source, 'ranking')};\n }\n\n const outputPath = `${path}.as`;\n if (!ALIAS_PATTERN.test(alias)) {\n return {ok: false, issue: invalidAlias(outputPath, alias)};\n }\n\n const collision = findOutputCollision(\n alias,\n source,\n outputPath,\n PROTECTED_RANKING_OUTPUT_NAMES,\n usedOutputNames\n );\n if (collision !== null) {\n return {ok: false, issue: collision};\n }\n\n usedSources.add(source);\n usedOutputNames.add(alias);\n return {ok: true, value: {source, outputKey: alias}};\n};\n\n/**\n * Resolves one configured list. Sources are unique per list, while output names are checked\n * against the policy-wide namespace the caller owns.\n */\nconst resolveEntries = (\n entries: readonly unknown[],\n listPath: string,\n usedOutputNames: Set,\n issues: PolicyIssue[],\n resolveEntry: (\n entry: unknown,\n path: string,\n usedSources: Set,\n usedOutputNames: Set\n ) => Resolution\n): readonly Field[] => {\n const usedSources = new Set();\n const fields: Field[] = [];\n for (const [index, entry] of entries.entries()) {\n const resolved = resolveEntry(entry, `${listPath}[${index}]`, usedSources, usedOutputNames);\n if (!resolved.ok) {\n issues.push(resolved.issue);\n continue;\n }\n fields.push(resolved.value);\n }\n\n return fields;\n};\n\nconst resolveFields = (\n value: unknown,\n usedOutputNames: Set,\n issues: PolicyIssue[]\n): readonly ResolvedProjectionField[] => {\n if (!Array.isArray(value)) {\n issues.push(invalidShape('contentProjection.fields', 'an array of projection fields'));\n return [];\n }\n\n return resolveEntries(\n value,\n 'contentProjection.fields',\n usedOutputNames,\n issues,\n resolveProjectionField\n );\n};\n\nconst resolveRankingFields = (\n value: unknown,\n usedOutputNames: Set,\n issues: PolicyIssue[]\n): readonly ResolvedRankingField[] => {\n if (value === undefined) {\n return [];\n }\n if (!Array.isArray(value)) {\n issues.push(invalidShape('contentProjection.customRanking', 'an array of ranking fields'));\n return [];\n }\n\n return resolveEntries(\n value,\n 'contentProjection.customRanking',\n usedOutputNames,\n issues,\n resolveRankingField\n );\n};\n\nconst createDefaultProjection = (): ResolvedProjection => ({\n fields: DEFAULT_PROJECTION_SOURCES.map(source => ({source, outputKey: source})),\n rankingFields: []\n});\n\nconst resolveContentProjection = (value: unknown, issues: PolicyIssue[]): ResolvedProjection => {\n if (value === undefined) {\n return createDefaultProjection();\n }\n if (!isPlainObject(value)) {\n issues.push(invalidShape('contentProjection', 'an object'));\n return {fields: [], rankingFields: []};\n }\n\n collectUnknownProperties(value, CONTENT_PROJECTION_PROPERTIES, 'contentProjection', issues);\n const usedOutputNames = new Set();\n\n return {\n fields: resolveFields(value.fields, usedOutputNames, issues),\n rankingFields: resolveRankingFields(value.customRanking, usedOutputNames, issues)\n };\n};\n\nconst resolveIgnoreSlugs = (value: unknown, issues: PolicyIssue[]): readonly string[] => {\n if (value === undefined) {\n return [];\n }\n if (!Array.isArray(value)) {\n issues.push(invalidShape('ignoreSlugs', 'an array of strings'));\n return [];\n }\n\n const slugs: string[] = [];\n for (const [index, entry] of value.entries()) {\n if (typeof entry !== 'string') {\n issues.push(invalidShape(`ignoreSlugs[${index}]`, 'a string'));\n continue;\n }\n slugs.push(entry);\n }\n\n return slugs;\n};\n\n/**\n * Validates caller options before any Ghost content is inspected and returns the policy the\n * projection, ranking, and record stages read. Every policy issue is collected in declaration\n * order rather than stopping at the first one.\n */\nexport const resolvePolicy = (options: unknown): PolicyResolution => {\n if (options === undefined) {\n const projection = createDefaultProjection();\n return {ok: true, policy: {ignoreSlugs: [], ...projection}};\n }\n if (!isPlainObject(options)) {\n return {ok: false, issues: [invalidShape('options', 'an object')]};\n }\n\n const issues: PolicyIssue[] = [];\n collectUnknownProperties(options, OPTIONS_PROPERTIES, 'options', issues);\n const ignoreSlugs = resolveIgnoreSlugs(options.ignoreSlugs, issues);\n const projection = resolveContentProjection(options.contentProjection, issues);\n\n if (issues.length > 0) {\n return {ok: false, issues};\n }\n\n return {ok: true, policy: {ignoreSlugs, ...projection}};\n};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/projection.d.mts b/packages/algolia-fragmenter/lib/projection.d.mts new file mode 100644 index 00000000..17ce9515 --- /dev/null +++ b/packages/algolia-fragmenter/lib/projection.d.mts @@ -0,0 +1,27 @@ +import type { GhostContentIssue } from './errors.mjs'; +import { type ResolvedPolicy } from './policy.mjs'; +export type GhostContent = Readonly>; +export type PreparedContent = Readonly<{ + index: number; + id: string; + slug: string; + url: string; + title: string; + html: string; + projected: Readonly>; + rankingSiblings: Readonly>; +}>; +export type ContentPreparation = Readonly<{ + ok: true; + contents: readonly PreparedContent[]; +}> | Readonly<{ + ok: false; + issues: readonly GhostContentIssue[]; +}>; +/** + * Validates the whole batch in input order and prepares the content that survives ignored-slug + * exclusion. Validation and projection share one reader, so a prepared item always projects the + * values that were validated. + */ +export declare const prepareGhostContent: (ghostContent: unknown, policy: ResolvedPolicy) => ContentPreparation; +//# sourceMappingURL=projection.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/projection.d.mts.map b/packages/algolia-fragmenter/lib/projection.d.mts.map new file mode 100644 index 00000000..d0f1405a --- /dev/null +++ b/packages/algolia-fragmenter/lib/projection.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"projection.d.mts","sourceRoot":"","sources":["../src/projection.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAoB,iBAAiB,EAA0B,MAAM,cAAc,CAAC;AAChG,OAAO,EAA+C,KAAK,cAAc,EAAC,MAAM,cAAc,CAAC;AAE/F,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACtD,CAAC,CAAC;AAEH,MAAM,MAAM,kBAAkB,GACxB,QAAQ,CAAC;IAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAA;CAAC,CAAC,GAC1D,QAAQ,CAAC;IAAC,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,SAAS,iBAAiB,EAAE,CAAA;CAAC,CAAC,CAAC;AAgUlE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,iBACd,OAAO,UACb,cAAc,KACvB,kBAuBF,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/projection.mjs b/packages/algolia-fragmenter/lib/projection.mjs new file mode 100644 index 00000000..469574b8 --- /dev/null +++ b/packages/algolia-fragmenter/lib/projection.mjs @@ -0,0 +1,225 @@ +import { isPlainObject } from './policy.mjs'; +/** + * The single projection-source descriptor table. Content validation and value projection read + * it through the same reader, so a Ghost source can never be validated as one type and + * projected as another. + */ +const PROJECTION_SOURCES = { + image: { ghostKey: 'feature_image', kind: 'string' }, + tags: { ghostKey: 'tags', kind: 'relations' }, + authors: { ghostKey: 'authors', kind: 'relations' }, + excerpt: { ghostKey: 'excerpt', kind: 'string' }, + custom_excerpt: { ghostKey: 'custom_excerpt', kind: 'string' }, + feature_image_alt: { ghostKey: 'feature_image_alt', kind: 'string' }, + feature_image_caption: { ghostKey: 'feature_image_caption', kind: 'string' }, + canonical_url: { ghostKey: 'canonical_url', kind: 'string' }, + featured: { ghostKey: 'featured', kind: 'boolean' }, + visibility: { ghostKey: 'visibility', kind: 'string' }, + created_at: { ghostKey: 'created_at', kind: 'string' }, + updated_at: { ghostKey: 'updated_at', kind: 'string' }, + published_at: { ghostKey: 'published_at', kind: 'string' }, + reading_time: { ghostKey: 'reading_time', kind: 'number' } +}; +const describeReceived = (value) => { + if (value === null) { + return 'null'; + } + if (Array.isArray(value)) { + return 'array'; + } + return typeof value; +}; +const createContentIssue = (context, reason, path, expected, message) => ({ + kind: 'content', + reason, + path, + index: context.index, + contentId: context.contentId, + expected, + message +}); +const missingIssue = (context, path) => createContentIssue(context, 'missing', path, 'string', `${path}: required Ghost field is missing.`); +const wrongTypeIssue = (context, path, expected, received) => createContentIssue(context, 'wrong-type', path, expected, `${path}: expected ${expected} but received ${describeReceived(received)}.`); +const emptyIdentityIssue = (context, path) => createContentIssue(context, 'wrong-type', path, 'string', `${path}: expected a non-empty string but received an empty string.`); +const createBatchShapeIssue = () => ({ + kind: 'content', + reason: 'invalid-shape', + path: 'ghostContent', + index: null, + contentId: null, + expected: 'array', + message: 'ghostContent: expected array.' +}); +const invalidShapeIssue = (context, path, expected) => createContentIssue(context, 'invalid-shape', path, expected, `${path}: expected ${expected}.`); +const isFailedRead = (read) => !read.ok; +const readContentString = (raw, path, context) => { + if (raw === undefined || raw === null) { + return { ok: false, issue: missingIssue(context, path) }; + } + if (typeof raw !== 'string') { + return { ok: false, issue: wrongTypeIssue(context, path, 'string', raw) }; + } + return { ok: true, value: raw }; +}; +const readIdentityString = (raw, path, context) => { + const read = readContentString(raw, path, context); + if (read.ok && read.value === '') { + return { ok: false, issue: emptyIdentityIssue(context, path) }; + } + return read; +}; +const readScalar = (kind, raw, path, context) => { + if (raw === undefined || raw === null) { + return { ok: true, value: null }; + } + if (typeof raw !== kind) { + return { ok: false, issue: wrongTypeIssue(context, path, kind, raw) }; + } + return { ok: true, value: raw }; +}; +const readRelations = (raw, path, context) => { + if (raw === undefined || raw === null) { + return { ok: true, value: [] }; + } + if (!Array.isArray(raw)) { + return { ok: false, issue: wrongTypeIssue(context, path, 'array', raw) }; + } + const relations = []; + for (const [index, element] of raw.entries()) { + const elementPath = `${path}[${index}]`; + if (!isPlainObject(element)) { + return { ok: false, issue: wrongTypeIssue(context, elementPath, 'object', element) }; + } + const { name, slug } = element; + if (typeof name !== 'string') { + return { + ok: false, + issue: wrongTypeIssue(context, `${elementPath}.name`, 'string', name) + }; + } + if (typeof slug !== 'string') { + return { + ok: false, + issue: wrongTypeIssue(context, `${elementPath}.slug`, 'string', slug) + }; + } + relations.push({ name, slug }); + } + return { ok: true, value: relations }; +}; +const readSource = (item, source, context) => { + const descriptor = PROJECTION_SOURCES[source]; + const path = `${context.path}.${descriptor.ghostKey}`; + const raw = item[descriptor.ghostKey]; + if (descriptor.kind === 'relations') { + return readRelations(raw, path, context); + } + return readScalar(descriptor.kind, raw, path, context); +}; +/** + * Every enabled projection source, then every ranking source that no projection field already + * reads, so a source feeding both a projection field and a ranking sibling is read once. + */ +const collectEnabledSources = (policy) => { + const sources = []; + for (const field of [...policy.fields, ...policy.rankingFields]) { + if (!sources.includes(field.source)) { + sources.push(field.source); + } + } + return sources; +}; +const readEnabledSources = (item, policy, context) => { + const values = new Map(); + const issues = []; + for (const source of collectEnabledSources(policy)) { + const read = readSource(item, source, context); + if (isFailedRead(read)) { + issues.push(read.issue); + continue; + } + values.set(source, read.value); + } + return { values, issues }; +}; +const projectValues = (fields, values) => { + const projected = {}; + for (const field of fields) { + projected[field.outputKey] = values.get(field.source); + } + return projected; +}; +const readContentId = (item) => { + return typeof item.id === 'string' && item.id !== '' ? item.id : null; +}; +const prepareItem = (value, index, policy) => { + const path = `ghostContent[${index}]`; + if (!isPlainObject(value)) { + const context = { index, contentId: null, path }; + return { kind: 'issues', issues: [invalidShapeIssue(context, path, 'object')] }; + } + const context = { index, contentId: readContentId(value), path }; + const slug = readIdentityString(value.slug, `${path}.slug`, context); + if (isFailedRead(slug)) { + return { kind: 'issues', issues: [slug.issue] }; + } + if (policy.ignoreSlugs.includes(slug.value)) { + return { kind: 'ignored' }; + } + const id = readIdentityString(value.id, `${path}.id`, context); + const url = readContentString(value.url, `${path}.url`, context); + const title = readContentString(value.title, `${path}.title`, context); + const html = readContentString(value.html, `${path}.html`, context); + const enabled = readEnabledSources(value, policy, context); + // The explicit chain is what narrows `id`, `url`, `title`, and `html` to successful reads + // for the return below; the `filter` only collects the issues in canonical order. Collapsing + // the two into one expression loses the narrowing. + if (isFailedRead(id) || + isFailedRead(url) || + isFailedRead(title) || + isFailedRead(html) || + enabled.issues.length > 0) { + const required = [id, url, title, html].filter(isFailedRead).map(read => read.issue); + return { kind: 'issues', issues: [...required, ...enabled.issues] }; + } + return { + kind: 'content', + content: { + index, + id: id.value, + slug: slug.value, + url: url.value, + title: title.value, + html: html.value, + projected: projectValues(policy.fields, enabled.values), + rankingSiblings: projectValues(policy.rankingFields, enabled.values) + } + }; +}; +/** + * Validates the whole batch in input order and prepares the content that survives ignored-slug + * exclusion. Validation and projection share one reader, so a prepared item always projects the + * values that were validated. + */ +export const prepareGhostContent = (ghostContent, policy) => { + if (!Array.isArray(ghostContent)) { + return { ok: false, issues: [createBatchShapeIssue()] }; + } + const contents = []; + const issues = []; + for (const [index, item] of ghostContent.entries()) { + const prepared = prepareItem(item, index, policy); + if (prepared.kind === 'issues') { + issues.push(...prepared.issues); + continue; + } + if (prepared.kind === 'content') { + contents.push(prepared.content); + } + } + if (issues.length > 0) { + return { ok: false, issues }; + } + return { ok: true, contents }; +}; +//# sourceMappingURL=projection.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/projection.mjs.map b/packages/algolia-fragmenter/lib/projection.mjs.map new file mode 100644 index 00000000..8a9b8838 --- /dev/null +++ b/packages/algolia-fragmenter/lib/projection.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"projection.mjs","sourceRoot":"","sources":["../src/projection.mts"],"names":[],"mappings":"AACA,OAAO,EAAC,aAAa,EAAqD,MAAM,cAAc,CAAC;AAkC/F;;;;GAIG;AACH,MAAM,kBAAkB,GAAG;IACvB,KAAK,EAAE,EAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAC;IAClD,IAAI,EAAE,EAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAC;IAC3C,OAAO,EAAE,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAC;IACjD,OAAO,EAAE,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC9C,cAAc,EAAE,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC5D,iBAAiB,EAAE,EAAC,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAClE,qBAAqB,EAAE,EAAC,QAAQ,EAAE,uBAAuB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC1E,aAAa,EAAE,EAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC1D,QAAQ,EAAE,EAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAC;IACjD,UAAU,EAAE,EAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAC;IACpD,UAAU,EAAE,EAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAC;IACpD,UAAU,EAAE,EAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAC;IACpD,YAAY,EAAE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAC;IACxD,YAAY,EAAE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAC;CAC+B,CAAC;AAE5F,MAAM,gBAAgB,GAAG,CAAC,KAAc,EAAU,EAAE;IAChD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,OAAO,OAAO,KAAK,CAAC;AACxB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACvB,OAAqB,EACrB,MAA+B,EAC/B,IAAY,EACZ,QAA2B,EAC3B,OAAe,EACE,EAAE,CAAC,CAAC;IACrB,IAAI,EAAE,SAAS;IACf,MAAM;IACN,IAAI;IACJ,KAAK,EAAE,OAAO,CAAC,KAAK;IACpB,SAAS,EAAE,OAAO,CAAC,SAAS;IAC5B,QAAQ;IACR,OAAO;CACV,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAE,IAAY,EAAqB,EAAE,CAC5E,kBAAkB,CACd,OAAO,EACP,SAAS,EACT,IAAI,EACJ,QAAQ,EACR,GAAG,IAAI,oCAAoC,CAC9C,CAAC;AAEN,MAAM,cAAc,GAAG,CACnB,OAAqB,EACrB,IAAY,EACZ,QAA2B,EAC3B,QAAiB,EACA,EAAE,CACnB,kBAAkB,CACd,OAAO,EACP,YAAY,EACZ,IAAI,EACJ,QAAQ,EACR,GAAG,IAAI,cAAc,QAAQ,iBAAiB,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAC9E,CAAC;AAEN,MAAM,kBAAkB,GAAG,CAAC,OAAqB,EAAE,IAAY,EAAqB,EAAE,CAClF,kBAAkB,CACd,OAAO,EACP,YAAY,EACZ,IAAI,EACJ,QAAQ,EACR,GAAG,IAAI,6DAA6D,CACvE,CAAC;AAEN,MAAM,qBAAqB,GAAG,GAAsB,EAAE,CAAC,CAAC;IACpD,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,eAAe;IACvB,IAAI,EAAE,cAAc;IACpB,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,OAAO;IACjB,OAAO,EAAE,+BAA+B;CAC3C,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,CACtB,OAAqB,EACrB,IAAY,EACZ,QAA2B,EACV,EAAE,CACnB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,cAAc,QAAQ,GAAG,CAAC,CAAC;AAEnG,MAAM,YAAY,GAAG,CAAS,IAAsB,EAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;AAEtF,MAAM,iBAAiB,GAAG,CACtB,GAAY,EACZ,IAAY,EACZ,OAAqB,EACJ,EAAE;IACnB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACpC,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,EAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAC,CAAC;IAC5E,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAC,CAAC;AAClC,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACvB,GAAY,EACZ,IAAY,EACZ,OAAqB,EACJ,EAAE;IACnB,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;QAC/B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,EAAC,CAAC;IACjE,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CACf,IAA+C,EAC/C,GAAY,EACZ,IAAY,EACZ,OAAqB,EACH,EAAE;IACpB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACpC,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC;IACnC,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,EAAC,CAAC;IACxE,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAC,CAAC;AAClC,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAClB,GAAY,EACZ,IAAY,EACZ,OAAqB,EACuC,EAAE;IAC9D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACpC,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAC,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,EAAC,CAAC;IAC3E,CAAC;IAED,MAAM,SAAS,GAAwC,EAAE,CAAC;IAC1D,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,WAAW,GAAG,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC;QACxC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAC,CAAC;QACvF,CAAC;QAED,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,OAAO,CAAC;QAC7B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO;gBACH,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,GAAG,WAAW,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC;aACxE,CAAC;QACN,CAAC;QACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO;gBACH,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,cAAc,CAAC,OAAO,EAAE,GAAG,WAAW,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC;aACxE,CAAC;QACN,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAC,CAAC;AACxC,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CACf,IAAuC,EACvC,MAAgC,EAChC,OAAqB,EACH,EAAE;IACpB,MAAM,UAAU,GAA+B,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,UAAU,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAC3D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,MAAsB,EAAuC,EAAE;IAC1F,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACvB,IAAuC,EACvC,MAAsB,EACtB,OAAqB,EAItB,EAAE;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqC,CAAC;IAC5D,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,KAAK,MAAM,MAAM,IAAI,qBAAqB,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,SAAS;QACb,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,EAAC,MAAM,EAAE,MAAM,EAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAClB,MAAkF,EAClF,MAAsD,EACrB,EAAE;IACnC,MAAM,SAAS,GAA4B,EAAE,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAuC,EAAiB,EAAE;IAC7E,OAAO,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1E,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,MAAsB,EAAmB,EAAE;IAC3F,MAAM,IAAI,GAAG,gBAAgB,KAAK,GAAG,CAAC;IACtC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,OAAO,GAAiB,EAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAC,CAAC;QAC7D,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAC,CAAC;IAClF,CAAC;IAED,MAAM,OAAO,GAAiB,EAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,IAAI,EAAC,CAAC;IAE7E,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,OAAO,CAAC,CAAC;IACrE,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAC,CAAC;IAClD,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;IAC7B,CAAC;IAED,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,IAAI,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,MAAM,EAAE,OAAO,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,IAAI,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAE3D,0FAA0F;IAC1F,6FAA6F;IAC7F,mDAAmD;IACnD,IACI,YAAY,CAAC,EAAE,CAAC;QAChB,YAAY,CAAC,GAAG,CAAC;QACjB,YAAY,CAAC,KAAK,CAAC;QACnB,YAAY,CAAC,IAAI,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAC3B,CAAC;QACC,MAAM,QAAQ,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrF,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,EAAC,CAAC;IACtE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,SAAS;QACf,OAAO,EAAE;YACL,KAAK;YACL,EAAE,EAAE,EAAE,CAAC,KAAK;YACZ,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,GAAG,EAAE,GAAG,CAAC,KAAK;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC;YACvD,eAAe,EAAE,aAAa,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC;SACvE;KACJ,CAAC;AACN,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAC/B,YAAqB,EACrB,MAAsB,EACJ,EAAE;IACpB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,qBAAqB,EAAE,CAAC,EAAC,CAAC;IAC1D,CAAC;IAED,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAClD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChC,SAAS;QACb,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,EAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAC,CAAC;IAC/B,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAC,CAAC;AAChC,CAAC,CAAC","sourcesContent":["import type {ExpectedValueType, GhostContentIssue, GhostContentIssueReason} from './errors.mjs';\nimport {isPlainObject, type OptionalProjectionSource, type ResolvedPolicy} from './policy.mjs';\n\nexport type GhostContent = Readonly>;\n\nexport type PreparedContent = Readonly<{\n index: number;\n id: string;\n slug: string;\n url: string;\n title: string;\n html: string;\n projected: Readonly>;\n rankingSiblings: Readonly>;\n}>;\n\nexport type ContentPreparation =\n | Readonly<{ok: true; contents: readonly PreparedContent[]}>\n | Readonly<{ok: false; issues: readonly GhostContentIssue[]}>;\n\ntype ProjectionValueKind = 'string' | 'boolean' | 'number' | 'relations';\n\ntype ProjectionSourceDescriptor = Readonly<{ghostKey: string; kind: ProjectionValueKind}>;\n\ntype IssueContext = Readonly<{index: number; contentId: string | null; path: string}>;\n\ntype FailedRead = Readonly<{ok: false; issue: GhostContentIssue}>;\n\ntype ValueRead = Readonly<{ok: true; value: Value}> | FailedRead;\n\ntype ItemPreparation =\n | Readonly<{kind: 'content'; content: PreparedContent}>\n | Readonly<{kind: 'ignored'}>\n | Readonly<{kind: 'issues'; issues: readonly GhostContentIssue[]}>;\n\n/**\n * The single projection-source descriptor table. Content validation and value projection read\n * it through the same reader, so a Ghost source can never be validated as one type and\n * projected as another.\n */\nconst PROJECTION_SOURCES = {\n image: {ghostKey: 'feature_image', kind: 'string'},\n tags: {ghostKey: 'tags', kind: 'relations'},\n authors: {ghostKey: 'authors', kind: 'relations'},\n excerpt: {ghostKey: 'excerpt', kind: 'string'},\n custom_excerpt: {ghostKey: 'custom_excerpt', kind: 'string'},\n feature_image_alt: {ghostKey: 'feature_image_alt', kind: 'string'},\n feature_image_caption: {ghostKey: 'feature_image_caption', kind: 'string'},\n canonical_url: {ghostKey: 'canonical_url', kind: 'string'},\n featured: {ghostKey: 'featured', kind: 'boolean'},\n visibility: {ghostKey: 'visibility', kind: 'string'},\n created_at: {ghostKey: 'created_at', kind: 'string'},\n updated_at: {ghostKey: 'updated_at', kind: 'string'},\n published_at: {ghostKey: 'published_at', kind: 'string'},\n reading_time: {ghostKey: 'reading_time', kind: 'number'}\n} as const satisfies Readonly>;\n\nconst describeReceived = (value: unknown): string => {\n if (value === null) {\n return 'null';\n }\n if (Array.isArray(value)) {\n return 'array';\n }\n\n return typeof value;\n};\n\nconst createContentIssue = (\n context: IssueContext,\n reason: GhostContentIssueReason,\n path: string,\n expected: ExpectedValueType,\n message: string\n): GhostContentIssue => ({\n kind: 'content',\n reason,\n path,\n index: context.index,\n contentId: context.contentId,\n expected,\n message\n});\n\nconst missingIssue = (context: IssueContext, path: string): GhostContentIssue =>\n createContentIssue(\n context,\n 'missing',\n path,\n 'string',\n `${path}: required Ghost field is missing.`\n );\n\nconst wrongTypeIssue = (\n context: IssueContext,\n path: string,\n expected: ExpectedValueType,\n received: unknown\n): GhostContentIssue =>\n createContentIssue(\n context,\n 'wrong-type',\n path,\n expected,\n `${path}: expected ${expected} but received ${describeReceived(received)}.`\n );\n\nconst emptyIdentityIssue = (context: IssueContext, path: string): GhostContentIssue =>\n createContentIssue(\n context,\n 'wrong-type',\n path,\n 'string',\n `${path}: expected a non-empty string but received an empty string.`\n );\n\nconst createBatchShapeIssue = (): GhostContentIssue => ({\n kind: 'content',\n reason: 'invalid-shape',\n path: 'ghostContent',\n index: null,\n contentId: null,\n expected: 'array',\n message: 'ghostContent: expected array.'\n});\n\nconst invalidShapeIssue = (\n context: IssueContext,\n path: string,\n expected: ExpectedValueType\n): GhostContentIssue =>\n createContentIssue(context, 'invalid-shape', path, expected, `${path}: expected ${expected}.`);\n\nconst isFailedRead = (read: ValueRead): read is FailedRead => !read.ok;\n\nconst readContentString = (\n raw: unknown,\n path: string,\n context: IssueContext\n): ValueRead => {\n if (raw === undefined || raw === null) {\n return {ok: false, issue: missingIssue(context, path)};\n }\n if (typeof raw !== 'string') {\n return {ok: false, issue: wrongTypeIssue(context, path, 'string', raw)};\n }\n\n return {ok: true, value: raw};\n};\n\nconst readIdentityString = (\n raw: unknown,\n path: string,\n context: IssueContext\n): ValueRead => {\n const read = readContentString(raw, path, context);\n if (read.ok && read.value === '') {\n return {ok: false, issue: emptyIdentityIssue(context, path)};\n }\n\n return read;\n};\n\nconst readScalar = (\n kind: Exclude,\n raw: unknown,\n path: string,\n context: IssueContext\n): ValueRead => {\n if (raw === undefined || raw === null) {\n return {ok: true, value: null};\n }\n if (typeof raw !== kind) {\n return {ok: false, issue: wrongTypeIssue(context, path, kind, raw)};\n }\n\n return {ok: true, value: raw};\n};\n\nconst readRelations = (\n raw: unknown,\n path: string,\n context: IssueContext\n): ValueRead[]> => {\n if (raw === undefined || raw === null) {\n return {ok: true, value: []};\n }\n if (!Array.isArray(raw)) {\n return {ok: false, issue: wrongTypeIssue(context, path, 'array', raw)};\n }\n\n const relations: Array<{name: string; slug: string}> = [];\n for (const [index, element] of raw.entries()) {\n const elementPath = `${path}[${index}]`;\n if (!isPlainObject(element)) {\n return {ok: false, issue: wrongTypeIssue(context, elementPath, 'object', element)};\n }\n\n const {name, slug} = element;\n if (typeof name !== 'string') {\n return {\n ok: false,\n issue: wrongTypeIssue(context, `${elementPath}.name`, 'string', name)\n };\n }\n if (typeof slug !== 'string') {\n return {\n ok: false,\n issue: wrongTypeIssue(context, `${elementPath}.slug`, 'string', slug)\n };\n }\n\n relations.push({name, slug});\n }\n\n return {ok: true, value: relations};\n};\n\nconst readSource = (\n item: Readonly>,\n source: OptionalProjectionSource,\n context: IssueContext\n): ValueRead => {\n const descriptor: ProjectionSourceDescriptor = PROJECTION_SOURCES[source];\n const path = `${context.path}.${descriptor.ghostKey}`;\n const raw = item[descriptor.ghostKey];\n if (descriptor.kind === 'relations') {\n return readRelations(raw, path, context);\n }\n\n return readScalar(descriptor.kind, raw, path, context);\n};\n\n/**\n * Every enabled projection source, then every ranking source that no projection field already\n * reads, so a source feeding both a projection field and a ranking sibling is read once.\n */\nconst collectEnabledSources = (policy: ResolvedPolicy): readonly OptionalProjectionSource[] => {\n const sources: OptionalProjectionSource[] = [];\n for (const field of [...policy.fields, ...policy.rankingFields]) {\n if (!sources.includes(field.source)) {\n sources.push(field.source);\n }\n }\n\n return sources;\n};\n\nconst readEnabledSources = (\n item: Readonly>,\n policy: ResolvedPolicy,\n context: IssueContext\n): Readonly<{\n values: ReadonlyMap;\n issues: readonly GhostContentIssue[];\n}> => {\n const values = new Map();\n const issues: GhostContentIssue[] = [];\n for (const source of collectEnabledSources(policy)) {\n const read = readSource(item, source, context);\n if (isFailedRead(read)) {\n issues.push(read.issue);\n continue;\n }\n values.set(source, read.value);\n }\n\n return {values, issues};\n};\n\nconst projectValues = (\n fields: readonly Readonly<{source: OptionalProjectionSource; outputKey: string}>[],\n values: ReadonlyMap\n): Readonly> => {\n const projected: Record = {};\n for (const field of fields) {\n projected[field.outputKey] = values.get(field.source);\n }\n\n return projected;\n};\n\nconst readContentId = (item: Readonly>): string | null => {\n return typeof item.id === 'string' && item.id !== '' ? item.id : null;\n};\n\nconst prepareItem = (value: unknown, index: number, policy: ResolvedPolicy): ItemPreparation => {\n const path = `ghostContent[${index}]`;\n if (!isPlainObject(value)) {\n const context: IssueContext = {index, contentId: null, path};\n return {kind: 'issues', issues: [invalidShapeIssue(context, path, 'object')]};\n }\n\n const context: IssueContext = {index, contentId: readContentId(value), path};\n\n const slug = readIdentityString(value.slug, `${path}.slug`, context);\n if (isFailedRead(slug)) {\n return {kind: 'issues', issues: [slug.issue]};\n }\n if (policy.ignoreSlugs.includes(slug.value)) {\n return {kind: 'ignored'};\n }\n\n const id = readIdentityString(value.id, `${path}.id`, context);\n const url = readContentString(value.url, `${path}.url`, context);\n const title = readContentString(value.title, `${path}.title`, context);\n const html = readContentString(value.html, `${path}.html`, context);\n const enabled = readEnabledSources(value, policy, context);\n\n // The explicit chain is what narrows `id`, `url`, `title`, and `html` to successful reads\n // for the return below; the `filter` only collects the issues in canonical order. Collapsing\n // the two into one expression loses the narrowing.\n if (\n isFailedRead(id) ||\n isFailedRead(url) ||\n isFailedRead(title) ||\n isFailedRead(html) ||\n enabled.issues.length > 0\n ) {\n const required = [id, url, title, html].filter(isFailedRead).map(read => read.issue);\n return {kind: 'issues', issues: [...required, ...enabled.issues]};\n }\n\n return {\n kind: 'content',\n content: {\n index,\n id: id.value,\n slug: slug.value,\n url: url.value,\n title: title.value,\n html: html.value,\n projected: projectValues(policy.fields, enabled.values),\n rankingSiblings: projectValues(policy.rankingFields, enabled.values)\n }\n };\n};\n\n/**\n * Validates the whole batch in input order and prepares the content that survives ignored-slug\n * exclusion. Validation and projection share one reader, so a prepared item always projects the\n * values that were validated.\n */\nexport const prepareGhostContent = (\n ghostContent: unknown,\n policy: ResolvedPolicy\n): ContentPreparation => {\n if (!Array.isArray(ghostContent)) {\n return {ok: false, issues: [createBatchShapeIssue()]};\n }\n\n const contents: PreparedContent[] = [];\n const issues: GhostContentIssue[] = [];\n for (const [index, item] of ghostContent.entries()) {\n const prepared = prepareItem(item, index, policy);\n if (prepared.kind === 'issues') {\n issues.push(...prepared.issues);\n continue;\n }\n if (prepared.kind === 'content') {\n contents.push(prepared.content);\n }\n }\n\n if (issues.length > 0) {\n return {ok: false, issues};\n }\n\n return {ok: true, contents};\n};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/records.d.mts b/packages/algolia-fragmenter/lib/records.d.mts new file mode 100644 index 00000000..c4cb8bf3 --- /dev/null +++ b/packages/algolia-fragmenter/lib/records.d.mts @@ -0,0 +1,15 @@ +import type { RecordSizeIssue } from './errors.mjs'; +import { type FragmentGroup } from './grouping.mjs'; +import type { PreparedContent } from './projection.mjs'; +export type AlgoliaRecord = Record; +export type ContentRecords = Readonly<{ + records: readonly AlgoliaRecord[]; + issues: readonly RecordSizeIssue[]; +}>; +export declare const measureRecordBytes: (record: AlgoliaRecord) => number; +/** + * Builds every Algolia record for one prepared Ghost content item in anchor-group order, then + * continuation order. Content without extraction fragments emits the single fallback record. + */ +export declare const createContentRecords: (content: PreparedContent, groups: readonly FragmentGroup[]) => ContentRecords; +//# sourceMappingURL=records.d.mts.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/records.d.mts.map b/packages/algolia-fragmenter/lib/records.d.mts.map new file mode 100644 index 00000000..aba9f8c6 --- /dev/null +++ b/packages/algolia-fragmenter/lib/records.d.mts.map @@ -0,0 +1 @@ +{"version":3,"file":"records.d.mts","sourceRoot":"","sources":["../src/records.mts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,cAAc,CAAC;AAClD,OAAO,EAAkB,KAAK,aAAa,EAAyB,MAAM,gBAAgB,CAAC;AAC3F,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEtD,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEpD,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC;IAClC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAClC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;CACtC,CAAC,CAAC;AAgBH,eAAO,MAAM,kBAAkB,WAAY,aAAa,KAAG,MAE1D,CAAC;AAmKF;;;GAGG;AACH,eAAO,MAAM,oBAAoB,YACpB,eAAe,UAChB,SAAS,aAAa,EAAE,KACjC,cAcF,CAAC"} \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/records.mjs b/packages/algolia-fragmenter/lib/records.mjs new file mode 100644 index 00000000..c081458d --- /dev/null +++ b/packages/algolia-fragmenter/lib/records.mjs @@ -0,0 +1,133 @@ +import { Buffer } from 'node:buffer'; +import { mergeRecordHtml } from './grouping.mjs'; +const MAX_RECORD_BYTES = 9_999; +const FALLBACK_POSITION = 0; +const FALLBACK_HEADING_RANK = 100; +export const measureRecordBytes = (record) => { + return Buffer.byteLength(JSON.stringify(record), 'utf8'); +}; +const assembleRecord = (content, parts) => ({ + objectID: parts.objectID, + slug: content.slug, + url: parts.url, + html: parts.html, + title: content.title, + headings: [...parts.headings], + anchor: parts.anchor, + ...content.projected, + customRanking: { + position: parts.position, + heading: parts.heading, + ...content.rankingSiblings + } +}); +const createObjectID = (content, groupIndex, continuationIndex) => { + const groupObjectID = `${content.id}_${groupIndex}`; + return continuationIndex === 0 ? groupObjectID : `${groupObjectID}_${continuationIndex}`; +}; +const buildFragmentRecord = (content, groupIndex, continuationIndex, packedFragments) => { + const [first] = packedFragments; + return assembleRecord(content, { + objectID: createObjectID(content, groupIndex, continuationIndex), + url: first.anchor === null ? content.url : `${content.url}#${first.anchor}`, + html: mergeRecordHtml(packedFragments), + headings: first.headingPath, + anchor: first.anchor, + position: first.position, + heading: first.headingRank + }); +}; +const createSizeIssue = (content, objectID, fragment, bytes) => { + const path = `ghostContent[${content.index}]`; + const excess = bytes - MAX_RECORD_BYTES; + const cause = fragment === null + ? `fallback record needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the required projected metadata.` + : `fragment at source position ${fragment.position} needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the indivisible source element or required projected metadata.`; + return { + kind: 'size', + reason: 'record-too-large', + path, + index: content.index, + contentId: content.id, + objectID, + anchor: fragment === null ? null : fragment.anchor, + position: fragment === null ? null : fragment.position, + bytes, + limit: MAX_RECORD_BYTES, + excess, + message: `${path}: content "${content.id}" ${cause}` + }; +}; +/** + * Greedily packs whole extraction fragments into records that stay within the record byte + * ceiling. A candidate is always measured under the continuation index it would be emitted + * with, so the identifier growth of a continuation is inside the measurement. An indivisible + * fragment yields an issue rather than a truncated record, and consumes no continuation index. + */ +const packAnchorGroup = (content, groupIndex, group) => { + const records = []; + const issues = []; + let packedFragments = null; + let continuationIndex = 0; + for (const fragment of group.fragments) { + const candidate = packedFragments === null ? [fragment] : [...packedFragments, fragment]; + const candidateBytes = measureRecordBytes(buildFragmentRecord(content, groupIndex, continuationIndex, candidate)); + if (candidateBytes <= MAX_RECORD_BYTES) { + packedFragments = candidate; + continue; + } + if (packedFragments === null) { + issues.push(createSizeIssue(content, createObjectID(content, groupIndex, continuationIndex), fragment, candidateBytes)); + continue; + } + records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments)); + continuationIndex += 1; + const single = [fragment]; + const singleBytes = measureRecordBytes(buildFragmentRecord(content, groupIndex, continuationIndex, single)); + if (singleBytes > MAX_RECORD_BYTES) { + issues.push(createSizeIssue(content, createObjectID(content, groupIndex, continuationIndex), fragment, singleBytes)); + packedFragments = null; + continue; + } + packedFragments = single; + } + if (packedFragments !== null) { + records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments)); + } + return { records, issues }; +}; +const createFallbackRecord = (content) => { + const objectID = `${content.id}_0`; + const record = assembleRecord(content, { + objectID, + url: content.url, + html: '', + headings: [], + anchor: null, + position: FALLBACK_POSITION, + heading: FALLBACK_HEADING_RANK + }); + const bytes = measureRecordBytes(record); + if (bytes > MAX_RECORD_BYTES) { + return { records: [], issues: [createSizeIssue(content, objectID, null, bytes)] }; + } + return { records: [record], issues: [] }; +}; +/** + * Builds every Algolia record for one prepared Ghost content item in anchor-group order, then + * continuation order. Content without extraction fragments emits the single fallback record. + */ +export const createContentRecords = (content, groups) => { + if (groups.length === 0) { + return createFallbackRecord(content); + } + const records = []; + const issues = []; + for (const [groupIndex, group] of groups.entries()) { + const packed = packAnchorGroup(content, groupIndex, group); + records.push(...packed.records); + issues.push(...packed.issues); + } + return { records, issues }; +}; +//# sourceMappingURL=records.mjs.map \ No newline at end of file diff --git a/packages/algolia-fragmenter/lib/records.mjs.map b/packages/algolia-fragmenter/lib/records.mjs.map new file mode 100644 index 00000000..3bf599df --- /dev/null +++ b/packages/algolia-fragmenter/lib/records.mjs.map @@ -0,0 +1 @@ +{"version":3,"file":"records.mjs","sourceRoot":"","sources":["../src/records.mts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAKnC,OAAO,EAAC,eAAe,EAA6C,MAAM,gBAAgB,CAAC;AAoB3F,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,qBAAqB,GAAgB,GAAG,CAAC;AAE/C,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,MAAqB,EAAU,EAAE;IAChE,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;AAC7D,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,OAAwB,EAAE,KAAkB,EAAiB,EAAE,CAAC,CAAC;IACrF,QAAQ,EAAE,KAAK,CAAC,QAAQ;IACxB,IAAI,EAAE,OAAO,CAAC,IAAI;IAClB,GAAG,EAAE,KAAK,CAAC,GAAG;IACd,IAAI,EAAE,KAAK,CAAC,IAAI;IAChB,KAAK,EAAE,OAAO,CAAC,KAAK;IACpB,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,GAAG,OAAO,CAAC,SAAS;IACpB,aAAa,EAAE;QACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,GAAG,OAAO,CAAC,eAAe;KAC7B;CACJ,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CACnB,OAAwB,EACxB,UAAkB,EAClB,iBAAyB,EACnB,EAAE;IACR,MAAM,aAAa,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpD,OAAO,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,IAAI,iBAAiB,EAAE,CAAC;AAC7F,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CACxB,OAAwB,EACxB,UAAkB,EAClB,iBAAyB,EACzB,eAAkC,EACrB,EAAE;IACf,MAAM,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC;IAEhC,OAAO,cAAc,CAAC,OAAO,EAAE;QAC3B,QAAQ,EAAE,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,CAAC;QAChE,GAAG,EAAE,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE;QAC3E,IAAI,EAAE,eAAe,CAAC,eAAe,CAAC;QACtC,QAAQ,EAAE,KAAK,CAAC,WAAW;QAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,WAAW;KAC7B,CAAC,CAAC;AACP,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACpB,OAAwB,EACxB,QAAgB,EAChB,QAAmC,EACnC,KAAa,EACE,EAAE;IACjB,MAAM,IAAI,GAAG,gBAAgB,OAAO,CAAC,KAAK,GAAG,CAAC;IAC9C,MAAM,MAAM,GAAG,KAAK,GAAG,gBAAgB,CAAC;IACxC,MAAM,KAAK,GACP,QAAQ,KAAK,IAAI;QACb,CAAC,CAAC,yBAAyB,KAAK,iBAAiB,MAAM,aAAa,gBAAgB,0DAA0D;QAC9I,CAAC,CAAC,+BAA+B,QAAQ,CAAC,QAAQ,UAAU,KAAK,iBAAiB,MAAM,aAAa,gBAAgB,wFAAwF,CAAC;IAEtN,OAAO;QACH,IAAI,EAAE,MAAM;QACZ,MAAM,EAAE,kBAAkB;QAC1B,IAAI;QACJ,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,QAAQ;QACR,MAAM,EAAE,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM;QAClD,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ;QACtD,KAAK;QACL,KAAK,EAAE,gBAAgB;QACvB,MAAM;QACN,OAAO,EAAE,GAAG,IAAI,cAAc,OAAO,CAAC,EAAE,KAAK,KAAK,EAAE;KACvD,CAAC;AACN,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,eAAe,GAAG,CACpB,OAAwB,EACxB,UAAkB,EAClB,KAAoB,EACN,EAAE;IAChB,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,eAAe,GAA6B,IAAI,CAAC;IACrD,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,SAAS,GACX,eAAe,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,eAAe,EAAE,QAAQ,CAAC,CAAC;QAC3E,MAAM,cAAc,GAAG,kBAAkB,CACrC,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,CAAC,CACzE,CAAC;QACF,IAAI,cAAc,IAAI,gBAAgB,EAAE,CAAC;YACrC,eAAe,GAAG,SAAS,CAAC;YAC5B,SAAS;QACb,CAAC;QAED,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CACP,eAAe,CACX,OAAO,EACP,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,CAAC,EACtD,QAAQ,EACR,cAAc,CACjB,CACJ,CAAC;YACF,SAAS;QACb,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;QAC3F,iBAAiB,IAAI,CAAC,CAAC;QAEvB,MAAM,MAAM,GAAsB,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,WAAW,GAAG,kBAAkB,CAClC,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,CACtE,CAAC;QACF,IAAI,WAAW,GAAG,gBAAgB,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CACP,eAAe,CACX,OAAO,EACP,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,CAAC,EACtD,QAAQ,EACR,WAAW,CACd,CACJ,CAAC;YACF,eAAe,GAAG,IAAI,CAAC;YACvB,SAAS;QACb,CAAC;QACD,eAAe,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,EAAC,OAAO,EAAE,MAAM,EAAC,CAAC;AAC7B,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,OAAwB,EAAkB,EAAE;IACtE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC;IACnC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE;QACnC,QAAQ;QACR,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,EAAE;QACZ,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,iBAAiB;QAC3B,OAAO,EAAE,qBAAqB;KACjC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;QAC3B,OAAO,EAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAC,CAAC;IACpF,CAAC;IAED,OAAO,EAAC,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,EAAC,CAAC;AAC3C,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,OAAwB,EACxB,MAAgC,EAClB,EAAE;IAChB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,EAAC,OAAO,EAAE,MAAM,EAAC,CAAC;AAC7B,CAAC,CAAC","sourcesContent":["import {Buffer} from 'node:buffer';\n\nimport type {ExtractionFragment, HeadingRank} from '@tryghost/algolia-html-extractor';\n\nimport type {RecordSizeIssue} from './errors.mjs';\nimport {mergeRecordHtml, type FragmentGroup, type NonEmptyFragments} from './grouping.mjs';\nimport type {PreparedContent} from './projection.mjs';\n\nexport type AlgoliaRecord = Record;\n\nexport type ContentRecords = Readonly<{\n records: readonly AlgoliaRecord[];\n issues: readonly RecordSizeIssue[];\n}>;\n\ntype RecordParts = Readonly<{\n objectID: string;\n url: string;\n html: string;\n headings: readonly string[];\n anchor: string | null;\n position: number;\n heading: HeadingRank;\n}>;\n\nconst MAX_RECORD_BYTES = 9_999;\nconst FALLBACK_POSITION = 0;\nconst FALLBACK_HEADING_RANK: HeadingRank = 100;\n\nexport const measureRecordBytes = (record: AlgoliaRecord): number => {\n return Buffer.byteLength(JSON.stringify(record), 'utf8');\n};\n\nconst assembleRecord = (content: PreparedContent, parts: RecordParts): AlgoliaRecord => ({\n objectID: parts.objectID,\n slug: content.slug,\n url: parts.url,\n html: parts.html,\n title: content.title,\n headings: [...parts.headings],\n anchor: parts.anchor,\n ...content.projected,\n customRanking: {\n position: parts.position,\n heading: parts.heading,\n ...content.rankingSiblings\n }\n});\n\nconst createObjectID = (\n content: PreparedContent,\n groupIndex: number,\n continuationIndex: number\n): string => {\n const groupObjectID = `${content.id}_${groupIndex}`;\n return continuationIndex === 0 ? groupObjectID : `${groupObjectID}_${continuationIndex}`;\n};\n\nconst buildFragmentRecord = (\n content: PreparedContent,\n groupIndex: number,\n continuationIndex: number,\n packedFragments: NonEmptyFragments\n): AlgoliaRecord => {\n const [first] = packedFragments;\n\n return assembleRecord(content, {\n objectID: createObjectID(content, groupIndex, continuationIndex),\n url: first.anchor === null ? content.url : `${content.url}#${first.anchor}`,\n html: mergeRecordHtml(packedFragments),\n headings: first.headingPath,\n anchor: first.anchor,\n position: first.position,\n heading: first.headingRank\n });\n};\n\nconst createSizeIssue = (\n content: PreparedContent,\n objectID: string,\n fragment: ExtractionFragment | null,\n bytes: number\n): RecordSizeIssue => {\n const path = `ghostContent[${content.index}]`;\n const excess = bytes - MAX_RECORD_BYTES;\n const cause =\n fragment === null\n ? `fallback record needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the required projected metadata.`\n : `fragment at source position ${fragment.position} needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the indivisible source element or required projected metadata.`;\n\n return {\n kind: 'size',\n reason: 'record-too-large',\n path,\n index: content.index,\n contentId: content.id,\n objectID,\n anchor: fragment === null ? null : fragment.anchor,\n position: fragment === null ? null : fragment.position,\n bytes,\n limit: MAX_RECORD_BYTES,\n excess,\n message: `${path}: content \"${content.id}\" ${cause}`\n };\n};\n\n/**\n * Greedily packs whole extraction fragments into records that stay within the record byte\n * ceiling. A candidate is always measured under the continuation index it would be emitted\n * with, so the identifier growth of a continuation is inside the measurement. An indivisible\n * fragment yields an issue rather than a truncated record, and consumes no continuation index.\n */\nconst packAnchorGroup = (\n content: PreparedContent,\n groupIndex: number,\n group: FragmentGroup\n): ContentRecords => {\n const records: AlgoliaRecord[] = [];\n const issues: RecordSizeIssue[] = [];\n let packedFragments: NonEmptyFragments | null = null;\n let continuationIndex = 0;\n\n for (const fragment of group.fragments) {\n const candidate: NonEmptyFragments =\n packedFragments === null ? [fragment] : [...packedFragments, fragment];\n const candidateBytes = measureRecordBytes(\n buildFragmentRecord(content, groupIndex, continuationIndex, candidate)\n );\n if (candidateBytes <= MAX_RECORD_BYTES) {\n packedFragments = candidate;\n continue;\n }\n\n if (packedFragments === null) {\n issues.push(\n createSizeIssue(\n content,\n createObjectID(content, groupIndex, continuationIndex),\n fragment,\n candidateBytes\n )\n );\n continue;\n }\n\n records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments));\n continuationIndex += 1;\n\n const single: NonEmptyFragments = [fragment];\n const singleBytes = measureRecordBytes(\n buildFragmentRecord(content, groupIndex, continuationIndex, single)\n );\n if (singleBytes > MAX_RECORD_BYTES) {\n issues.push(\n createSizeIssue(\n content,\n createObjectID(content, groupIndex, continuationIndex),\n fragment,\n singleBytes\n )\n );\n packedFragments = null;\n continue;\n }\n packedFragments = single;\n }\n\n if (packedFragments !== null) {\n records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments));\n }\n\n return {records, issues};\n};\n\nconst createFallbackRecord = (content: PreparedContent): ContentRecords => {\n const objectID = `${content.id}_0`;\n const record = assembleRecord(content, {\n objectID,\n url: content.url,\n html: '',\n headings: [],\n anchor: null,\n position: FALLBACK_POSITION,\n heading: FALLBACK_HEADING_RANK\n });\n\n const bytes = measureRecordBytes(record);\n if (bytes > MAX_RECORD_BYTES) {\n return {records: [], issues: [createSizeIssue(content, objectID, null, bytes)]};\n }\n\n return {records: [record], issues: []};\n};\n\n/**\n * Builds every Algolia record for one prepared Ghost content item in anchor-group order, then\n * continuation order. Content without extraction fragments emits the single fallback record.\n */\nexport const createContentRecords = (\n content: PreparedContent,\n groups: readonly FragmentGroup[]\n): ContentRecords => {\n if (groups.length === 0) {\n return createFallbackRecord(content);\n }\n\n const records: AlgoliaRecord[] = [];\n const issues: RecordSizeIssue[] = [];\n for (const [groupIndex, group] of groups.entries()) {\n const packed = packAnchorGroup(content, groupIndex, group);\n records.push(...packed.records);\n issues.push(...packed.issues);\n }\n\n return {records, issues};\n};\n"]} \ No newline at end of file diff --git a/packages/algolia-fragmenter/src/create-algolia-records.mts b/packages/algolia-fragmenter/src/create-algolia-records.mts new file mode 100644 index 00000000..634f1fea --- /dev/null +++ b/packages/algolia-fragmenter/src/create-algolia-records.mts @@ -0,0 +1,48 @@ +import {extract} from '@tryghost/algolia-html-extractor'; + +import {FragmenterError, type RecordSizeIssue} from './errors.mjs'; +import {groupFragmentsByAnchor} from './grouping.mjs'; +import {resolvePolicy, type CreateAlgoliaRecordsOptions} from './policy.mjs'; +import {prepareGhostContent, type GhostContent} from './projection.mjs'; +import {createContentRecords, type AlgoliaRecord} from './records.mjs'; + +/** + * Turns Ghost content into complete final Algolia records: projection, HTML extraction, legacy + * anchor grouping, fallback records, deep links, identifiers, ranking metadata, and + * deterministic record-size handling. + * + * The whole batch is validated before any record is returned. A deterministic policy, Ghost + * content, or record-size problem throws one {@link FragmenterError} carrying every issue in + * input order; a partial batch is never returned. + * + * @throws {FragmenterError} `INVALID_POLICY`, `INVALID_GHOST_CONTENT`, or `RECORD_TOO_LARGE`. + */ +export const createAlgoliaRecords = ( + ghostContent: readonly GhostContent[], + options?: CreateAlgoliaRecordsOptions +): readonly AlgoliaRecord[] => { + const policy = resolvePolicy(options); + if (!policy.ok) { + throw new FragmenterError('INVALID_POLICY', policy.issues); + } + + const prepared = prepareGhostContent(ghostContent, policy.policy); + if (!prepared.ok) { + throw new FragmenterError('INVALID_GHOST_CONTENT', prepared.issues); + } + + const records: AlgoliaRecord[] = []; + const issues: RecordSizeIssue[] = []; + for (const content of prepared.contents) { + const groups = groupFragmentsByAnchor(extract(content.html)); + const contentRecords = createContentRecords(content, groups); + records.push(...contentRecords.records); + issues.push(...contentRecords.issues); + } + + if (issues.length > 0) { + throw new FragmenterError('RECORD_TOO_LARGE', issues); + } + + return records; +}; diff --git a/packages/algolia-fragmenter/src/errors.mts b/packages/algolia-fragmenter/src/errors.mts new file mode 100644 index 00000000..caae213d --- /dev/null +++ b/packages/algolia-fragmenter/src/errors.mts @@ -0,0 +1,78 @@ +export type FragmenterErrorCode = 'INVALID_POLICY' | 'INVALID_GHOST_CONTENT' | 'RECORD_TOO_LARGE'; + +export type PolicyIssueReason = + | 'invalid-shape' + | 'unknown-property' + | 'unknown-source' + | 'repeated-source' + | 'repeated-output' + | 'invalid-alias' + | 'protected-collision' + | 'container-collision' + | 'canonical-collision' + | 'reserved-collision'; + +export type PolicyIssue = Readonly<{ + kind: 'policy'; + reason: PolicyIssueReason; + path: string; + message: string; +}>; + +export type GhostContentIssueReason = 'invalid-shape' | 'missing' | 'wrong-type'; + +export type ExpectedValueType = 'string' | 'number' | 'boolean' | 'object' | 'array'; + +export type GhostContentIssue = Readonly<{ + kind: 'content'; + reason: GhostContentIssueReason; + path: string; + index: number | null; + contentId: string | null; + expected: ExpectedValueType; + message: string; +}>; + +export type RecordSizeIssue = Readonly<{ + kind: 'size'; + reason: 'record-too-large'; + path: string; + index: number; + contentId: string; + objectID: string; + anchor: string | null; + position: number | null; + bytes: number; + limit: number; + excess: number; + message: string; +}>; + +export type FragmenterIssue = PolicyIssue | GhostContentIssue | RecordSizeIssue; + +const MESSAGE_ISSUE_LIMIT = 5; + +const describeIssues = (code: FragmenterErrorCode, issues: readonly FragmenterIssue[]): string => { + const listed = issues.slice(0, MESSAGE_ISSUE_LIMIT).map(issue => issue.message); + const remaining = issues.length - listed.length; + const suffix = remaining > 0 ? `; and ${remaining} more` : ''; + const count = `${issues.length} issue${issues.length === 1 ? '' : 's'}`; + + return `${code}: ${count}. ${listed.join('; ')}${suffix}`; +}; + +/** + * The single public error for every deterministic policy, Ghost content, or record size + * problem found while building Algolia records. It never carries a partial record batch. + */ +export class FragmenterError extends Error { + readonly code: FragmenterErrorCode; + readonly issues: readonly FragmenterIssue[]; + + constructor(code: FragmenterErrorCode, issues: readonly FragmenterIssue[]) { + super(describeIssues(code, issues)); + this.name = 'FragmenterError'; + this.code = code; + this.issues = Object.freeze([...issues]); + } +} diff --git a/packages/algolia-fragmenter/src/grouping.mts b/packages/algolia-fragmenter/src/grouping.mts new file mode 100644 index 00000000..4638eb2a --- /dev/null +++ b/packages/algolia-fragmenter/src/grouping.mts @@ -0,0 +1,46 @@ +import type {ExtractionFragment} from '@tryghost/algolia-html-extractor'; + +export type NonEmptyFragments = readonly [ExtractionFragment, ...ExtractionFragment[]]; + +export type FragmentGroup = Readonly<{ + anchor: string | null; + fragments: NonEmptyFragments; +}>; + +type MutableFragmentGroup = { + anchor: string | null; + fragments: [ExtractionFragment, ...ExtractionFragment[]]; +}; + +/** + * Collects extraction fragments into first-seen anchor groups. Non-adjacent fragments that + * repeat an anchor join the existing group, which is the legacy grouping rule shared by the + * deprecated wrappers and the deep record interface. + */ +export const groupFragmentsByAnchor = ( + fragments: readonly ExtractionFragment[] +): readonly FragmentGroup[] => { + const groups: MutableFragmentGroup[] = []; + for (const fragment of fragments) { + const existingGroup = groups.find(group => group.anchor === fragment.anchor); + if (existingGroup === undefined) { + groups.push({anchor: fragment.anchor, fragments: [fragment]}); + continue; + } + existingGroup.fragments.push(fragment); + } + + return groups; +}; + +/** + * Merges the fragments of one record. The first fragment contributes its markup verbatim; + * every later preformatted fragment contributes its text only. + */ +export const mergeRecordHtml = (fragments: readonly ExtractionFragment[]): string => { + return fragments + .map((fragment, index) => + index > 0 && fragment.sourceTag === 'pre' ? ` ${fragment.text}` : fragment.html + ) + .join(''); +}; diff --git a/packages/algolia-fragmenter/src/index.mts b/packages/algolia-fragmenter/src/index.mts index f0ca7ad0..6a0ce650 100644 --- a/packages/algolia-fragmenter/src/index.mts +++ b/packages/algolia-fragmenter/src/index.mts @@ -1,8 +1,31 @@ -import {extract, type ExtractedTagName, type HeadingRank} from '@tryghost/algolia-html-extractor'; - -export type GhostContent = Readonly>; - -export type AlgoliaRecord = Record; +import {extract} from '@tryghost/algolia-html-extractor'; + +import {groupFragmentsByAnchor, mergeRecordHtml, type FragmentGroup} from './grouping.mjs'; +import type {GhostContent} from './projection.mjs'; +import type {AlgoliaRecord} from './records.mjs'; + +export {createAlgoliaRecords} from './create-algolia-records.mjs'; +export {FragmenterError} from './errors.mjs'; +export type { + ExpectedValueType, + FragmenterErrorCode, + FragmenterIssue, + GhostContentIssue, + GhostContentIssueReason, + PolicyIssue, + PolicyIssueReason, + RecordSizeIssue +} from './errors.mjs'; +export type { + ContentProjection, + CreateAlgoliaRecordsOptions, + OptionalProjectionSource, + ProjectionField, + RankingField, + RankingSource +} from './policy.mjs'; +export type {GhostContent} from './projection.mjs'; +export type {AlgoliaRecord} from './records.mjs'; type GhostRelation = Readonly>; @@ -11,60 +34,23 @@ type LegacyRelationCollection = { forEach(callback: (relation: GhostRelation) => void): void; }; -type GroupedFragment = { - html: string; - headings: string[]; - anchor: string | null; - customRanking: { - position: number; - heading: HeadingRank; - }; -}; - -type LegacyFragment = GroupedFragment & { - content: string; - sourceTag: ExtractedTagName; -}; - -const createLegacyFragment = (fragment: ReturnType[number]): LegacyFragment => ({ - html: fragment.html, - content: fragment.text, - headings: [...fragment.headingPath], - anchor: fragment.anchor, - sourceTag: fragment.sourceTag, - customRanking: { - position: fragment.position, - heading: fragment.headingRank - } -}); - -const reduceFragmentsUnderHeadings = ( - groups: LegacyFragment[], - fragment: LegacyFragment -): LegacyFragment[] => { - const existingGroup = groups.find(group => group.anchor === fragment.anchor); - if (existingGroup === undefined) { - groups.push(fragment); - return groups; - } - - existingGroup.html += fragment.sourceTag === 'pre' ? ` ${fragment.content}` : fragment.html; - existingGroup.content += ` ${fragment.content}`; - return groups; -}; - const toAlgoliaRecord = ( ghostContent: AlgoliaRecord, - fragment: LegacyFragment, + group: FragmentGroup, index: number ): AlgoliaRecord => { - const {content: _content, sourceTag: _sourceTag, ...groupedFragment} = fragment; - const url = - fragment.anchor === null ? ghostContent.url : `${ghostContent.url}#${fragment.anchor}`; + const [first] = group.fragments; + const url = group.anchor === null ? ghostContent.url : `${ghostContent.url}#${group.anchor}`; return { ...ghostContent, - ...groupedFragment, + html: mergeRecordHtml(group.fragments), + headings: [...first.headingPath], + anchor: group.anchor, + customRanking: { + position: first.position, + heading: first.headingRank + }, url, objectID: `${ghostContent.objectID}_${index}` }; @@ -77,12 +63,8 @@ export const fragmentTransformer = ( recordAccumulator: AlgoliaRecord[], ghostContent: AlgoliaRecord ): AlgoliaRecord[] => { - const groupedFragments = extract(ghostContent.html as string) - .map(createLegacyFragment) - .reduce(reduceFragmentsUnderHeadings, []); - const records = groupedFragments.map((fragment, index) => - toAlgoliaRecord(ghostContent, fragment, index) - ); + const groups = groupFragmentsByAnchor(extract(ghostContent.html as string)); + const records = groups.map((group, index) => toAlgoliaRecord(ghostContent, group, index)); return [...recordAccumulator, ...records]; }; diff --git a/packages/algolia-fragmenter/src/policy.mts b/packages/algolia-fragmenter/src/policy.mts new file mode 100644 index 00000000..a511d538 --- /dev/null +++ b/packages/algolia-fragmenter/src/policy.mts @@ -0,0 +1,504 @@ +import type {PolicyIssue, PolicyIssueReason} from './errors.mjs'; + +export type OptionalProjectionSource = + | 'image' + | 'tags' + | 'authors' + | 'excerpt' + | 'custom_excerpt' + | 'feature_image_alt' + | 'feature_image_caption' + | 'canonical_url' + | 'featured' + | 'visibility' + | 'created_at' + | 'updated_at' + | 'published_at' + | 'reading_time'; + +export type ProjectionField = + | OptionalProjectionSource + | Readonly<{ + source: OptionalProjectionSource; + as: string; + }>; + +export type RankingSource = 'featured' | 'reading_time'; + +export type RankingField = Readonly<{ + source: RankingSource; + as: string; +}>; + +export type ContentProjection = Readonly<{ + fields: readonly ProjectionField[]; + customRanking?: readonly RankingField[]; +}>; + +export type CreateAlgoliaRecordsOptions = Readonly<{ + ignoreSlugs?: readonly string[]; + contentProjection?: ContentProjection; +}>; + +export type ResolvedProjectionField = Readonly<{ + source: OptionalProjectionSource; + outputKey: string; +}>; + +export type ResolvedRankingField = Readonly<{ + source: RankingSource; + outputKey: string; +}>; + +export type ResolvedPolicy = Readonly<{ + ignoreSlugs: readonly string[]; + fields: readonly ResolvedProjectionField[]; + rankingFields: readonly ResolvedRankingField[]; +}>; + +export type PolicyResolution = + | Readonly<{ok: true; policy: ResolvedPolicy}> + | Readonly<{ok: false; issues: readonly PolicyIssue[]}>; + +type AliasedFieldShape = Readonly<{source: string; alias: string}>; + +type ProjectionFieldShape = Readonly<{source: string; alias: string | null}>; + +type Resolution = + | Readonly<{ok: true; value: Value}> + | Readonly<{ok: false; issue: PolicyIssue}>; + +type ResolvedProjection = Readonly<{ + fields: readonly ResolvedProjectionField[]; + rankingFields: readonly ResolvedRankingField[]; +}>; + +/** + * Every canonical allowlist name. The `Record` keeps this set exhaustive: a source added to + * `OptionalProjectionSource` without a name here fails to compile. + */ +const CANONICAL_SOURCE_NAMES: Readonly> = { + image: true, + tags: true, + authors: true, + excerpt: true, + custom_excerpt: true, + feature_image_alt: true, + feature_image_caption: true, + canonical_url: true, + featured: true, + visibility: true, + created_at: true, + updated_at: true, + published_at: true, + reading_time: true +}; + +const RANKING_SOURCE_NAMES: Readonly> = { + featured: true, + reading_time: true +}; + +const DEFAULT_PROJECTION_SOURCES: readonly OptionalProjectionSource[] = [ + 'image', + 'tags', + 'authors', + 'excerpt' +]; + +const PROTECTED_RECORD_FIELDS: readonly string[] = [ + 'objectID', + 'slug', + 'url', + 'title', + 'html', + 'headings', + 'anchor' +]; + +const PROTECTED_RANKING_FIELDS: readonly string[] = ['heading', 'position']; + +const PROTECTED_RANKING_OUTPUT_NAMES: readonly string[] = [ + ...PROTECTED_RECORD_FIELDS, + ...PROTECTED_RANKING_FIELDS +]; + +const ALGOLIA_RESERVED_NAMES: readonly string[] = [ + '_highlightResult', + '_snippetResult', + '_rankingInfo', + '_distinctSeqID', + 'distinctSeqId', + '_tags', + '_geoloc' +]; + +const RANKING_CONTAINER = 'customRanking'; +const ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/u; +const OPTIONS_PROPERTIES: readonly string[] = ['ignoreSlugs', 'contentProjection']; +const CONTENT_PROJECTION_PROPERTIES: readonly string[] = ['fields', 'customRanking']; +const PROJECTION_FIELD_PROPERTIES: readonly string[] = ['source', 'as']; + +export const isPlainObject = (value: unknown): value is Record => { + return typeof value === 'object' && value !== null && !Array.isArray(value); +}; + +const isProjectionSource = (value: string): value is OptionalProjectionSource => { + return Object.hasOwn(CANONICAL_SOURCE_NAMES, value); +}; + +const isRankingSource = (value: string): value is RankingSource => { + return Object.hasOwn(RANKING_SOURCE_NAMES, value); +}; + +const createIssue = (reason: PolicyIssueReason, path: string, message: string): PolicyIssue => ({ + kind: 'policy', + reason, + path, + message +}); + +const invalidShape = (path: string, expectedShape: string): PolicyIssue => + createIssue('invalid-shape', path, `${path}: expected ${expectedShape}.`); + +const unknownProperty = (path: string, name: string): PolicyIssue => + createIssue('unknown-property', path, `${path}: unknown property "${name}".`); + +const unknownSource = (path: string, value: string, kind: string): PolicyIssue => + createIssue('unknown-source', path, `${path}: "${value}" is not an allowed ${kind} source.`); + +const repeatedSource = (path: string, source: string, kind: string): PolicyIssue => + createIssue( + 'repeated-source', + path, + `${path}: ${kind} source "${source}" is configured more than once.` + ); + +const invalidAlias = (path: string, name: string): PolicyIssue => + createIssue( + 'invalid-alias', + path, + `${path}: alias "${name}" must match ^[A-Za-z][A-Za-z0-9_]*$.` + ); + +const findUnknownProperties = ( + value: Readonly>, + allowed: readonly string[] +): readonly string[] => { + return Object.keys(value).filter(key => !allowed.includes(key)); +}; + +const collectUnknownProperties = ( + value: Readonly>, + allowed: readonly string[], + path: string, + issues: PolicyIssue[] +): void => { + for (const key of findUnknownProperties(value, allowed)) { + issues.push(unknownProperty(path, key)); + } +}; + +/** + * Output names live in one policy-wide namespace shared by projection fields and ranking + * siblings, so the checks below run in a fixed order for every configured output name. + */ +const findOutputCollision = ( + outputName: string, + source: string, + path: string, + protectedNames: readonly string[], + usedOutputNames: ReadonlySet +): PolicyIssue | null => { + if (protectedNames.includes(outputName)) { + const owner = PROTECTED_RANKING_FIELDS.includes(outputName) ? 'ranking' : 'record'; + return createIssue( + 'protected-collision', + path, + `${path}: output name "${outputName}" is a protected ${owner} field.` + ); + } + if (outputName === RANKING_CONTAINER) { + return createIssue( + 'container-collision', + path, + `${path}: output name "customRanking" is the package-owned ranking container.` + ); + } + if (ALGOLIA_RESERVED_NAMES.includes(outputName)) { + return createIssue( + 'reserved-collision', + path, + `${path}: output name "${outputName}" is reserved by Algolia.` + ); + } + if (outputName !== source && isProjectionSource(outputName)) { + return createIssue( + 'canonical-collision', + path, + `${path}: output name "${outputName}" impersonates a canonical allowlist field.` + ); + } + if (usedOutputNames.has(outputName)) { + return createIssue( + 'repeated-output', + path, + `${path}: output name "${outputName}" is produced more than once.` + ); + } + + return null; +}; + +const readAliasedFieldShape = ( + entry: unknown, + path: string, + expectedShape: string +): Resolution => { + if (!isPlainObject(entry)) { + return {ok: false, issue: invalidShape(path, expectedShape)}; + } + + const [unknownKey] = findUnknownProperties(entry, PROJECTION_FIELD_PROPERTIES); + if (unknownKey !== undefined) { + return {ok: false, issue: unknownProperty(path, unknownKey)}; + } + if (typeof entry.source !== 'string' || typeof entry.as !== 'string') { + return {ok: false, issue: invalidShape(path, 'a {source, as} object of strings')}; + } + + return {ok: true, value: {source: entry.source, alias: entry.as}}; +}; + +const readProjectionFieldShape = ( + entry: unknown, + path: string +): Resolution => { + if (typeof entry === 'string') { + return {ok: true, value: {source: entry, alias: null}}; + } + + return readAliasedFieldShape(entry, path, 'a projection source name or a {source, as} object'); +}; + +const resolveProjectionField = ( + entry: unknown, + path: string, + usedSources: Set, + usedOutputNames: Set +): Resolution => { + const shape = readProjectionFieldShape(entry, path); + if (!shape.ok) { + return shape; + } + + const {source, alias} = shape.value; + if (!isProjectionSource(source)) { + return {ok: false, issue: unknownSource(path, source, 'projection')}; + } + if (usedSources.has(source)) { + return {ok: false, issue: repeatedSource(path, source, 'projection')}; + } + + const outputPath = alias === null ? path : `${path}.as`; + if (alias !== null && !ALIAS_PATTERN.test(alias)) { + return {ok: false, issue: invalidAlias(outputPath, alias)}; + } + + const outputKey = alias ?? source; + const collision = findOutputCollision( + outputKey, + source, + outputPath, + PROTECTED_RECORD_FIELDS, + usedOutputNames + ); + if (collision !== null) { + return {ok: false, issue: collision}; + } + + usedSources.add(source); + usedOutputNames.add(outputKey); + return {ok: true, value: {source, outputKey}}; +}; + +const resolveRankingField = ( + entry: unknown, + path: string, + usedSources: Set, + usedOutputNames: Set +): Resolution => { + const shape = readAliasedFieldShape(entry, path, 'a {source, as} object'); + if (!shape.ok) { + return shape; + } + + const {source, alias} = shape.value; + if (!isRankingSource(source)) { + return {ok: false, issue: unknownSource(path, source, 'ranking')}; + } + if (usedSources.has(source)) { + return {ok: false, issue: repeatedSource(path, source, 'ranking')}; + } + + const outputPath = `${path}.as`; + if (!ALIAS_PATTERN.test(alias)) { + return {ok: false, issue: invalidAlias(outputPath, alias)}; + } + + const collision = findOutputCollision( + alias, + source, + outputPath, + PROTECTED_RANKING_OUTPUT_NAMES, + usedOutputNames + ); + if (collision !== null) { + return {ok: false, issue: collision}; + } + + usedSources.add(source); + usedOutputNames.add(alias); + return {ok: true, value: {source, outputKey: alias}}; +}; + +/** + * Resolves one configured list. Sources are unique per list, while output names are checked + * against the policy-wide namespace the caller owns. + */ +const resolveEntries = ( + entries: readonly unknown[], + listPath: string, + usedOutputNames: Set, + issues: PolicyIssue[], + resolveEntry: ( + entry: unknown, + path: string, + usedSources: Set, + usedOutputNames: Set + ) => Resolution +): readonly Field[] => { + const usedSources = new Set(); + const fields: Field[] = []; + for (const [index, entry] of entries.entries()) { + const resolved = resolveEntry(entry, `${listPath}[${index}]`, usedSources, usedOutputNames); + if (!resolved.ok) { + issues.push(resolved.issue); + continue; + } + fields.push(resolved.value); + } + + return fields; +}; + +const resolveFields = ( + value: unknown, + usedOutputNames: Set, + issues: PolicyIssue[] +): readonly ResolvedProjectionField[] => { + if (!Array.isArray(value)) { + issues.push(invalidShape('contentProjection.fields', 'an array of projection fields')); + return []; + } + + return resolveEntries( + value, + 'contentProjection.fields', + usedOutputNames, + issues, + resolveProjectionField + ); +}; + +const resolveRankingFields = ( + value: unknown, + usedOutputNames: Set, + issues: PolicyIssue[] +): readonly ResolvedRankingField[] => { + if (value === undefined) { + return []; + } + if (!Array.isArray(value)) { + issues.push(invalidShape('contentProjection.customRanking', 'an array of ranking fields')); + return []; + } + + return resolveEntries( + value, + 'contentProjection.customRanking', + usedOutputNames, + issues, + resolveRankingField + ); +}; + +const createDefaultProjection = (): ResolvedProjection => ({ + fields: DEFAULT_PROJECTION_SOURCES.map(source => ({source, outputKey: source})), + rankingFields: [] +}); + +const resolveContentProjection = (value: unknown, issues: PolicyIssue[]): ResolvedProjection => { + if (value === undefined) { + return createDefaultProjection(); + } + if (!isPlainObject(value)) { + issues.push(invalidShape('contentProjection', 'an object')); + return {fields: [], rankingFields: []}; + } + + collectUnknownProperties(value, CONTENT_PROJECTION_PROPERTIES, 'contentProjection', issues); + const usedOutputNames = new Set(); + + return { + fields: resolveFields(value.fields, usedOutputNames, issues), + rankingFields: resolveRankingFields(value.customRanking, usedOutputNames, issues) + }; +}; + +const resolveIgnoreSlugs = (value: unknown, issues: PolicyIssue[]): readonly string[] => { + if (value === undefined) { + return []; + } + if (!Array.isArray(value)) { + issues.push(invalidShape('ignoreSlugs', 'an array of strings')); + return []; + } + + const slugs: string[] = []; + for (const [index, entry] of value.entries()) { + if (typeof entry !== 'string') { + issues.push(invalidShape(`ignoreSlugs[${index}]`, 'a string')); + continue; + } + slugs.push(entry); + } + + return slugs; +}; + +/** + * Validates caller options before any Ghost content is inspected and returns the policy the + * projection, ranking, and record stages read. Every policy issue is collected in declaration + * order rather than stopping at the first one. + */ +export const resolvePolicy = (options: unknown): PolicyResolution => { + if (options === undefined) { + const projection = createDefaultProjection(); + return {ok: true, policy: {ignoreSlugs: [], ...projection}}; + } + if (!isPlainObject(options)) { + return {ok: false, issues: [invalidShape('options', 'an object')]}; + } + + const issues: PolicyIssue[] = []; + collectUnknownProperties(options, OPTIONS_PROPERTIES, 'options', issues); + const ignoreSlugs = resolveIgnoreSlugs(options.ignoreSlugs, issues); + const projection = resolveContentProjection(options.contentProjection, issues); + + if (issues.length > 0) { + return {ok: false, issues}; + } + + return {ok: true, policy: {ignoreSlugs, ...projection}}; +}; diff --git a/packages/algolia-fragmenter/src/projection.mts b/packages/algolia-fragmenter/src/projection.mts new file mode 100644 index 00000000..5f0dc880 --- /dev/null +++ b/packages/algolia-fragmenter/src/projection.mts @@ -0,0 +1,370 @@ +import type {ExpectedValueType, GhostContentIssue, GhostContentIssueReason} from './errors.mjs'; +import {isPlainObject, type OptionalProjectionSource, type ResolvedPolicy} from './policy.mjs'; + +export type GhostContent = Readonly>; + +export type PreparedContent = Readonly<{ + index: number; + id: string; + slug: string; + url: string; + title: string; + html: string; + projected: Readonly>; + rankingSiblings: Readonly>; +}>; + +export type ContentPreparation = + | Readonly<{ok: true; contents: readonly PreparedContent[]}> + | Readonly<{ok: false; issues: readonly GhostContentIssue[]}>; + +type ProjectionValueKind = 'string' | 'boolean' | 'number' | 'relations'; + +type ProjectionSourceDescriptor = Readonly<{ghostKey: string; kind: ProjectionValueKind}>; + +type IssueContext = Readonly<{index: number; contentId: string | null; path: string}>; + +type FailedRead = Readonly<{ok: false; issue: GhostContentIssue}>; + +type ValueRead = Readonly<{ok: true; value: Value}> | FailedRead; + +type ItemPreparation = + | Readonly<{kind: 'content'; content: PreparedContent}> + | Readonly<{kind: 'ignored'}> + | Readonly<{kind: 'issues'; issues: readonly GhostContentIssue[]}>; + +/** + * The single projection-source descriptor table. Content validation and value projection read + * it through the same reader, so a Ghost source can never be validated as one type and + * projected as another. + */ +const PROJECTION_SOURCES = { + image: {ghostKey: 'feature_image', kind: 'string'}, + tags: {ghostKey: 'tags', kind: 'relations'}, + authors: {ghostKey: 'authors', kind: 'relations'}, + excerpt: {ghostKey: 'excerpt', kind: 'string'}, + custom_excerpt: {ghostKey: 'custom_excerpt', kind: 'string'}, + feature_image_alt: {ghostKey: 'feature_image_alt', kind: 'string'}, + feature_image_caption: {ghostKey: 'feature_image_caption', kind: 'string'}, + canonical_url: {ghostKey: 'canonical_url', kind: 'string'}, + featured: {ghostKey: 'featured', kind: 'boolean'}, + visibility: {ghostKey: 'visibility', kind: 'string'}, + created_at: {ghostKey: 'created_at', kind: 'string'}, + updated_at: {ghostKey: 'updated_at', kind: 'string'}, + published_at: {ghostKey: 'published_at', kind: 'string'}, + reading_time: {ghostKey: 'reading_time', kind: 'number'} +} as const satisfies Readonly>; + +const describeReceived = (value: unknown): string => { + if (value === null) { + return 'null'; + } + if (Array.isArray(value)) { + return 'array'; + } + + return typeof value; +}; + +const createContentIssue = ( + context: IssueContext, + reason: GhostContentIssueReason, + path: string, + expected: ExpectedValueType, + message: string +): GhostContentIssue => ({ + kind: 'content', + reason, + path, + index: context.index, + contentId: context.contentId, + expected, + message +}); + +const missingIssue = (context: IssueContext, path: string): GhostContentIssue => + createContentIssue( + context, + 'missing', + path, + 'string', + `${path}: required Ghost field is missing.` + ); + +const wrongTypeIssue = ( + context: IssueContext, + path: string, + expected: ExpectedValueType, + received: unknown +): GhostContentIssue => + createContentIssue( + context, + 'wrong-type', + path, + expected, + `${path}: expected ${expected} but received ${describeReceived(received)}.` + ); + +const emptyIdentityIssue = (context: IssueContext, path: string): GhostContentIssue => + createContentIssue( + context, + 'wrong-type', + path, + 'string', + `${path}: expected a non-empty string but received an empty string.` + ); + +const createBatchShapeIssue = (): GhostContentIssue => ({ + kind: 'content', + reason: 'invalid-shape', + path: 'ghostContent', + index: null, + contentId: null, + expected: 'array', + message: 'ghostContent: expected array.' +}); + +const invalidShapeIssue = ( + context: IssueContext, + path: string, + expected: ExpectedValueType +): GhostContentIssue => + createContentIssue(context, 'invalid-shape', path, expected, `${path}: expected ${expected}.`); + +const isFailedRead = (read: ValueRead): read is FailedRead => !read.ok; + +const readContentString = ( + raw: unknown, + path: string, + context: IssueContext +): ValueRead => { + if (raw === undefined || raw === null) { + return {ok: false, issue: missingIssue(context, path)}; + } + if (typeof raw !== 'string') { + return {ok: false, issue: wrongTypeIssue(context, path, 'string', raw)}; + } + + return {ok: true, value: raw}; +}; + +const readIdentityString = ( + raw: unknown, + path: string, + context: IssueContext +): ValueRead => { + const read = readContentString(raw, path, context); + if (read.ok && read.value === '') { + return {ok: false, issue: emptyIdentityIssue(context, path)}; + } + + return read; +}; + +const readScalar = ( + kind: Exclude, + raw: unknown, + path: string, + context: IssueContext +): ValueRead => { + if (raw === undefined || raw === null) { + return {ok: true, value: null}; + } + if (typeof raw !== kind) { + return {ok: false, issue: wrongTypeIssue(context, path, kind, raw)}; + } + + return {ok: true, value: raw}; +}; + +const readRelations = ( + raw: unknown, + path: string, + context: IssueContext +): ValueRead[]> => { + if (raw === undefined || raw === null) { + return {ok: true, value: []}; + } + if (!Array.isArray(raw)) { + return {ok: false, issue: wrongTypeIssue(context, path, 'array', raw)}; + } + + const relations: Array<{name: string; slug: string}> = []; + for (const [index, element] of raw.entries()) { + const elementPath = `${path}[${index}]`; + if (!isPlainObject(element)) { + return {ok: false, issue: wrongTypeIssue(context, elementPath, 'object', element)}; + } + + const {name, slug} = element; + if (typeof name !== 'string') { + return { + ok: false, + issue: wrongTypeIssue(context, `${elementPath}.name`, 'string', name) + }; + } + if (typeof slug !== 'string') { + return { + ok: false, + issue: wrongTypeIssue(context, `${elementPath}.slug`, 'string', slug) + }; + } + + relations.push({name, slug}); + } + + return {ok: true, value: relations}; +}; + +const readSource = ( + item: Readonly>, + source: OptionalProjectionSource, + context: IssueContext +): ValueRead => { + const descriptor: ProjectionSourceDescriptor = PROJECTION_SOURCES[source]; + const path = `${context.path}.${descriptor.ghostKey}`; + const raw = item[descriptor.ghostKey]; + if (descriptor.kind === 'relations') { + return readRelations(raw, path, context); + } + + return readScalar(descriptor.kind, raw, path, context); +}; + +/** + * Every enabled projection source, then every ranking source that no projection field already + * reads, so a source feeding both a projection field and a ranking sibling is read once. + */ +const collectEnabledSources = (policy: ResolvedPolicy): readonly OptionalProjectionSource[] => { + const sources: OptionalProjectionSource[] = []; + for (const field of [...policy.fields, ...policy.rankingFields]) { + if (!sources.includes(field.source)) { + sources.push(field.source); + } + } + + return sources; +}; + +const readEnabledSources = ( + item: Readonly>, + policy: ResolvedPolicy, + context: IssueContext +): Readonly<{ + values: ReadonlyMap; + issues: readonly GhostContentIssue[]; +}> => { + const values = new Map(); + const issues: GhostContentIssue[] = []; + for (const source of collectEnabledSources(policy)) { + const read = readSource(item, source, context); + if (isFailedRead(read)) { + issues.push(read.issue); + continue; + } + values.set(source, read.value); + } + + return {values, issues}; +}; + +const projectValues = ( + fields: readonly Readonly<{source: OptionalProjectionSource; outputKey: string}>[], + values: ReadonlyMap +): Readonly> => { + const projected: Record = {}; + for (const field of fields) { + projected[field.outputKey] = values.get(field.source); + } + + return projected; +}; + +const readContentId = (item: Readonly>): string | null => { + return typeof item.id === 'string' && item.id !== '' ? item.id : null; +}; + +const prepareItem = (value: unknown, index: number, policy: ResolvedPolicy): ItemPreparation => { + const path = `ghostContent[${index}]`; + if (!isPlainObject(value)) { + const context: IssueContext = {index, contentId: null, path}; + return {kind: 'issues', issues: [invalidShapeIssue(context, path, 'object')]}; + } + + const context: IssueContext = {index, contentId: readContentId(value), path}; + + const slug = readIdentityString(value.slug, `${path}.slug`, context); + if (isFailedRead(slug)) { + return {kind: 'issues', issues: [slug.issue]}; + } + if (policy.ignoreSlugs.includes(slug.value)) { + return {kind: 'ignored'}; + } + + const id = readIdentityString(value.id, `${path}.id`, context); + const url = readContentString(value.url, `${path}.url`, context); + const title = readContentString(value.title, `${path}.title`, context); + const html = readContentString(value.html, `${path}.html`, context); + const enabled = readEnabledSources(value, policy, context); + + // The explicit chain is what narrows `id`, `url`, `title`, and `html` to successful reads + // for the return below; the `filter` only collects the issues in canonical order. Collapsing + // the two into one expression loses the narrowing. + if ( + isFailedRead(id) || + isFailedRead(url) || + isFailedRead(title) || + isFailedRead(html) || + enabled.issues.length > 0 + ) { + const required = [id, url, title, html].filter(isFailedRead).map(read => read.issue); + return {kind: 'issues', issues: [...required, ...enabled.issues]}; + } + + return { + kind: 'content', + content: { + index, + id: id.value, + slug: slug.value, + url: url.value, + title: title.value, + html: html.value, + projected: projectValues(policy.fields, enabled.values), + rankingSiblings: projectValues(policy.rankingFields, enabled.values) + } + }; +}; + +/** + * Validates the whole batch in input order and prepares the content that survives ignored-slug + * exclusion. Validation and projection share one reader, so a prepared item always projects the + * values that were validated. + */ +export const prepareGhostContent = ( + ghostContent: unknown, + policy: ResolvedPolicy +): ContentPreparation => { + if (!Array.isArray(ghostContent)) { + return {ok: false, issues: [createBatchShapeIssue()]}; + } + + const contents: PreparedContent[] = []; + const issues: GhostContentIssue[] = []; + for (const [index, item] of ghostContent.entries()) { + const prepared = prepareItem(item, index, policy); + if (prepared.kind === 'issues') { + issues.push(...prepared.issues); + continue; + } + if (prepared.kind === 'content') { + contents.push(prepared.content); + } + } + + if (issues.length > 0) { + return {ok: false, issues}; + } + + return {ok: true, contents}; +}; diff --git a/packages/algolia-fragmenter/src/records.mts b/packages/algolia-fragmenter/src/records.mts new file mode 100644 index 00000000..63f03ba1 --- /dev/null +++ b/packages/algolia-fragmenter/src/records.mts @@ -0,0 +1,216 @@ +import {Buffer} from 'node:buffer'; + +import type {ExtractionFragment, HeadingRank} from '@tryghost/algolia-html-extractor'; + +import type {RecordSizeIssue} from './errors.mjs'; +import {mergeRecordHtml, type FragmentGroup, type NonEmptyFragments} from './grouping.mjs'; +import type {PreparedContent} from './projection.mjs'; + +export type AlgoliaRecord = Record; + +export type ContentRecords = Readonly<{ + records: readonly AlgoliaRecord[]; + issues: readonly RecordSizeIssue[]; +}>; + +type RecordParts = Readonly<{ + objectID: string; + url: string; + html: string; + headings: readonly string[]; + anchor: string | null; + position: number; + heading: HeadingRank; +}>; + +const MAX_RECORD_BYTES = 9_999; +const FALLBACK_POSITION = 0; +const FALLBACK_HEADING_RANK: HeadingRank = 100; + +export const measureRecordBytes = (record: AlgoliaRecord): number => { + return Buffer.byteLength(JSON.stringify(record), 'utf8'); +}; + +const assembleRecord = (content: PreparedContent, parts: RecordParts): AlgoliaRecord => ({ + objectID: parts.objectID, + slug: content.slug, + url: parts.url, + html: parts.html, + title: content.title, + headings: [...parts.headings], + anchor: parts.anchor, + ...content.projected, + customRanking: { + position: parts.position, + heading: parts.heading, + ...content.rankingSiblings + } +}); + +const createObjectID = ( + content: PreparedContent, + groupIndex: number, + continuationIndex: number +): string => { + const groupObjectID = `${content.id}_${groupIndex}`; + return continuationIndex === 0 ? groupObjectID : `${groupObjectID}_${continuationIndex}`; +}; + +const buildFragmentRecord = ( + content: PreparedContent, + groupIndex: number, + continuationIndex: number, + packedFragments: NonEmptyFragments +): AlgoliaRecord => { + const [first] = packedFragments; + + return assembleRecord(content, { + objectID: createObjectID(content, groupIndex, continuationIndex), + url: first.anchor === null ? content.url : `${content.url}#${first.anchor}`, + html: mergeRecordHtml(packedFragments), + headings: first.headingPath, + anchor: first.anchor, + position: first.position, + heading: first.headingRank + }); +}; + +const createSizeIssue = ( + content: PreparedContent, + objectID: string, + fragment: ExtractionFragment | null, + bytes: number +): RecordSizeIssue => { + const path = `ghostContent[${content.index}]`; + const excess = bytes - MAX_RECORD_BYTES; + const cause = + fragment === null + ? `fallback record needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the required projected metadata.` + : `fragment at source position ${fragment.position} needs ${bytes} UTF-8 bytes (${excess} over the ${MAX_RECORD_BYTES}-byte ceiling). Shorten the indivisible source element or required projected metadata.`; + + return { + kind: 'size', + reason: 'record-too-large', + path, + index: content.index, + contentId: content.id, + objectID, + anchor: fragment === null ? null : fragment.anchor, + position: fragment === null ? null : fragment.position, + bytes, + limit: MAX_RECORD_BYTES, + excess, + message: `${path}: content "${content.id}" ${cause}` + }; +}; + +/** + * Greedily packs whole extraction fragments into records that stay within the record byte + * ceiling. A candidate is always measured under the continuation index it would be emitted + * with, so the identifier growth of a continuation is inside the measurement. An indivisible + * fragment yields an issue rather than a truncated record, and consumes no continuation index. + */ +const packAnchorGroup = ( + content: PreparedContent, + groupIndex: number, + group: FragmentGroup +): ContentRecords => { + const records: AlgoliaRecord[] = []; + const issues: RecordSizeIssue[] = []; + let packedFragments: NonEmptyFragments | null = null; + let continuationIndex = 0; + + for (const fragment of group.fragments) { + const candidate: NonEmptyFragments = + packedFragments === null ? [fragment] : [...packedFragments, fragment]; + const candidateBytes = measureRecordBytes( + buildFragmentRecord(content, groupIndex, continuationIndex, candidate) + ); + if (candidateBytes <= MAX_RECORD_BYTES) { + packedFragments = candidate; + continue; + } + + if (packedFragments === null) { + issues.push( + createSizeIssue( + content, + createObjectID(content, groupIndex, continuationIndex), + fragment, + candidateBytes + ) + ); + continue; + } + + records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments)); + continuationIndex += 1; + + const single: NonEmptyFragments = [fragment]; + const singleBytes = measureRecordBytes( + buildFragmentRecord(content, groupIndex, continuationIndex, single) + ); + if (singleBytes > MAX_RECORD_BYTES) { + issues.push( + createSizeIssue( + content, + createObjectID(content, groupIndex, continuationIndex), + fragment, + singleBytes + ) + ); + packedFragments = null; + continue; + } + packedFragments = single; + } + + if (packedFragments !== null) { + records.push(buildFragmentRecord(content, groupIndex, continuationIndex, packedFragments)); + } + + return {records, issues}; +}; + +const createFallbackRecord = (content: PreparedContent): ContentRecords => { + const objectID = `${content.id}_0`; + const record = assembleRecord(content, { + objectID, + url: content.url, + html: '', + headings: [], + anchor: null, + position: FALLBACK_POSITION, + heading: FALLBACK_HEADING_RANK + }); + + const bytes = measureRecordBytes(record); + if (bytes > MAX_RECORD_BYTES) { + return {records: [], issues: [createSizeIssue(content, objectID, null, bytes)]}; + } + + return {records: [record], issues: []}; +}; + +/** + * Builds every Algolia record for one prepared Ghost content item in anchor-group order, then + * continuation order. Content without extraction fragments emits the single fallback record. + */ +export const createContentRecords = ( + content: PreparedContent, + groups: readonly FragmentGroup[] +): ContentRecords => { + if (groups.length === 0) { + return createFallbackRecord(content); + } + + const records: AlgoliaRecord[] = []; + const issues: RecordSizeIssue[] = []; + for (const [groupIndex, group] of groups.entries()) { + const packed = packAnchorGroup(content, groupIndex, group); + records.push(...packed.records); + issues.push(...packed.issues); + } + + return {records, issues}; +}; diff --git a/packages/algolia-fragmenter/test/create-algolia-records.test.mts b/packages/algolia-fragmenter/test/create-algolia-records.test.mts new file mode 100644 index 00000000..3ce8c91f --- /dev/null +++ b/packages/algolia-fragmenter/test/create-algolia-records.test.mts @@ -0,0 +1,1518 @@ +import {Buffer} from 'node:buffer'; +import fs from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import {describe, expect, it} from 'vitest'; + +import { + createAlgoliaRecords, + FragmenterError, + fragmentTransformer, + transformToAlgoliaObject, + type AlgoliaRecord, + type CreateAlgoliaRecordsOptions, + type GhostContent, + type GhostContentIssue, + type RecordSizeIssue +} from '../src/index.mjs'; + +const MAX_RECORD_BYTES = 9999; +const testDirectory = path.dirname(fileURLToPath(import.meta.url)); + +type TestContent = Record; + +const readFixture = (fileName: string): string => { + return fs.readFileSync(path.join(testDirectory, 'fixtures', `${fileName}.html`), { + encoding: 'utf8' + }); +}; + +const createContent = (overrides: TestContent = {}): TestContent => ({ + id: 'post-1', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + title: 'Getting started', + html: '

Introduction.

', + ...overrides +}); + +/** + * Every rejection case feeds deliberately invalid values through the public signature, so the + * runtime validation rather than the compiler is what the assertions observe. + */ +const buildRecords = (ghostContent: unknown, options?: unknown): readonly AlgoliaRecord[] => { + return createAlgoliaRecords( + ghostContent as readonly GhostContent[], + options as CreateAlgoliaRecordsOptions + ); +}; + +const bytesOf = (record: unknown): number => { + return Buffer.byteLength(JSON.stringify(record), 'utf8'); +}; + +const expectFragmenterError = (run: () => unknown): FragmenterError => { + let caught: unknown; + try { + run(); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(FragmenterError); + return caught as FragmenterError; +}; + +const describeIssues = (error: FragmenterError): readonly string[] => { + return error.issues.map(issue => `${issue.reason} @ ${issue.path}`); +}; + +const contentIssuesOf = (error: FragmenterError): readonly GhostContentIssue[] => { + return error.issues.filter(issue => issue.kind === 'content'); +}; + +const sizeIssuesOf = (error: FragmenterError): readonly RecordSizeIssue[] => { + return error.issues.filter(issue => issue.kind === 'size'); +}; + +const paragraphs = (...texts: readonly string[]): string => { + return texts.map(text => `

${text}

`).join(''); +}; + +const ALL_PROJECTION_SOURCES = [ + 'image', + 'tags', + 'authors', + 'excerpt', + 'custom_excerpt', + 'feature_image_alt', + 'feature_image_caption', + 'canonical_url', + 'featured', + 'visibility', + 'created_at', + 'updated_at', + 'published_at', + 'reading_time' +]; + +const createFullyProjectedContent = (overrides: TestContent = {}): TestContent => + createContent({ + feature_image: 'https://example.com/feature.jpg', + tags: [{id: 'tag-id', name: 'Guide', slug: 'guide', description: 'not indexed'}], + authors: [{id: 'author-id', name: 'Ada Lovelace', slug: 'ada', bio: 'not indexed'}], + excerpt: 'Ghost computed excerpt', + custom_excerpt: 'Custom excerpt', + feature_image_alt: 'Feature image alt', + feature_image_caption: 'Feature image caption', + canonical_url: 'https://example.com/canonical/', + featured: true, + visibility: 'public', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-02T00:00:00.000Z', + published_at: '2026-01-03T00:00:00.000Z', + reading_time: 4, + ...overrides + }); + +describe('createAlgoliaRecords projection', () => { + it('projects image, tags, authors, and excerpt by default', () => { + const records = buildRecords([createFullyProjectedContent()]); + + expect(records).toEqual([ + { + objectID: 'post-1_0', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + html: '

Introduction.

', + title: 'Getting started', + headings: [], + anchor: null, + image: 'https://example.com/feature.jpg', + tags: [{name: 'Guide', slug: 'guide'}], + authors: [{name: 'Ada Lovelace', slug: 'ada'}], + excerpt: 'Ghost computed excerpt', + customRanking: {position: 0, heading: 100} + } + ]); + }); + + it('projects no optional fields when the configured field list is empty', () => { + const records = buildRecords([createFullyProjectedContent()], { + contentProjection: {fields: []} + }); + + expect(records).toEqual([ + { + objectID: 'post-1_0', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + html: '

Introduction.

', + title: 'Getting started', + headings: [], + anchor: null, + customRanking: {position: 0, heading: 100} + } + ]); + }); + + it('projects every allowlisted source under its canonical name', () => { + const [record] = buildRecords([createFullyProjectedContent()], { + contentProjection: {fields: ALL_PROJECTION_SOURCES} + }); + + expect(record).toEqual({ + objectID: 'post-1_0', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + html: '

Introduction.

', + title: 'Getting started', + headings: [], + anchor: null, + image: 'https://example.com/feature.jpg', + tags: [{name: 'Guide', slug: 'guide'}], + authors: [{name: 'Ada Lovelace', slug: 'ada'}], + excerpt: 'Ghost computed excerpt', + custom_excerpt: 'Custom excerpt', + feature_image_alt: 'Feature image alt', + feature_image_caption: 'Feature image caption', + canonical_url: 'https://example.com/canonical/', + featured: true, + visibility: 'public', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-02T00:00:00.000Z', + published_at: '2026-01-03T00:00:00.000Z', + reading_time: 4, + customRanking: {position: 0, heading: 100} + }); + }); + + it('reads image from feature_image', () => { + const [record] = buildRecords([ + createContent({feature_image: 'https://example.com/feature.jpg', image: 'ignored'}) + ]); + + expect(record).toMatchObject({image: 'https://example.com/feature.jpg'}); + }); + + it('reduces tags and authors to name and slug', () => { + const [record] = buildRecords([ + createContent({ + tags: [{id: 'tag-id', name: 'Guide', slug: 'guide', description: 'not indexed'}], + authors: [{id: 'author-id', name: 'Ada', slug: 'ada', bio: 'not indexed'}] + }) + ]); + + expect(record).toMatchObject({ + tags: [{name: 'Guide', slug: 'guide'}], + authors: [{name: 'Ada', slug: 'ada'}] + }); + }); + + it('renames optional fields with validated aliases', () => { + const [record] = buildRecords([createFullyProjectedContent()], { + contentProjection: { + fields: [ + {source: 'image', as: 'heroImage'}, + {source: 'reading_time', as: 'readingMinutes'} + ] + } + }); + + expect(record).toMatchObject({ + heroImage: 'https://example.com/feature.jpg', + readingMinutes: 4 + }); + expect(record).not.toHaveProperty('image'); + expect(record).not.toHaveProperty('reading_time'); + }); + + it("reads Ghost's computed excerpt without deriving it from custom_excerpt", () => { + const [record] = buildRecords([createContent({custom_excerpt: 'Custom excerpt'})], { + contentProjection: {fields: ['excerpt', 'custom_excerpt']} + }); + + expect(record).toMatchObject({excerpt: null, custom_excerpt: 'Custom excerpt'}); + }); + + it('normalizes missing scalars to null and missing relations to empty arrays', () => { + const records = buildRecords([ + createContent({id: 'undefined-fields'}), + createContent({ + id: 'null-fields', + feature_image: null, + excerpt: null, + tags: null, + authors: null + }) + ]); + + expect( + records.map(({objectID, image, excerpt, tags, authors}) => ({ + objectID, + image, + excerpt, + tags, + authors + })) + ).toEqual([ + {objectID: 'undefined-fields_0', image: null, excerpt: null, tags: [], authors: []}, + {objectID: 'null-fields_0', image: null, excerpt: null, tags: [], authors: []} + ]); + }); + + it('preserves false, zero, and empty-string values', () => { + const [record] = buildRecords( + [createContent({featured: false, reading_time: 0, custom_excerpt: ''})], + {contentProjection: {fields: ['featured', 'reading_time', 'custom_excerpt']}} + ); + + expect(record).toMatchObject({featured: false, reading_time: 0, custom_excerpt: ''}); + }); + + it('repeats every enabled optional field in every record of the same content', () => { + const records = buildRecords([ + createFullyProjectedContent({ + html: '

One

First.

Two

Second.

' + }) + ]); + + expect(records).toHaveLength(2); + expect( + records.map(({image, tags, authors, excerpt}) => ({image, tags, authors, excerpt})) + ).toEqual([ + { + image: 'https://example.com/feature.jpg', + tags: [{name: 'Guide', slug: 'guide'}], + authors: [{name: 'Ada Lovelace', slug: 'ada'}], + excerpt: 'Ghost computed excerpt' + }, + { + image: 'https://example.com/feature.jpg', + tags: [{name: 'Guide', slug: 'guide'}], + authors: [{name: 'Ada Lovelace', slug: 'ada'}], + excerpt: 'Ghost computed excerpt' + } + ]); + }); +}); + +describe('createAlgoliaRecords policy validation', () => { + const rejections = [ + { + name: 'an unknown projection source', + options: {contentProjection: {fields: ['plaintext']}}, + reason: 'unknown-source', + path: 'contentProjection.fields[0]' + }, + { + name: 'a repeated projection source', + options: {contentProjection: {fields: ['tags', 'tags']}}, + reason: 'repeated-source', + path: 'contentProjection.fields[1]' + }, + { + name: 'a repeated output name inside the projection list', + options: { + contentProjection: { + fields: [ + {source: 'excerpt', as: 'blurb'}, + {source: 'custom_excerpt', as: 'blurb'} + ] + } + }, + reason: 'repeated-output', + path: 'contentProjection.fields[1].as' + }, + { + name: 'a repeated output name across projection and ranking fields', + options: { + contentProjection: { + fields: [{source: 'reading_time', as: 'minutes'}], + customRanking: [{source: 'featured', as: 'minutes'}] + } + }, + reason: 'repeated-output', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'an alias colliding with a protected record field', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'html'}]}}, + reason: 'protected-collision', + path: 'contentProjection.fields[0].as' + }, + { + name: 'an alias equal to the customRanking container', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'customRanking'}]}}, + reason: 'container-collision', + path: 'contentProjection.fields[0].as' + }, + { + name: 'a ranking alias equal to the customRanking container', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'featured', as: 'customRanking'}] + } + }, + reason: 'container-collision', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'an alias impersonating another canonical allowlist name', + options: {contentProjection: {fields: [{source: 'custom_excerpt', as: 'excerpt'}]}}, + reason: 'canonical-collision', + path: 'contentProjection.fields[0].as' + }, + { + name: 'a ranking alias impersonating another canonical allowlist name', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'featured', as: 'reading_time'}] + } + }, + reason: 'canonical-collision', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'heading inside customRanking', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'featured', as: 'heading'}] + } + }, + reason: 'protected-collision', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'position inside customRanking', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'reading_time', as: 'position'}] + } + }, + reason: 'protected-collision', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'an Algolia-reserved output name', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'distinctSeqId'}]}}, + reason: 'reserved-collision', + path: 'contentProjection.fields[0].as' + }, + { + name: 'an alias containing a dot', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'meta.excerpt'}]}}, + reason: 'invalid-alias', + path: 'contentProjection.fields[0].as' + }, + { + name: 'an alias with a leading underscore', + options: {contentProjection: {fields: [{source: 'tags', as: '_tags'}]}}, + reason: 'invalid-alias', + path: 'contentProjection.fields[0].as' + }, + { + name: 'an alias containing a wildcard', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'excerpt*'}]}}, + reason: 'invalid-alias', + path: 'contentProjection.fields[0].as' + }, + { + name: 'an alias that is an object path', + options: {contentProjection: {fields: [{source: 'excerpt', as: 'meta[0].text'}]}}, + reason: 'invalid-alias', + path: 'contentProjection.fields[0].as' + }, + { + name: 'a ranking alias that is not a record attribute name', + options: { + contentProjection: {fields: [], customRanking: [{source: 'featured', as: '_rank'}]} + }, + reason: 'invalid-alias', + path: 'contentProjection.customRanking[0].as' + }, + { + name: 'a ranking sibling without an alias', + options: {contentProjection: {fields: [], customRanking: ['featured']}}, + reason: 'invalid-shape', + path: 'contentProjection.customRanking[0]' + }, + { + name: 'a ranking source outside featured and reading_time', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'created_at', as: 'createdAt'}] + } + }, + reason: 'unknown-source', + path: 'contentProjection.customRanking[0]' + }, + { + name: 'a repeated ranking source', + options: { + contentProjection: { + fields: [], + customRanking: [ + {source: 'featured', as: 'isFeatured'}, + {source: 'featured', as: 'promoted'} + ] + } + }, + reason: 'repeated-source', + path: 'contentProjection.customRanking[1]' + }, + { + name: 'an unknown policy property', + options: {customRankings: []}, + reason: 'unknown-property', + path: 'options' + }, + { + name: 'an unknown contentProjection property', + options: {contentProjection: {fields: [], ranking: []}}, + reason: 'unknown-property', + path: 'contentProjection' + }, + { + name: 'an unknown projection field property', + options: { + contentProjection: {fields: [{source: 'excerpt', as: 'blurb', transform: 'upper'}]} + }, + reason: 'unknown-property', + path: 'contentProjection.fields[0]' + }, + { + name: 'an unknown ranking field property', + options: { + contentProjection: { + fields: [], + customRanking: [{source: 'featured', as: 'isFeatured', weight: 2}] + } + }, + reason: 'unknown-property', + path: 'contentProjection.customRanking[0]' + }, + { + name: 'options that are not an object', + options: 42, + reason: 'invalid-shape', + path: 'options' + }, + { + name: 'a non-array fields property', + options: {contentProjection: {fields: 'image'}}, + reason: 'invalid-shape', + path: 'contentProjection.fields' + }, + { + name: 'a missing fields property', + options: {contentProjection: {}}, + reason: 'invalid-shape', + path: 'contentProjection.fields' + }, + { + name: 'a null contentProjection', + options: {contentProjection: null}, + reason: 'invalid-shape', + path: 'contentProjection' + }, + { + name: 'an array contentProjection', + options: {contentProjection: []}, + reason: 'invalid-shape', + path: 'contentProjection' + }, + { + name: 'a non-array customRanking property', + options: {contentProjection: {fields: [], customRanking: 'featured'}}, + reason: 'invalid-shape', + path: 'contentProjection.customRanking' + }, + { + name: 'a projection field that is neither a name nor an object', + options: {contentProjection: {fields: [42]}}, + reason: 'invalid-shape', + path: 'contentProjection.fields[0]' + }, + { + name: 'a projection field alias that is not a string', + options: {contentProjection: {fields: [{source: 'excerpt', as: 42}]}}, + reason: 'invalid-shape', + path: 'contentProjection.fields[0]' + }, + { + name: 'a non-array ignoreSlugs property', + options: {ignoreSlugs: 'ignored'}, + reason: 'invalid-shape', + path: 'ignoreSlugs' + }, + { + name: 'a non-string ignoreSlugs entry', + options: {ignoreSlugs: ['ignored', 42]}, + reason: 'invalid-shape', + path: 'ignoreSlugs[1]' + } + ]; + + it.each(rejections)('rejects $name', ({options, reason, path: issuePath}) => { + const error = expectFragmenterError(() => buildRecords([createContent()], options)); + + expect(error.code).toBe('INVALID_POLICY'); + expect(describeIssues(error)).toEqual([`${reason} @ ${issuePath}`]); + expect(error.issues.every(issue => issue.kind === 'policy')).toBe(true); + }); + + it('accepts an alias equal to its own source name', () => { + const [record] = buildRecords([createFullyProjectedContent()], { + contentProjection: { + fields: [{source: 'featured', as: 'featured'}], + customRanking: [{source: 'reading_time', as: 'reading_time'}] + } + }); + + expect(record).toMatchObject({ + featured: true, + customRanking: {position: 0, heading: 100, reading_time: 4} + }); + }); + + it('reports every policy issue in declaration order', () => { + const error = expectFragmenterError(() => + buildRecords([createContent()], { + unexpected: true, + ignoreSlugs: [42], + contentProjection: { + fields: ['plaintext', {source: 'excerpt', as: 'html'}], + customRanking: [{source: 'featured', as: 'heading'}] + } + }) + ); + + expect(error.code).toBe('INVALID_POLICY'); + expect(error.issues.map(issue => issue.path)).toEqual([ + 'options', + 'ignoreSlugs[0]', + 'contentProjection.fields[0]', + 'contentProjection.fields[1].as', + 'contentProjection.customRanking[0].as' + ]); + }); + + it('summarizes at most five issue messages and counts the rest', () => { + const error = expectFragmenterError(() => + buildRecords([createContent()], { + contentProjection: { + fields: ['a', 'b', 'c', 'd', 'e', 'f'] + } + }) + ); + + expect(error.issues).toHaveLength(6); + expect(error.message.startsWith('INVALID_POLICY: 6 issues.')).toBe(true); + expect(error.message.endsWith('; and 1 more')).toBe(true); + }); + + it('names a single issue in the singular', () => { + const error = expectFragmenterError(() => + buildRecords([createContent()], {contentProjection: {fields: ['plaintext']}}) + ); + + expect(error.message).toBe( + 'INVALID_POLICY: 1 issue. contentProjection.fields[0]: "plaintext" is not an allowed projection source.' + ); + }); + + it('names the repeated source kind in its message', () => { + const projection = expectFragmenterError(() => + buildRecords([createContent()], {contentProjection: {fields: ['tags', 'tags']}}) + ); + const ranking = expectFragmenterError(() => + buildRecords([createContent()], { + contentProjection: { + fields: [], + customRanking: [ + {source: 'featured', as: 'isFeatured'}, + {source: 'featured', as: 'promoted'} + ] + } + }) + ); + + expect(projection.issues[0]?.message).toBe( + 'contentProjection.fields[1]: projection source "tags" is configured more than once.' + ); + expect(ranking.issues[0]?.message).toBe( + 'contentProjection.customRanking[1]: ranking source "featured" is configured more than once.' + ); + }); + + it('throws INVALID_POLICY before any content is inspected', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({id: 42, html: null})], { + contentProjection: {fields: ['plaintext']} + }) + ); + + expect(error.code).toBe('INVALID_POLICY'); + expect(error.issues.some(issue => issue.kind === 'content')).toBe(false); + }); + + it('validates the policy even for an empty batch', () => { + const error = expectFragmenterError(() => + buildRecords([], {contentProjection: {fields: ['plaintext']}}) + ); + + expect(error.code).toBe('INVALID_POLICY'); + }); + + it('returns an empty array for an empty batch', () => { + expect(buildRecords([])).toEqual([]); + }); + + it('freezes the reported issues', () => { + const error = expectFragmenterError(() => + buildRecords([], {contentProjection: {fields: ['plaintext']}}) + ); + + expect(Object.isFrozen(error.issues)).toBe(true); + expect(error.name).toBe('FragmenterError'); + }); +}); + +describe('createAlgoliaRecords ignored slugs', () => { + it('removes ignored content and preserves input order for the rest', () => { + const records = buildRecords( + [ + createContent({id: 'first', slug: 'first'}), + createContent({id: 'ignored', slug: 'ignored'}), + createContent({id: 'last', slug: 'last'}) + ], + {ignoreSlugs: ['ignored']} + ); + + expect(records.map(record => record.objectID)).toEqual(['first_0', 'last_0']); + }); + + it.each([ + {name: 'a missing slug', slug: undefined, reason: 'missing'}, + {name: 'a non-string slug', slug: 42, reason: 'wrong-type'}, + {name: 'an empty slug', slug: '', reason: 'wrong-type'} + ])('fails on $name even when the caller listed the content as ignored', ({slug, reason}) => { + const error = expectFragmenterError(() => + buildRecords([createContent({slug})], {ignoreSlugs: ['getting-started', '']}) + ); + + expect(error.code).toBe('INVALID_GHOST_CONTENT'); + expect(describeIssues(error)).toEqual([`${reason} @ ghostContent[0].slug`]); + }); + + it('does not validate other fields of an ignored item', () => { + const records = buildRecords( + [ + createContent({ + id: 42, + slug: 'ignored', + html: 42, + url: null, + reading_time: 'four', + tags: 'none' + }), + createContent({id: 'kept', slug: 'kept'}) + ], + {ignoreSlugs: ['ignored']} + ); + + expect(records.map(record => record.objectID)).toEqual(['kept_0']); + }); + + it('emits only the slug issue for an item whose slug is invalid', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({slug: 42, id: undefined, html: 42, excerpt: 42})]) + ); + + expect(describeIssues(error)).toEqual(['wrong-type @ ghostContent[0].slug']); + }); +}); + +describe('createAlgoliaRecords content validation', () => { + it('rejects missing and non-string required fields in canonical order', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({id: undefined, url: 42, title: [], html: null})]) + ); + + expect(error.code).toBe('INVALID_GHOST_CONTENT'); + expect(error.issues.map(issue => issue.path)).toEqual([ + 'ghostContent[0].id', + 'ghostContent[0].url', + 'ghostContent[0].title', + 'ghostContent[0].html' + ]); + expect(error.issues.map(issue => issue.message)).toEqual([ + 'ghostContent[0].id: required Ghost field is missing.', + 'ghostContent[0].url: expected string but received number.', + 'ghostContent[0].title: expected string but received array.', + 'ghostContent[0].html: required Ghost field is missing.' + ]); + }); + + it('accepts empty url, title, and html but rejects an empty id', () => { + const records = buildRecords([createContent({url: '', title: '', html: ''})]); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({url: '', title: '', html: ''}); + + const error = expectFragmenterError(() => buildRecords([createContent({id: ''})])); + expect(describeIssues(error)).toEqual(['wrong-type @ ghostContent[0].id']); + expect(error.issues[0]).toMatchObject({ + contentId: null, + message: 'ghostContent[0].id: expected a non-empty string but received an empty string.' + }); + }); + + it('rejects a present optional field of the wrong documented type', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({featured: 'yes', reading_time: '4', excerpt: 12})], { + contentProjection: {fields: ['featured', 'reading_time', 'excerpt']} + }) + ); + + expect( + contentIssuesOf(error).map(issue => ({path: issue.path, expected: issue.expected})) + ).toEqual([ + {path: 'ghostContent[0].featured', expected: 'boolean'}, + {path: 'ghostContent[0].reading_time', expected: 'number'}, + {path: 'ghostContent[0].excerpt', expected: 'string'} + ]); + }); + + it('treats an explicit null as missing rather than as a wrong type', () => { + const records = buildRecords( + [ + createContent({ + feature_image: null, + featured: null, + reading_time: null, + canonical_url: null + }) + ], + { + contentProjection: { + fields: ['image', 'featured', 'reading_time', 'canonical_url'] + } + } + ); + + expect(records[0]).toMatchObject({ + image: null, + featured: null, + reading_time: null, + canonical_url: null + }); + }); + + it('rejects a relation that is not an array', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({tags: {length: 1, forEach: () => undefined}})]) + ); + + expect(error.issues[0]).toMatchObject({ + reason: 'wrong-type', + path: 'ghostContent[0].tags', + expected: 'array' + }); + }); + + it.each([ + { + name: 'an element that is not an object', + tags: [null], + path: 'ghostContent[0].tags[0]', + expected: 'object' + }, + { + name: 'an element that is an array', + tags: [[]], + path: 'ghostContent[0].tags[0]', + expected: 'object' + }, + { + name: 'an element without a string name', + tags: [{name: null, slug: 'guide'}], + path: 'ghostContent[0].tags[0].name', + expected: 'string' + }, + { + name: 'an element without a string slug', + tags: [{name: 'Guide'}], + path: 'ghostContent[0].tags[0].slug', + expected: 'string' + } + ])('rejects a relation with $name', ({tags, path: issuePath, expected}) => { + const error = expectFragmenterError(() => buildRecords([createContent({tags})])); + + expect(error.issues[0]).toMatchObject({ + reason: 'wrong-type', + path: issuePath, + expected + }); + }); + + it('validates only enabled optional fields', () => { + const records = buildRecords([createContent({visibility: 42, reading_time: 'four'})], { + contentProjection: {fields: ['image']} + }); + + expect(records).toHaveLength(1); + expect(records[0]).not.toHaveProperty('visibility'); + }); + + it('reports a wrong-typed value once when it feeds both a projection field and a ranking sibling', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({featured: 'yes'})], { + contentProjection: { + fields: ['featured'], + customRanking: [{source: 'featured', as: 'isFeatured'}] + } + }) + ); + + expect(describeIssues(error)).toEqual(['wrong-type @ ghostContent[0].featured']); + }); + + it('reports content issues for every item in input order', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({id: 'first', slug: 'first', html: 42}), + createContent({id: 'second', slug: 'second', title: null}), + createContent({id: 'third', slug: 42}) + ]) + ); + + expect( + contentIssuesOf(error).map(issue => ({path: issue.path, contentId: issue.contentId})) + ).toEqual([ + {path: 'ghostContent[0].html', contentId: 'first'}, + {path: 'ghostContent[1].title', contentId: 'second'}, + {path: 'ghostContent[2].slug', contentId: 'third'} + ]); + }); + + it('rejects a non-array ghostContent argument', () => { + const error = expectFragmenterError(() => buildRecords({length: 0})); + + expect(error.code).toBe('INVALID_GHOST_CONTENT'); + expect(error.issues).toEqual([ + { + kind: 'content', + reason: 'invalid-shape', + path: 'ghostContent', + index: null, + contentId: null, + expected: 'array', + message: 'ghostContent: expected array.' + } + ]); + }); + + it('rejects a batch item that is not an object', () => { + const error = expectFragmenterError(() => buildRecords([createContent(), 'post'])); + + expect(error.issues).toEqual([ + { + kind: 'content', + reason: 'invalid-shape', + path: 'ghostContent[1]', + index: 1, + contentId: null, + expected: 'object', + message: 'ghostContent[1]: expected object.' + } + ]); + }); + + it('throws INVALID_GHOST_CONTENT when content and size problems coexist', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({ + id: 'oversized', + slug: 'oversized', + html: paragraphs('A'.repeat(11000)) + }), + createContent({id: 'invalid', slug: 'invalid', title: 42}) + ]) + ); + + expect(error.code).toBe('INVALID_GHOST_CONTENT'); + expect(describeIssues(error)).toEqual(['wrong-type @ ghostContent[1].title']); + }); + + it('returns no records when any item fails validation', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({id: 'valid', slug: 'valid'}), createContent({url: 42})]) + ); + + expect(error.issues).toHaveLength(1); + expect(error.issues.every(issue => issue.kind === 'content')).toBe(true); + }); +}); + +describe('createAlgoliaRecords ranking siblings', () => { + it("always emits heading and position from each record's first packed fragment", () => { + const records = buildRecords([ + createContent({ + html: '

One

First.

Two

Second.

' + }) + ]); + + expect(records.map(record => record.customRanking)).toEqual([ + {position: 0, heading: 80}, + {position: 1, heading: 70} + ]); + }); + + it('adds featured and reading_time siblings under their aliases', () => { + const [record] = buildRecords([createFullyProjectedContent()], { + contentProjection: { + fields: [], + customRanking: [ + {source: 'featured', as: 'isFeatured'}, + {source: 'reading_time', as: 'readingMinutes'} + ] + } + }); + + expect(record?.customRanking).toEqual({ + position: 0, + heading: 100, + isFeatured: true, + readingMinutes: 4 + }); + }); + + it('emits null for a missing ranking sibling value', () => { + const [record] = buildRecords([createContent()], { + contentProjection: { + fields: [], + customRanking: [{source: 'featured', as: 'isFeatured'}] + } + }); + + expect(record?.customRanking).toEqual({position: 0, heading: 100, isFeatured: null}); + }); + + it('repeats identical sibling values in every record of one content item', () => { + const records = buildRecords( + [ + createFullyProjectedContent({ + html: '

One

First.

Two

Second.

' + }) + ], + { + contentProjection: { + fields: [], + customRanking: [{source: 'reading_time', as: 'readingMinutes'}] + } + } + ); + + expect(records.map(record => record.customRanking)).toEqual([ + {position: 0, heading: 80, readingMinutes: 4}, + {position: 1, heading: 80, readingMinutes: 4} + ]); + }); + + it('uses the headingless rank for content with no headings', () => { + const [record] = buildRecords([createContent({html: paragraphs('Only text.')})]); + + expect(record?.customRanking).toEqual({position: 0, heading: 100}); + }); +}); + +describe('createAlgoliaRecords fallback record', () => { + it('emits one fallback record for content with no extraction fragments', () => { + const records = buildRecords([createContent({html: ''})]); + + expect(records).toEqual([ + { + objectID: 'post-1_0', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + html: '', + title: 'Getting started', + headings: [], + anchor: null, + image: null, + tags: [], + authors: [], + excerpt: null, + customRanking: {position: 0, heading: 100} + } + ]); + }); + + it('emits a fallback record for markup with no extractable text', () => { + const records = buildRecords([ + createContent({html: '
Only inline text.
'}) + ]); + + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({objectID: 'post-1_0', html: '', anchor: null}); + }); + + it('includes every enabled optional field and ranking sibling in the fallback record', () => { + const [record] = buildRecords([createFullyProjectedContent({html: ''})], { + contentProjection: { + fields: ['image', {source: 'reading_time', as: 'readingMinutes'}], + customRanking: [{source: 'featured', as: 'isFeatured'}] + } + }); + + expect(record).toEqual({ + objectID: 'post-1_0', + slug: 'getting-started', + url: 'https://example.com/getting-started/', + html: '', + title: 'Getting started', + headings: [], + anchor: null, + image: 'https://example.com/feature.jpg', + readingMinutes: 4, + customRanking: {position: 0, heading: 100, isFeatured: true} + }); + }); + + it('fails when a fallback record alone exceeds the ceiling', () => { + const error = expectFragmenterError(() => + buildRecords([createContent({html: '', title: 'Required title '.repeat(760)})]) + ); + + expect(error.code).toBe('RECORD_TOO_LARGE'); + expect(error.issues).toHaveLength(1); + expect(error.issues[0]).toMatchObject({ + kind: 'size', + reason: 'record-too-large', + path: 'ghostContent[0]', + index: 0, + contentId: 'post-1', + objectID: 'post-1_0', + anchor: null, + position: null, + limit: MAX_RECORD_BYTES + }); + expect(error.issues[0]?.message).toContain('fallback record needs'); + }); +}); + +describe('createAlgoliaRecords grouping, deep links, and ordering', () => { + it('emits one record per legacy anchor group with legacy identifiers', () => { + const records = buildRecords([ + createContent({ + html: '

Lead.

One

First.

Two

Second.

' + }) + ]); + + expect(records.map(record => ({objectID: record.objectID, anchor: record.anchor}))).toEqual( + [ + {objectID: 'post-1_0', anchor: null}, + {objectID: 'post-1_1', anchor: 'one'}, + {objectID: 'post-1_2', anchor: 'two'} + ] + ); + }); + + it('merges non-adjacent fragments that repeat an anchor', () => { + const records = buildRecords([ + createContent({ + html: [ + '

Setup

First setup.

', + '

Overview

Overview.

', + '

Setup again

Later setup.

' + ].join('') + }) + ]); + + expect(records.map(({objectID, html, headings}) => ({objectID, html, headings}))).toEqual([ + { + objectID: 'post-1_0', + html: '

First setup.

Later setup.

', + headings: ['Setup'] + }, + {objectID: 'post-1_1', html: '

Overview.

', headings: ['Overview']} + ]); + }); + + it("keeps preformatted markup for a record's first packed fragment and merges later ones as text", () => { + const records = buildRecords([ + createContent({ + html: [ + '

Code

first

After.

', + '

Prose

Run:

later
' + ].join('') + }) + ]); + + expect(records.map(record => record.html)).toEqual([ + '
first

After.

', + '

Run:

later' + ]); + }); + + it('links each record to its anchor and falls back to the base URL', () => { + const records = buildRecords([ + createContent({html: '

Lead.

One

First.

'}) + ]); + + expect(records.map(record => record.url)).toEqual([ + 'https://example.com/getting-started/', + 'https://example.com/getting-started/#one' + ]); + }); + + it('returns records in content order, then group order, then continuation order', () => { + const longParagraph = 'A'.repeat(6000); + const records = buildRecords([ + createContent({ + id: 'first', + slug: 'first', + html: [ + `

A

${paragraphs(longParagraph, longParagraph)}`, + '

B

Short.

' + ].join('') + }), + createContent({id: 'second', slug: 'second', html: '

Second content.

'}) + ]); + + expect(records.map(record => record.objectID)).toEqual([ + 'first_0', + 'first_0_1', + 'first_1', + 'second_0' + ]); + }); +}); + +describe('createAlgoliaRecords record size', () => { + it('keeps every complete record within 9,999 UTF-8 bytes', () => { + const records = buildRecords([ + createFullyProjectedContent({html: readFixture('massive-example')}) + ]); + + expect(records.length).toBeGreaterThan(1); + records.forEach(record => { + expect(bytesOf(record)).toBeLessThanOrEqual(MAX_RECORD_BYTES); + }); + }); + + it('packs whole fragments greedily into continuation records', () => { + const records = buildRecords([ + createContent({ + html: paragraphs( + 'A'.repeat(3900), + 'B'.repeat(3900), + 'C'.repeat(3900), + 'D'.repeat(3900) + ) + }) + ]); + + expect(records).toHaveLength(2); + expect(records.map(record => record.html)).toEqual([ + paragraphs('A'.repeat(3900), 'B'.repeat(3900)), + paragraphs('C'.repeat(3900), 'D'.repeat(3900)) + ]); + records.forEach(record => { + expect(bytesOf(record)).toBeLessThanOrEqual(MAX_RECORD_BYTES); + }); + }); + + it('numbers continuations deterministically', () => { + const records = buildRecords([ + createContent({ + html: paragraphs('A'.repeat(9000), 'B'.repeat(9000), 'C'.repeat(9000)) + }) + ]); + + expect(records.map(record => record.objectID)).toEqual([ + 'post-1_0', + 'post-1_0_1', + 'post-1_0_2' + ]); + }); + + it("uses each record's first packed fragment for position and heading rank", () => { + const records = buildRecords([ + createContent({ + html: `

A

${paragraphs('A'.repeat(9000), 'B'.repeat(9000))}` + }) + ]); + + expect(records.map(record => record.customRanking)).toEqual([ + {position: 0, heading: 80}, + {position: 1, heading: 80} + ]); + }); + + it('repeats projection, anchor and URL in every continuation', () => { + const records = buildRecords([ + createFullyProjectedContent({ + html: `

A

${paragraphs('A'.repeat(9000), 'B'.repeat(9000))}` + }) + ]); + + const shared = records.map(({url, anchor, image, excerpt}) => ({ + url, + anchor, + image, + excerpt + })); + expect(shared[0]).toEqual({ + url: 'https://example.com/getting-started/#a', + anchor: 'a', + image: 'https://example.com/feature.jpg', + excerpt: 'Ghost computed excerpt' + }); + expect(shared[1]).toEqual(shared[0]); + }); + + it('describes each continuation with its own heading context', () => { + const records = buildRecords([ + createFullyProjectedContent({ + html: [ + `

A

${'A'.repeat(9000)}

`, + `

Sub

${'B'.repeat(9000)}

` + ].join('') + }) + ]); + + expect( + records.map(({objectID, headings, anchor, url, customRanking, image}) => ({ + objectID, + headings, + anchor, + url, + customRanking, + image + })) + ).toEqual([ + { + objectID: 'post-1_0', + headings: ['A'], + anchor: 'a', + url: 'https://example.com/getting-started/#a', + customRanking: {position: 0, heading: 80}, + image: 'https://example.com/feature.jpg' + }, + { + objectID: 'post-1_0_1', + headings: ['A', 'Sub'], + anchor: 'a', + url: 'https://example.com/getting-started/#a', + customRanking: {position: 1, heading: 70}, + image: 'https://example.com/feature.jpg' + } + ]); + }); + + it('measures multi-byte characters as UTF-8 bytes', () => { + const emojiParagraph = '👻'.repeat(1000); + const records = buildRecords([ + createContent({html: paragraphs(emojiParagraph, emojiParagraph, emojiParagraph)}) + ]); + + const mergedHtml = records.map(record => String(record.html)).join(''); + expect(records).toHaveLength(2); + expect(mergedHtml.length).toBeLessThan(MAX_RECORD_BYTES); + expect(Buffer.byteLength(mergedHtml, 'utf8')).toBeGreaterThan(MAX_RECORD_BYTES); + }); + + it('measures JSON escaping', () => { + const escapedParagraph = '"\\'.repeat(1300); + const records = buildRecords([ + createContent({html: paragraphs(escapedParagraph, escapedParagraph)}) + ]); + + const mergedHtml = records.map(record => String(record.html)).join(''); + expect(records).toHaveLength(2); + expect(Buffer.byteLength(mergedHtml, 'utf8')).toBeLessThan(MAX_RECORD_BYTES); + records.forEach(record => { + expect(bytesOf(record)).toBeLessThanOrEqual(MAX_RECORD_BYTES); + }); + }); + + it('counts repeated projected metadata toward every record', () => { + const content = createContent({ + excerpt: 'E'.repeat(3000), + html: paragraphs('A'.repeat(3400), 'B'.repeat(3400)) + }); + + expect(buildRecords([content], {contentProjection: {fields: []}})).toHaveLength(1); + expect(buildRecords([content], {contentProjection: {fields: ['excerpt']}})).toHaveLength(2); + }); + + it('accepts a record of exactly 9,999 bytes and splits at 10,000', () => { + const [smallest] = buildRecords([createContent({html: '

a

'})]); + const fillLength = MAX_RECORD_BYTES - (bytesOf(smallest) - 1); + const exactHtml = `

${'a'.repeat(fillLength)}

`; + + const exact = buildRecords([createContent({html: exactHtml})]); + expect(exact).toHaveLength(1); + expect(bytesOf(exact[0])).toBe(MAX_RECORD_BYTES); + + const split = buildRecords([createContent({html: `${exactHtml}

b

`})]); + expect(split.map(record => record.objectID)).toEqual(['post-1_0', 'post-1_0_1']); + expect(bytesOf(split[0])).toBe(MAX_RECORD_BYTES); + + const error = expectFragmenterError(() => + buildRecords([createContent({html: `

${'a'.repeat(fillLength + 1)}

`})]) + ); + expect(error.issues[0]).toMatchObject({bytes: MAX_RECORD_BYTES + 1, excess: 1}); + }); + + it('fails on an indivisible fragment with actionable size context', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({ + html: `

Appendix

${paragraphs('G'.repeat(11000))}` + }) + ]) + ); + + expect(error.code).toBe('RECORD_TOO_LARGE'); + expect(error.issues).toHaveLength(1); + + const [issue] = sizeIssuesOf(error); + expect(issue).toMatchObject({ + kind: 'size', + reason: 'record-too-large', + path: 'ghostContent[0]', + index: 0, + contentId: 'post-1', + objectID: 'post-1_0', + anchor: 'appendix', + position: 0, + limit: MAX_RECORD_BYTES + }); + expect(issue?.excess).toBe((issue?.bytes ?? 0) - MAX_RECORD_BYTES); + expect(issue?.message).toContain('fragment at source position 0'); + }); + + it('fails when required metadata leaves no room for the smallest fragment', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({ + title: 'Required title '.repeat(760), + html: '

Details

Small paragraph.

' + }) + ]) + ); + + expect(error.code).toBe('RECORD_TOO_LARGE'); + expect(sizeIssuesOf(error)).toHaveLength(1); + expect(sizeIssuesOf(error)[0]).toMatchObject({ + path: 'ghostContent[0]', + objectID: 'post-1_0', + anchor: 'details', + position: 0 + }); + expect(sizeIssuesOf(error)[0]?.message).toContain('fragment at source position 0'); + }); + + it('reports every size issue in input order', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({ + id: 'first', + slug: 'first', + html: paragraphs('Short.', 'A'.repeat(11000)) + }), + createContent({id: 'second', slug: 'second', html: paragraphs('B'.repeat(11000))}) + ]) + ); + + expect(error.code).toBe('RECORD_TOO_LARGE'); + expect(sizeIssuesOf(error).map(issue => `${issue.contentId}:${issue.objectID}`)).toEqual([ + 'first:first_0_1', + 'second:second_0' + ]); + }); + + it('returns no records when any record is too large', () => { + const error = expectFragmenterError(() => + buildRecords([ + createContent({id: 'fine', slug: 'fine'}), + createContent({ + id: 'oversized', + slug: 'oversized', + html: paragraphs('A'.repeat(11000)) + }) + ]) + ); + + expect(error.code).toBe('RECORD_TOO_LARGE'); + expect(error.issues).toHaveLength(1); + }); +}); + +describe('deprecated wrapper non-regression', () => { + it('fragmentTransformer emits one oversized record without packing', () => { + const post = { + id: 'oversized', + slug: 'oversized', + url: 'https://example.com/oversized/', + html: paragraphs('A'.repeat(6000), 'B'.repeat(6000)), + feature_image: null, + title: 'Oversized', + tags: [], + authors: [] + }; + + const records = transformToAlgoliaObject([post]).reduce(fragmentTransformer, []); + + expect(records.map(record => record.objectID)).toEqual(['oversized_0']); + expect(bytesOf(records[0])).toBeGreaterThan(MAX_RECORD_BYTES); + }); + + it('transformToAlgoliaObject keeps rejecting a relation collection without forEach', () => { + expect(() => + transformToAlgoliaObject([ + { + id: 'legacy', + slug: 'legacy', + url: 'https://example.com/legacy/', + html: '

Legacy.

', + feature_image: null, + title: 'Legacy', + tags: {length: 1}, + authors: [] + } + ]) + ).toThrow(new TypeError('post.tags.forEach is not a function')); + }); + + it('transformToAlgoliaObject ignores projection-only Ghost fields', () => { + const [record] = transformToAlgoliaObject([ + { + id: 'legacy', + slug: 'legacy', + url: 'https://example.com/legacy/', + html: '

Legacy.

', + feature_image: null, + title: 'Legacy', + tags: [], + authors: [], + excerpt: 'not indexed', + featured: true, + reading_time: 4 + } + ]); + + expect(record).not.toHaveProperty('excerpt'); + expect(record).not.toHaveProperty('featured'); + expect(record).not.toHaveProperty('reading_time'); + }); +}); diff --git a/packages/algolia-fragmenter/test/package.acceptance.test.mts b/packages/algolia-fragmenter/test/package.acceptance.test.mts index ee781c6e..bafc574a 100644 --- a/packages/algolia-fragmenter/test/package.acceptance.test.mts +++ b/packages/algolia-fragmenter/test/package.acceptance.test.mts @@ -98,10 +98,34 @@ describe('@tryghost/algolia-fragmenter packed artifact', () => { expect(packedFragmenter.files.map(file => file.path).sort()).toEqual([ 'LICENSE', 'README.md', + 'lib/create-algolia-records.d.mts', + 'lib/create-algolia-records.d.mts.map', + 'lib/create-algolia-records.mjs', + 'lib/create-algolia-records.mjs.map', + 'lib/errors.d.mts', + 'lib/errors.d.mts.map', + 'lib/errors.mjs', + 'lib/errors.mjs.map', + 'lib/grouping.d.mts', + 'lib/grouping.d.mts.map', + 'lib/grouping.mjs', + 'lib/grouping.mjs.map', 'lib/index.d.mts', 'lib/index.d.mts.map', 'lib/index.mjs', 'lib/index.mjs.map', + 'lib/policy.d.mts', + 'lib/policy.d.mts.map', + 'lib/policy.mjs', + 'lib/policy.mjs.map', + 'lib/projection.d.mts', + 'lib/projection.d.mts.map', + 'lib/projection.mjs', + 'lib/projection.mjs.map', + 'lib/records.d.mts', + 'lib/records.d.mts.map', + 'lib/records.mjs', + 'lib/records.mjs.map', 'package.json' ]); @@ -122,16 +146,41 @@ describe('@tryghost/algolia-fragmenter packed artifact', () => { import * as fragmenter from '@tryghost/algolia-fragmenter'; import transforms from '@tryghost/algolia-fragmenter'; import { + createAlgoliaRecords, + FragmenterError, fragmentTransformer, transformToAlgoliaObject } from '@tryghost/algolia-fragmenter'; const records = transformToAlgoliaObject([${JSON.stringify(runtimeInput)}]) .reduce(fragmentTransformer, []); + const deepRecords = createAlgoliaRecords([${JSON.stringify(runtimeInput)}], { + contentProjection: { + fields: ['image'], + customRanking: [{source: 'featured', as: 'isFeatured'}] + } + }); + let policyFailure = null; + try { + createAlgoliaRecords([${JSON.stringify(runtimeInput)}], { + contentProjection: {fields: ['plaintext']} + }); + } catch (error) { + policyFailure = { + name: error.name, + code: error.code, + issueCount: error.issues.length, + firstIssueKind: error.issues[0].kind, + isFragmenterError: error instanceof FragmenterError + }; + } console.log(JSON.stringify({ exports: Object.keys(fragmenter).sort(), defaultExports: Object.keys(transforms).sort(), records, - synchronous: !(records instanceof Promise) + synchronous: !(records instanceof Promise), + deepRecords, + deepSynchronous: !(deepRecords instanceof Promise), + policyFailure })); ` ); @@ -155,10 +204,37 @@ describe('@tryghost/algolia-fragmenter packed artifact', () => { } ]; expect(JSON.parse(esmResult.stdout)).toEqual({ - exports: ['default', 'fragmentTransformer', 'transformToAlgoliaObject'], + exports: [ + 'FragmenterError', + 'createAlgoliaRecords', + 'default', + 'fragmentTransformer', + 'transformToAlgoliaObject' + ], defaultExports: ['fragmentTransformer', 'transformToAlgoliaObject'], records: expectedRecords, - synchronous: true + synchronous: true, + deepRecords: [ + { + objectID: 'packed_0', + slug: 'packed', + url: 'https://fixture.invalid/packed/#packed-heading', + html: '

Ready.

', + title: 'Packed consumer', + headings: ['Packed'], + anchor: 'packed-heading', + image: null, + customRanking: {position: 0, heading: 80, isFeatured: null} + } + ], + deepSynchronous: true, + policyFailure: { + name: 'FragmenterError', + code: 'INVALID_POLICY', + issueCount: 1, + firstIssueKind: 'policy', + isFragmenterError: true + } }); const commonJsConsumer = path.join(temporaryDirectory, 'consumer.cjs'); @@ -176,12 +252,45 @@ describe('@tryghost/algolia-fragmenter packed artifact', () => { ` import fragmenter from '@tryghost/algolia-fragmenter'; import { + createAlgoliaRecords, + FragmenterError, fragmentTransformer, - transformToAlgoliaObject + transformToAlgoliaObject, + type AlgoliaRecord, + type ContentProjection, + type CreateAlgoliaRecordsOptions, + type ProjectionField, + type RankingField } from '@tryghost/algolia-fragmenter'; const transformed = transformToAlgoliaObject([${JSON.stringify(runtimeInput)}]); transformed.reduce(fragmentTransformer, []); fragmenter.transformToAlgoliaObject([]).reduce(fragmenter.fragmentTransformer, []); + + const fields: readonly ProjectionField[] = [ + 'image', + {source: 'reading_time', as: 'readingMinutes'} + ]; + const customRanking: readonly RankingField[] = [ + {source: 'featured', as: 'isFeatured'} + ]; + const contentProjection: ContentProjection = {fields, customRanking}; + const options: CreateAlgoliaRecordsOptions = { + ignoreSlugs: ['ignored'], + contentProjection + }; + let reported = ''; + try { + const deepRecords: readonly AlgoliaRecord[] = createAlgoliaRecords( + [${JSON.stringify(runtimeInput)}], + options + ); + reported = String(deepRecords.length); + } catch (error) { + if (error instanceof FragmenterError) { + reported = error.code + error.issues.map(issue => issue.kind).join(); + } + } + export default reported; ` ); const tsc = path.join(packageDirectory, 'node_modules/.bin/tsc'); @@ -205,12 +314,28 @@ describe('@tryghost/algolia-fragmenter packed artifact', () => { sources: string[]; sourcesContent?: string[]; }; - for (const mapName of ['index.mjs.map', 'index.d.mts.map']) { - const sourceMap = JSON.parse( - await readFile(path.join(installedPackage, 'lib', mapName), 'utf8') - ) as SourceMap; - expect(sourceMap.sources).toEqual(['../src/index.mts']); - expect(sourceMap.sourcesContent?.join('\n') ?? '').not.toContain(workspaceDirectory); + const emittedModules = [ + 'create-algolia-records', + 'errors', + 'grouping', + 'index', + 'policy', + 'projection', + 'records' + ]; + for (const moduleName of emittedModules) { + for (const extension of ['mjs.map', 'd.mts.map']) { + const sourceMap = JSON.parse( + await readFile( + path.join(installedPackage, 'lib', `${moduleName}.${extension}`), + 'utf8' + ) + ) as SourceMap; + expect(sourceMap.sources).toEqual([`../src/${moduleName}.mts`]); + expect(sourceMap.sourcesContent?.join('\n') ?? '').not.toContain( + workspaceDirectory + ); + } } const declarations = await readFile(path.join(installedPackage, 'lib/index.d.mts'), 'utf8'); diff --git a/vitest.config.mjs b/vitest.config.mjs index 7a516444..212979b5 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -16,7 +16,7 @@ export default defineConfig({ 'packages/*/index.js', 'packages/*/lib/**/*.js', 'packages/algolia/bin/**/*.js', - 'packages/algolia-fragmenter/src/index.mts', + 'packages/algolia-fragmenter/src/**/*.mts', 'packages/algolia-html-extractor/index.mts', 'packages/algolia-html-extractor/smoke/live-ghost-content-smoke.mts', 'packages/algolia-netlify/functions/**/*.{ts,mts}'