diff --git a/README.md b/README.md index 9cda3c715..c15fc2f5e 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,10 @@ the plugin's asset list; the site build gate requires the page to have a title. The root JavaScript dependency authority is `package.json` plus Yarn Classic `yarn.lock`. Do not add a root `package-lock.json`. The site build gate is the published `@netdata/site-build-gate` -package, so its parser dependencies are part of the root Yarn graph. Dependabot watches that graph and -GitHub Actions; owner-controlled vendor packages are intentionally excluded from automatic updates. +package, so its parser dependencies are part of the root Yarn graph. The IndexNow build plugin is +the published `@netdata/netlify-plugin-indexnow` package, owned by `netdata/website`; its releases +and its dependencies arrive as ordinary root Yarn updates. Dependabot watches that graph and GitHub +Actions. Root Yarn resolutions are only used for security releases that are compatible with every selected parent range. Do not use a resolution to bypass an incompatible upstream dependency constraint. diff --git a/netlify.toml b/netlify.toml index dc2923020..7c7a1ba8e 100644 --- a/netlify.toml +++ b/netlify.toml @@ -7,7 +7,7 @@ environment = { NPM_VERSION = "10.9.9", NODE_VERSION = "22.23.3", NODE_OPTIONS = "--max-old-space-size=3072", DOCUSAURUS_SSR_CONCURRENCY = "4"} [[plugins]] - package = "./plugins/netlify-plugin-indexnow" + package = "@netdata/netlify-plugin-indexnow" [plugins.inputs] host = "learn.netdata.cloud" diff --git a/package.json b/package.json index 1464f608c..ac4607bba 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", - "build:netlify": "node scripts/verify-git-history.js && node scripts/verify-indexnow-vendor.js && docusaurus build && node scripts/run-post-build-gates.mjs", + "build:netlify": "node scripts/verify-git-history.js && docusaurus build && node scripts/run-post-build-gates.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -47,7 +47,6 @@ "@docusaurus/theme-mermaid": "^3.10.2", "@easyops-cn/docusaurus-search-local": "0.55.3", "@mdx-js/react": "^3.1.1", - "@netlify/blobs": "11.1.4", "@tailwindcss/typography": "^0.5.20", "@types/react": "^19.3.0", "acorn": "^8.19.0", @@ -67,12 +66,12 @@ "react-markdown": "^10.1.0", "react-player": "^3.4.0", "remark-gfm": "^4.0.1", - "saxes": "6.0.0", "typescript": "^7.0.2", "webpack": "^5.111.1" }, "devDependencies": { "@docusaurus/faster": "3.10.2", + "@netdata/netlify-plugin-indexnow": "^1.0.0", "@netdata/site-build-gate": "^10.1.0", "@tailwindcss/postcss": "^4.3.3", "@testing-library/dom": "^10.4.2", diff --git a/plugins/netlify-plugin-indexnow/README.md b/plugins/netlify-plugin-indexnow/README.md deleted file mode 100644 index efc7fb2e3..000000000 --- a/plugins/netlify-plugin-indexnow/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Vendored website IndexNow build plugin - -`netdata/website` owns this shared IndexNow implementation. Learn vendors the checksum-covered -schema-2 bytes from website commit `5670562137379be4cfeac2538297d13ad503138b` and adapts only -the host and published-route fixtures outside those covered files. -The thin Netlify entry adapter, core implementation, manifest, dependency declaration, contract -fixtures, and receipt schema are the six-file byte-identical vendor contract. Their SHA-256 values -and exact dependency versions are recorded in -`vendor-checksums.json` and checked before every Netlify Learn build. - -`index.js` is only the Netlify lifecycle interface and exports exactly `onPreBuild` and -`onSuccess`. Netlify validates every enumerable entry as an event, so implementation helpers must -remain in `core.js`; tests import that module directly. The shared contract fixture pins the two -allowed entry events and the owner suite rejects any extra export. - -The plugin restores its prior HTML-hash state before the build and runs only after a successful -production deploy. It parses direct `url`/`loc` entries from a well-formed published sitemap, -supports Hugo trailing-slash and Docusaurus non-trailing-slash routes, hashes the corresponding -HTML files, and classifies added, updated, and removed URLs. A removed URL is submitted so -IndexNow can recrawl its redirect, 404, or 410 response. Sitemap comments and extension elements -cannot create entries; namespace prefixes, CDATA, and XML named or numeric entities are handled. -The root, direct `url`, and direct `loc` must all use the sitemap root namespace. Missing, reset, -or mixed namespaces and malformed XML fail closed before state or submission changes. - -A missing, empty, invalid, legacy, non-canonical, or cross-host cache seeds the current state -without bulk submission. The public key uses the protocol's letters, digits, and dashes grammar; -this shared implementation deliberately supports only the root `/.txt` authorization used -by both sites. Before reading state or making a request it proves that the built root key artifact -is a regular non-symlink file whose bytes equal the key exactly, with no newline. Sitemap, state, -and payload URLs are serialized to one canonical HTTPS identity. -URL paths use one strict decoder for sitemap acceptance and built-file mapping. Repeated -separators, raw or encoded dot segments or separators, invalid UTF-8, controls, and NUL are -rejected. Raw empty query or fragment delimiters are rejected too; URL parsers otherwise erase -that ambiguity. A URL must resolve to exactly one regular, non-symlink output file, and two URLs -cannot claim the same HTML file. Ambiguous directory-index and direct-HTML candidates fail the build. -Canonical state and receipt objects use JavaScript code-unit key order, independent of locale or -timezone. - -Independent request batches continue after a failure. Each request has one bounded deadline that -covers the HTTP request and response body. HTTP 200 and 202 advance state; timeouts, transport -errors, and every other status remain pending for a later deploy and the weekly SEO safety-net -submission. Before state advances, the plugin writes one deterministic, content-addressed receipt -to the deploy-specific `netdata-indexnow-receipts` Netlify Blob store. It records exact submitted -URL identities and each batch's accepted or pending status without the public key or key location. -Only the deploy ID, digest, counts, and response classes are logged. The SEO archive collector can -enumerate production deploys and retrieve the receipt through Netlify's supported SDK; build logs -and the local cache are not the audit record. -Only integer HTTP statuses from 100 through 599 enter a receipt; every other response status is -recorded as a schema-valid invalid response with a null status. `COMMIT_REF` is either null or an -exact non-empty, already-trimmed string, matching the collector's provider identity boundary. - -State-cache persistence is transactional: the candidate file is written atomically, cache saving must -return `true`, and a false result or exception restores the exact prior local state (or removes a -new seed). The plugin never claims accepted state was cached before that succeeds. IndexNow, -receipt, and cache failures are logged but never invalidate a successful site deployment. The -external submission and local evidence cannot be one atomic transaction, so delivery is at least -once: a crash after an accepted response but before durable receipt/cache state can resend the -same notification, which is safer than losing it. - -The cache state machine assumes production hooks for one site do not overlap against the same -restored cache workspace. If Netlify overlaps them, both hooks may submit the same change; the -deploy-specific receipts still preserve exact evidence and the later cache state is only an -optimization. Preventing even that permitted at-least-once duplicate would require an external -compare-and-swap state owner. - -Learn copies every file covered by `vendor-checksums.json`, uses -`scripts/verify-indexnow-vendor.js` in `build:netlify`, and runs the complete owner adversarial -suite in `tests/indexnow_plugin.test.js`. Do not copy stale checksums from prose; -`vendor-checksums.json` is the machine-checked source of truth. diff --git a/plugins/netlify-plugin-indexnow/contract-tests.json b/plugins/netlify-plugin-indexnow/contract-tests.json deleted file mode 100644 index 32947eee7..000000000 --- a/plugins/netlify-plugin-indexnow/contract-tests.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "schema_version": 2, - "plugin_entry_events": [ - "onPreBuild", - "onSuccess" - ], - "invalid_keys": [ - "short", - "unsafe/key", - "unsafe_key" - ], - "malformed_sitemaps": [ - "https://www.netdata.cloud/", - "https://www.netdata.cloud/", - "https://www.netdata.cloud/" - ], - "invalid_paths": [ - "/one//", - "/./one/", - "/one/../two/", - "/%2E/one/", - "/%2E%2E/one/", - "/one%2Ftwo/", - "/one%5Ctwo/", - "/one\\two/", - "/bad%C3%28/", - "/bad%00path/", - "/bad%1Fpath/", - "/raw\"quote/", - "/rawgreater/", - "/raw`tick/", - "/raw{open/", - "/raw}close/", - "/one/?", - "/one/#" - ], - "accepted_http_statuses": [ - 200, - 202 - ], - "receipt_schema_version": 2, - "maximum_batch_size": 10000 -} diff --git a/plugins/netlify-plugin-indexnow/core.js b/plugins/netlify-plugin-indexnow/core.js deleted file mode 100644 index 551629bc0..000000000 --- a/plugins/netlify-plugin-indexnow/core.js +++ /dev/null @@ -1,773 +0,0 @@ -const crypto = require('node:crypto'); -const fs = require('node:fs/promises'); -const path = require('node:path'); -const {getDeployStore} = require('@netlify/blobs'); -const {SaxesParser} = require('saxes'); - -const ENDPOINT = 'https://api.indexnow.org/indexnow'; -const MAX_URLS_PER_REQUEST = 10_000; -const MAX_URL_LENGTH = 2_048; -const MAX_RECEIPT_BYTES = 24 * 1024 * 1024; -const REQUEST_TIMEOUT_MS = 15_000; -const STATE_PATH = path.resolve('.netlify/indexnow-state.json'); -const STATE_VERSION = 2; -const RECEIPT_SCHEMA_VERSION = 2; -const RECEIPT_STORE = 'netdata-indexnow-receipts'; -const SITEMAP_NAMESPACE = 'http://www.sitemaps.org/schemas/sitemap/0.9'; - -function codeUnitCompare(left, right) { - return left < right ? -1 : left > right ? 1 : 0; -} - -function validateInputs(inputs) { - if (!inputs || typeof inputs !== 'object') { - throw new Error('IndexNow plugin inputs are required'); - } - - const host = inputs.host; - if (typeof host !== 'string' || host !== host.trim().toLowerCase()) { - throw new Error('IndexNow host must be a lowercase hostname'); - } - let hostUrl; - try { - hostUrl = new URL(`https://${host}/`); - } catch { - throw new Error('IndexNow host must be a valid hostname'); - } - if ( - hostUrl.hostname !== host || - hostUrl.host !== host || - hostUrl.username || - hostUrl.password - ) { - throw new Error('IndexNow host must not contain a scheme, credentials, port, or path'); - } - - const key = inputs.key; - if (typeof key !== 'string' || !/^[A-Za-z0-9-]{8,128}$/.test(key)) { - throw new Error('IndexNow key must contain 8-128 letters, digits, or dashes'); - } - - let keyLocation; - try { - keyLocation = new URL(inputs.keyLocation); - } catch { - throw new Error('IndexNow keyLocation must be a valid URL'); - } - const expectedKeyLocation = `https://${host}/${key}.txt`; - if (keyLocation.href !== expectedKeyLocation || inputs.keyLocation !== expectedKeyLocation) { - throw new Error('IndexNow keyLocation must be the exact HTTPS root key file on the submitted host'); - } - - return {host, key, keyLocation: keyLocation.href}; -} - -function parseSitemapLocations(xml) { - if (typeof xml !== 'string') throw new Error('Malformed sitemap XML: input is not text'); - const locations = []; - const stack = []; - let rootSeen = false; - const fail = (message) => { throw new Error(`Malformed sitemap XML: ${message}`); }; - const parser = new SaxesParser({xmlns: true}); - - parser.on('doctype', () => fail('DOCTYPE is not allowed')); - parser.on('error', (error) => fail(error.message)); - parser.on('opentag', (node) => { - const parent = stack.at(-1); - if (!parent) { - if (rootSeen || node.local !== 'urlset' || node.uri !== SITEMAP_NAMESPACE) { - fail('expected one sitemap urlset root element'); - } - rootSeen = true; - } else if (parent.captureLocation) { - fail('direct loc contains a child element'); - } else if (stack.length === 1 && - (node.local !== 'url' || node.uri !== SITEMAP_NAMESPACE)) { - fail('urlset contains a non-sitemap URL child element'); - } else if (stack.length === 2 && parent.directUrl && - node.local === 'loc' && node.uri !== SITEMAP_NAMESPACE) { - fail('direct loc is not in the sitemap root namespace'); - } - - const directUrl = stack.length === 1 && node.local === 'url' && node.uri === SITEMAP_NAMESPACE; - const captureLocation = stack.length === 2 && parent.directUrl && - node.local === 'loc' && node.uri === SITEMAP_NAMESPACE; - stack.push({ - name: node.name, - localName: node.local, - directUrl, - captureLocation, - location: directUrl ? null : undefined, - text: captureLocation ? '' : undefined, - }); - }); - const consumeText = (text) => { - const current = stack.at(-1); - if (current?.captureLocation) current.text += text; - else if (!stack.length && text.trim()) fail('text outside the root element'); - }; - parser.on('text', consumeText); - parser.on('cdata', consumeText); - parser.on('closetag', (node) => { - const current = stack.pop(); - if (!current || current.name !== node.name) fail(`mismatched closing tag ${node.name}`); - if (current.captureLocation) { - const parent = stack.at(-1); - if (parent.location !== null) fail('URL contains multiple direct loc elements'); - parent.location = current.text.trim(); - } - if (current.directUrl) { - if (!current.location) fail('direct URL is missing a loc value'); - locations.push(current.location); - } - }); - - try { - parser.write(xml).close(); - } catch (error) { - if (error.message.startsWith('Malformed sitemap XML:')) throw error; - fail(error.message); - } - if (!rootSeen || stack.length) fail('incomplete document'); - return locations; -} - -function strictPublicUrl(value, host) { - try { - if (typeof value !== 'string') return null; - const origin = `https://${host}`; - if (!value.startsWith(`${origin}/`)) return null; - const rawPathname = value.slice(origin.length); - if ( - !rawPathname.startsWith('/') || - rawPathname.includes('?') || - rawPathname.includes('#') || - /\\/.test(rawPathname) || - /\/{2,}/.test(rawPathname) || - /%(?![0-9a-f]{2})/i.test(rawPathname) || - /%(?:2f|5c)/i.test(rawPathname) - ) { - return null; - } - const decodedPathname = decodeURIComponent(rawPathname); - if ( - /[\u0000-\u001f\u007f-\u009f]/u.test(decodedPathname) || - /\\/.test(decodedPathname) || - /\/{2,}/.test(decodedPathname) || - decodedPathname.split('/').some((segment) => segment === '.' || segment === '..') - ) { - return null; - } - const url = new URL(value); - const pathname = url.pathname; - const normalizedPathname = pathname.replace(/%[0-9a-f]{2}/gi, (encoded) => { - const character = String.fromCharCode(Number.parseInt(encoded.slice(1), 16)); - return /[A-Za-z0-9._~-]/.test(character) ? character : encoded.toUpperCase(); - }); - if ( - url.protocol !== 'https:' || - url.hostname !== host || - url.host !== host || - url.username || - url.password || - url.search || - url.hash || - pathname !== normalizedPathname || - value !== url.href || - value.length > MAX_URL_LENGTH - ) { - return null; - } - return {url, decodedPathname}; - } catch { - return null; - } -} - -function canonicalPublicUrl(value, host) { - return strictPublicUrl(value, host)?.url || null; -} - -function sitemapUrls(xml, host) { - const urls = []; - for (const value of parseSitemapLocations(xml)) { - const url = canonicalPublicUrl(value, host); - if (url) urls.push(url.href); - } - return [...new Set(urls)].sort(codeUnitCompare); -} - -function isWithin(root, candidate) { - const relative = path.relative(root, candidate); - return ( - relative !== '' && - !relative.startsWith(`..${path.sep}`) && - relative !== '..' && - !path.isAbsolute(relative) - ); -} - -async function publishedFileForUrl(publishDir, value) { - const root = path.resolve(publishDir); - const realRoot = await fs.realpath(root); - let candidateUrl; - try { - candidateUrl = new URL(value); - } catch { - throw new Error(`Cannot map published URL to a file: ${value}`); - } - const parsed = strictPublicUrl(value, candidateUrl.hostname); - if (!parsed) { - throw new Error(`Cannot map published URL to a file: ${value}`); - } - const relativePath = parsed.decodedPathname.replace(/^\/|\/$/g, ''); - const candidates = relativePath - ? [ - path.resolve(root, relativePath, 'index.html'), - path.resolve(root, `${relativePath}.html`), - path.resolve(root, relativePath), - ] - : [path.resolve(root, 'index.html')]; - - const matches = []; - for (const candidate of [...new Set(candidates)]) { - if (candidate !== root && !isWithin(root, candidate)) { - throw new Error(`Published URL escapes the output directory: ${value}`); - } - try { - const stat = await fs.lstat(candidate); - if (stat.isSymbolicLink()) { - throw new Error(`Published URL maps to a symbolic link: ${value}`); - } - if (!stat.isFile()) continue; - const realCandidate = await fs.realpath(candidate); - const expectedRealCandidate = path.resolve(realRoot, path.relative(root, candidate)); - if (realCandidate !== expectedRealCandidate) { - throw new Error(`Published URL maps through a symbolic link: ${value}`); - } - matches.push({candidate, realCandidate}); - } catch (error) { - if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') throw error; - } - } - - if (matches.length > 1) { - throw new Error(`Published URL maps to multiple files: ${value}`); - } - if (matches.length === 1) { - const {candidate, realCandidate} = matches[0]; - return path.extname(candidate).toLowerCase() === '.html' - ? {htmlPath: candidate, resolvedPath: realCandidate, skipped: false} - : {htmlPath: null, resolvedPath: realCandidate, skipped: true}; - } - - throw new Error(`No published file found for ${value}`); -} - -async function hashPublishedPages(publishDir, urls) { - const pages = {}; - const skipped = []; - const fileOwners = new Map(); - for (const url of urls) { - const published = await publishedFileForUrl(publishDir, url); - if (published.skipped) { - skipped.push(url); - continue; - } - const owner = fileOwners.get(published.resolvedPath); - if (owner) { - throw new Error(`Published URLs map to the same HTML file: ${owner} and ${url}`); - } - fileOwners.set(published.resolvedPath, url); - const html = await fs.readFile(published.htmlPath); - pages[url] = crypto.createHash('sha256').update(html).digest('hex'); - } - return {pages, skipped}; -} - -function diffPages(previousPages, currentPages) { - const added = []; - const updated = []; - const removed = []; - for (const [url, hash] of Object.entries(currentPages)) { - if (!Object.hasOwn(previousPages, url)) added.push(url); - else if (previousPages[url] !== hash) updated.push(url); - } - for (const url of Object.keys(previousPages)) { - if (!Object.hasOwn(currentPages, url)) removed.push(url); - } - added.sort(codeUnitCompare); - updated.sort(codeUnitCompare); - removed.sort(codeUnitCompare); - return { - added, - updated, - removed, - all: [...added, ...updated, ...removed].sort(codeUnitCompare), - }; -} - -function changedUrls(previousPages, currentPages) { - return diffPages(previousPages, currentPages).all; -} - -function chunks(values, size = MAX_URLS_PER_REQUEST) { - if (!Number.isInteger(size) || size < 1 || size > MAX_URLS_PER_REQUEST) { - throw new Error(`IndexNow batch size must be between 1 and ${MAX_URLS_PER_REQUEST}`); - } - return Array.from( - {length: Math.ceil(values.length / size)}, - (_, index) => values.slice(index * size, (index + 1) * size), - ); -} - -function validState(state, host) { - if ( - state?.version !== STATE_VERSION || - state.host !== host || - !state.pages || - typeof state.pages !== 'object' || - Array.isArray(state.pages) || - !Object.keys(state.pages).length - ) { - return false; - } - return Object.entries(state.pages).every(([value, hash]) => { - return canonicalPublicUrl(value, host) !== null && /^[0-9a-f]{64}$/.test(hash); - }); -} - -function sortedPages(pages) { - return Object.fromEntries( - Object.entries(pages).sort(([left], [right]) => codeUnitCompare(left, right)), - ); -} - -function stateBytes(host, pages) { - return `${JSON.stringify({version: STATE_VERSION, host, pages: sortedPages(pages)}, null, 2)}\n`; -} - -async function readState(statePath, host) { - try { - const contents = await fs.readFile(statePath, 'utf8'); - const state = JSON.parse(contents); - if (!validState(state, host) || contents !== stateBytes(host, state.pages)) return null; - return state; - } catch (error) { - if (error.code === 'ENOENT' || error instanceof SyntaxError) return null; - throw error; - } -} - -async function writeState(statePath, host, pages) { - await atomicWrite(statePath, stateBytes(host, pages)); -} - -async function atomicWrite(filename, contents) { - await fs.mkdir(path.dirname(filename), {recursive: true}); - const temporaryPath = `${filename}.tmp-${process.pid}-${crypto.randomUUID()}`; - try { - await fs.writeFile(temporaryPath, contents); - await fs.rename(temporaryPath, filename); - } finally { - await fs.rm(temporaryPath, {force: true}); - } -} - -async function readFileIfPresent(filename) { - try { - return await fs.readFile(filename); - } catch (error) { - if (error.code === 'ENOENT') return null; - throw error; - } -} - -async function restoreLocalFile(filename, contents) { - if (contents === null) { - await fs.rm(filename, {force: true}); - } else { - await atomicWrite(filename, contents); - } -} - -async function validateBuiltKeyFile(publishDir, key) { - const keyPath = path.join(path.resolve(publishDir), `${key}.txt`); - let stat; - try { - stat = await fs.lstat(keyPath); - } catch (error) { - if (error.code === 'ENOENT') throw new Error(`IndexNow built root key file is missing: ${keyPath}`); - throw error; - } - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error(`IndexNow built root key artifact is not a regular non-symlink file: ${keyPath}`); - } - const contents = await fs.readFile(keyPath); - if (!contents.equals(Buffer.from(key, 'utf8'))) { - throw new Error('IndexNow built root key file bytes do not exactly match the configured key'); - } - return keyPath; -} - -async function persistState({statePath, host, pages, cache}) { - const previousContents = await readFileIfPresent(statePath); - await writeState(statePath, host, pages); - try { - const saved = await cache.save(statePath); - if (saved !== true) throw new Error('Netlify cache save returned false'); - } catch (error) { - try { - await restoreLocalFile(statePath, previousContents); - } catch (rollbackError) { - throw new Error( - `IndexNow state cache persistence failed (${error.message}) and local rollback failed (${rollbackError.message})`, - ); - } - throw new Error(`IndexNow state cache persistence failed; local state was rolled back: ${error.message}`); - } -} - -function validateSubmissionUrls(urls, host) { - for (const value of urls) { - if (!canonicalPublicUrl(value, host)) { - throw new Error(`IndexNow payload URL is not canonical HTTPS on ${host}: ${value}`); - } - } -} - -function requestDeadline(timeoutMs) { - if (!Number.isInteger(timeoutMs) || timeoutMs < 1) { - throw new Error('IndexNow request timeout must be a positive integer'); - } - const controller = new AbortController(); - let expired = false; - let rejectTimeout; - const timeout = new Promise((_, reject) => { - rejectTimeout = reject; - }); - const timer = setTimeout(() => { - expired = true; - controller.abort(); - rejectTimeout(new Error(`IndexNow request exceeded ${timeoutMs}ms`)); - }, timeoutMs); - return { - signal: controller.signal, - get expired() { return expired; }, - wait(promise) { return Promise.race([promise, timeout]); }, - clear() { clearTimeout(timer); }, - }; -} - -async function submitUrlBatches({ - urls, - inputs, - fetchImpl, - logger, - batchSize = MAX_URLS_PER_REQUEST, - requestTimeoutMs = REQUEST_TIMEOUT_MS, -}) { - const normalizedInputs = validateInputs(inputs); - validateSubmissionUrls(urls, normalizedInputs.host); - const batches = chunks(urls, batchSize); - const accepted = []; - const pending = []; - const batchResults = []; - - for (const [index, urlList] of batches.entries()) { - logger.log(`IndexNow request ${index + 1}/${batches.length}: ${urlList.length} URL(s).`); - const deadline = requestDeadline(requestTimeoutMs); - let phase = 'request'; - let httpStatus = null; - let status = 'pending'; - let reason = 'transport-error'; - try { - const response = await deadline.wait( - Promise.resolve().then(() => fetchImpl(ENDPOINT, { - method: 'POST', - headers: {'content-type': 'application/json; charset=utf-8'}, - body: JSON.stringify({...normalizedInputs, urlList}), - signal: deadline.signal, - })), - ); - httpStatus = Number.isInteger(response?.status) && - response.status >= 100 && response.status <= 599 - ? response.status - : null; - phase = 'response-body'; - if (typeof response?.text === 'function') { - await deadline.wait(Promise.resolve().then(() => response.text())); - } - if (httpStatus === null) { - reason = 'invalid-response'; - } else if (![200, 202].includes(httpStatus)) { - reason = 'http-error'; - } else { - status = 'accepted'; - reason = null; - accepted.push(...urlList); - logger.log( - `IndexNow request ${index + 1}/${batches.length} returned HTTP ${httpStatus}; durable receipt pending.`, - ); - } - } catch { - reason = deadline.expired - ? 'timeout' - : phase === 'response-body' - ? 'response-body-error' - : 'transport-error'; - } finally { - deadline.clear(); - } - if (status === 'pending') { - pending.push(...urlList); - logger.error( - `IndexNow request ${index + 1}/${batches.length} failed for ${urlList.length} URL(s): ${reason}${httpStatus === null ? '' : ` (HTTP ${httpStatus})`}.`, - ); - } - batchResults.push({ - batch: index + 1, - batchCount: batches.length, - status, - httpStatus, - reason, - urls: [...urlList], - }); - } - - return {accepted, pending, batches: batchResults}; -} - -function canonicalize(value) { - if (Array.isArray(value)) return value.map(canonicalize); - if (value && typeof value === 'object') { - return Object.fromEntries( - Object.keys(value).sort(codeUnitCompare).map((key) => [key, canonicalize(value[key])]), - ); - } - return value; -} - -function canonicalJson(value) { - return `${JSON.stringify(canonicalize(value), null, 2)}\n`; -} - -function validateReceiptIdentity(deployId, siteId, commitRef) { - if (typeof deployId !== 'string' || !deployId || typeof siteId !== 'string' || !siteId) { - throw new Error('IndexNow receipt persistence requires Netlify DEPLOY_ID and SITE_ID'); - } - if ( - commitRef !== null && - (typeof commitRef !== 'string' || !commitRef || commitRef !== commitRef.trim()) - ) { - throw new Error('IndexNow receipt COMMIT_REF must be a non-empty string or null'); - } -} - -function receiptDocument({host, batches, deployId, siteId, commitRef}) { - validateReceiptIdentity(deployId, siteId, commitRef); - const acceptedCount = batches - .filter((batch) => batch.status === 'accepted') - .reduce((total, batch) => total + batch.urls.length, 0); - const pendingCount = batches - .filter((batch) => batch.status === 'pending') - .reduce((total, batch) => total + batch.urls.length, 0); - return { - schema_version: RECEIPT_SCHEMA_VERSION, - producer: 'netdata/website-indexnow', - endpoint: ENDPOINT, - site_id: siteId, - deploy_id: deployId, - commit_ref: commitRef, - host, - batch_count: batches.length, - accepted_url_count: acceptedCount, - pending_url_count: pendingCount, - url_count: acceptedCount + pendingCount, - batches: batches.map((batch) => ({ - batch: batch.batch, - batch_count: batch.batchCount, - status: batch.status, - http_status: batch.httpStatus, - reason: batch.reason, - url_count: batch.urls.length, - urls: [...batch.urls], - })), - }; -} - -async function persistSubmissionReceipt({host, batches, deployId, siteId, commitRef, store}) { - validateReceiptIdentity(deployId, siteId, commitRef); - const receipt = receiptDocument({host, batches, deployId, siteId, commitRef}); - const contents = canonicalJson(receipt); - if (Buffer.byteLength(contents) > MAX_RECEIPT_BYTES) { - throw new Error(`IndexNow receipt exceeds ${MAX_RECEIPT_BYTES} bytes`); - } - const sha256 = crypto.createHash('sha256').update(contents).digest('hex'); - const blobKey = `receipts/${sha256}.json`; - const receiptStore = store || getDeployStore({ - name: RECEIPT_STORE, - deployID: deployId, - siteID: siteId, - consistency: 'strong', - }); - try { - const result = await receiptStore.set(blobKey, contents, { - onlyIfNew: true, - metadata: {schema_version: RECEIPT_SCHEMA_VERSION, sha256}, - }); - if (!result || typeof result.modified !== 'boolean') { - throw new Error('Netlify Blob set returned an invalid result'); - } - if (!result.modified) { - const existing = await receiptStore.get(blobKey, {consistency: 'strong'}); - if (existing !== contents) { - throw new Error('existing Netlify Blob does not match its content digest'); - } - } - } catch (error) { - throw new Error(`IndexNow receipt persistence failed: ${error.message}`); - } - return { - sha256, - blobKey, - batchCount: batches.length, - acceptedCount: receipt.accepted_url_count, - pendingCount: receipt.pending_url_count, - urlCount: receipt.url_count, - receipt, - }; -} - -function advanceState(previousPages, currentPages, accepted) { - const pages = {...previousPages}; - for (const url of accepted) { - if (Object.hasOwn(currentPages, url)) pages[url] = currentPages[url]; - else delete pages[url]; - } - return sortedPages(pages); -} - -async function runOnSuccess({ - constants, - inputs, - utils, - statePath = STATE_PATH, - fetchImpl = globalThis.fetch, - logger = console, - batchSize = MAX_URLS_PER_REQUEST, - requestTimeoutMs = REQUEST_TIMEOUT_MS, - deployId = process.env.DEPLOY_ID, - siteId = process.env.SITE_ID, - commitRef = process.env.COMMIT_REF === undefined ? null : process.env.COMMIT_REF, - receiptStore, -}) { - const normalizedInputs = validateInputs(inputs); - await validateBuiltKeyFile(constants.PUBLISH_DIR, normalizedInputs.key); - validateReceiptIdentity(deployId, siteId, commitRef); - const sitemapPath = path.join(constants.PUBLISH_DIR, 'sitemap.xml'); - const xml = await fs.readFile(sitemapPath, 'utf8'); - const urls = sitemapUrls(xml, normalizedInputs.host); - if (!urls.length) { - throw new Error(`No ${normalizedInputs.host} URLs found in ${sitemapPath}`); - } - - const {pages: currentPages, skipped} = await hashPublishedPages(constants.PUBLISH_DIR, urls); - if (!Object.keys(currentPages).length) { - throw new Error(`No published HTML pages found for ${normalizedInputs.host}`); - } - if (skipped.length) { - logger.log(`IndexNow ignored ${skipped.length} non-HTML sitemap URL(s).`); - } - - const previousState = await readState(statePath, normalizedInputs.host); - if (!previousState) { - await persistState({ - statePath, - host: normalizedInputs.host, - pages: currentPages, - cache: utils.cache, - }); - logger.log( - `IndexNow cache seeded with ${Object.keys(currentPages).length} published HTML pages; no bulk submission sent.`, - ); - return { - seeded: true, - changes: {added: [], updated: [], removed: [], all: []}, - accepted: [], - pending: [], - batches: [], - receipt: null, - }; - } - - const changes = diffPages(previousState.pages, currentPages); - logger.log( - `IndexNow found ${changes.added.length} added, ${changes.updated.length} updated, and ${changes.removed.length} removed HTML URL(s).`, - ); - - const result = changes.all.length - ? await submitUrlBatches({ - urls: changes.all, - inputs: normalizedInputs, - fetchImpl, - logger, - batchSize, - requestTimeoutMs, - }) - : {accepted: [], pending: [], batches: []}; - if (!changes.all.length) { - logger.log(`IndexNow found no changed pages among ${Object.keys(currentPages).length} HTML URLs.`); - } - - let receipt = null; - if (result.batches.length) { - receipt = await persistSubmissionReceipt({ - host: normalizedInputs.host, - batches: result.batches, - deployId, - siteId, - commitRef, - store: receiptStore, - }); - const status = receipt.pendingCount ? 'partial' : 'accepted'; - logger.log( - `IndexNow receipt deploy=${deployId} digest=${receipt.sha256} urls=${receipt.urlCount} batches=${receipt.batchCount} accepted=${receipt.acceptedCount} pending=${receipt.pendingCount} status=${status}.`, - ); - } - - const nextPages = advanceState(previousState.pages, currentPages, result.accepted); - await persistState({ - statePath, - host: normalizedInputs.host, - pages: nextPages, - cache: utils.cache, - }); - if (result.pending.length) { - logger.error( - `IndexNow left ${result.pending.length} URL(s) pending after partial submission failure; accepted URLs were persisted.`, - ); - } - return {seeded: false, changes, ...result, receipt}; -} - -module.exports = { - STATE_PATH, - advanceState, - canonicalJson, - changedUrls, - chunks, - diffPages, - hashPublishedPages, - persistSubmissionReceipt, - publishedFileForUrl, - readState, - receiptDocument, - runOnSuccess, - sitemapUrls, - stateBytes, - submitUrlBatches, - validateBuiltKeyFile, - validateInputs, - validateSubmissionUrls, -}; diff --git a/plugins/netlify-plugin-indexnow/index.js b/plugins/netlify-plugin-indexnow/index.js deleted file mode 100644 index 183d469e0..000000000 --- a/plugins/netlify-plugin-indexnow/index.js +++ /dev/null @@ -1,27 +0,0 @@ -const path = require('node:path'); - -const {STATE_PATH, runOnSuccess} = require('./core.js'); - -module.exports = { - onPreBuild: async ({utils}) => { - try { - await utils.cache.restore(STATE_PATH); - } catch (error) { - console.warn( - `IndexNow cache restore failed for ${path.basename(STATE_PATH)}; the deploy will continue: ${error.message}`, - ); - } - }, - onSuccess: async (event) => { - const context = process.env.CONTEXT || event.netlifyConfig?.build?.environment?.CONTEXT; - if (context !== 'production') { - console.log(`IndexNow skipped non-production deploy context: ${context || 'unknown'}.`); - return; - } - try { - await runOnSuccess(event); - } catch (error) { - console.error(`IndexNow submission failed; the successful deploy remains valid: ${error.message}`); - } - }, -}; diff --git a/plugins/netlify-plugin-indexnow/manifest.yml b/plugins/netlify-plugin-indexnow/manifest.yml deleted file mode 100644 index 6177ef861..000000000 --- a/plugins/netlify-plugin-indexnow/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: netlify-plugin-indexnow -inputs: - - name: host - description: Hostname submitted to IndexNow. - required: true - - name: key - description: Public IndexNow key. - required: true - - name: keyLocation - description: Exact HTTPS URL of the root IndexNow key file. - required: true diff --git a/plugins/netlify-plugin-indexnow/package.json b/plugins/netlify-plugin-indexnow/package.json deleted file mode 100644 index da7c89dc3..000000000 --- a/plugins/netlify-plugin-indexnow/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "netlify-plugin-indexnow", - "version": "1.0.0", - "private": true, - "main": "index.js", - "dependencies": { - "@netlify/blobs": "11.1.4", - "saxes": "6.0.0" - }, - "engines": { - "node": ">=22.12.0" - } -} diff --git a/plugins/netlify-plugin-indexnow/receipt-schema.json b/plugins/netlify-plugin-indexnow/receipt-schema.json deleted file mode 100644 index c5bb46d40..000000000 --- a/plugins/netlify-plugin-indexnow/receipt-schema.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://www.netdata.cloud/schemas/indexnow-deploy-receipt-v2.json", - "title": "Netdata IndexNow production deploy receipt", - "type": "object", - "additionalProperties": false, - "required": [ - "accepted_url_count", - "batch_count", - "batches", - "commit_ref", - "deploy_id", - "endpoint", - "host", - "pending_url_count", - "producer", - "schema_version", - "site_id", - "url_count" - ], - "properties": { - "schema_version": {"const": 2}, - "producer": {"const": "netdata/website-indexnow"}, - "endpoint": {"const": "https://api.indexnow.org/indexnow"}, - "site_id": {"type": "string", "minLength": 1}, - "deploy_id": {"type": "string", "minLength": 1}, - "commit_ref": {"type": ["string", "null"], "minLength": 1}, - "host": {"type": "string", "minLength": 1}, - "batch_count": {"type": "integer", "minimum": 1}, - "accepted_url_count": {"type": "integer", "minimum": 0}, - "pending_url_count": {"type": "integer", "minimum": 0}, - "url_count": {"type": "integer", "minimum": 1}, - "batches": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "batch", - "batch_count", - "http_status", - "reason", - "status", - "url_count", - "urls" - ], - "properties": { - "batch": {"type": "integer", "minimum": 1}, - "batch_count": {"type": "integer", "minimum": 1}, - "status": {"enum": ["accepted", "pending"]}, - "http_status": {"type": ["integer", "null"], "minimum": 100, "maximum": 599}, - "reason": { - "enum": [null, "http-error", "invalid-response", "response-body-error", "timeout", "transport-error"] - }, - "url_count": {"type": "integer", "minimum": 1, "maximum": 10000}, - "urls": { - "type": "array", - "minItems": 1, - "maxItems": 10000, - "items": {"type": "string", "minLength": 1, "maxLength": 2048} - } - } - } - } - } -} diff --git a/plugins/netlify-plugin-indexnow/vendor-checksums.json b/plugins/netlify-plugin-indexnow/vendor-checksums.json deleted file mode 100644 index ee1d57d22..000000000 --- a/plugins/netlify-plugin-indexnow/vendor-checksums.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "schema_version": 2, - "owner": "netdata/website", - "algorithm": "sha256", - "dependencies": { - "@netlify/blobs": "11.1.4", - "saxes": "6.0.0" - }, - "receipt_schema_version": 2, - "contract_tests_schema_version": 2, - "files": { - "contract-tests.json": "ee42ff0683ed74dd969a4394fd357bc477e3fddcefe0b1821d33d3a036c0e882", - "core.js": "54c5c4dd8ddb06e51daec6ed653f6cbe789d7ad9334c8b67c1acb2a25b2a7193", - "index.js": "a9e4d1d3ede1e70cfc5d9c806bd29e29a38969b3576eef5120ee7b28f181abe1", - "manifest.yml": "6ad2d0238e4836b0d4d8454b88fc0dd1dfafa9b2cf6197ef25d5cae71512ad6a", - "package.json": "51b09fa7d5f2fca93670dab6fb0416a1d1877b484f66ee7af75a7559758e1850", - "receipt-schema.json": "e4a20ce04f2cce43fa7cf5bdcf11e1dc36da284b5709f60d77843a1392fa5145" - } -} diff --git a/scripts/verify-indexnow-vendor.js b/scripts/verify-indexnow-vendor.js deleted file mode 100644 index b2488adf4..000000000 --- a/scripts/verify-indexnow-vendor.js +++ /dev/null @@ -1,69 +0,0 @@ -const crypto = require('node:crypto'); -const fs = require('node:fs'); -const path = require('node:path'); - -const pluginDirectory = path.resolve('plugins/netlify-plugin-indexnow'); -const contractPath = path.join(pluginDirectory, 'vendor-checksums.json'); - -try { - const contract = JSON.parse(fs.readFileSync(contractPath, 'utf8')); - if ( - contract.schema_version !== 2 || - contract.owner !== 'netdata/website' || - contract.algorithm !== 'sha256' - ) { - throw new Error('Unsupported IndexNow vendor contract'); - } - const filenames = Object.keys(contract.files || {}).sort(); - const required = 'contract-tests.json,core.js,index.js,manifest.yml,package.json,receipt-schema.json'; - if (filenames.join(',') !== required) { - throw new Error(`IndexNow vendor contract must cover ${required}`); - } - - const expectedDependencies = {'@netlify/blobs': '11.1.4', saxes: '6.0.0'}; - if (JSON.stringify(contract.dependencies) !== JSON.stringify(expectedDependencies)) { - throw new Error('IndexNow vendor dependency contract is invalid'); - } - const pluginPackage = JSON.parse( - fs.readFileSync(path.join(pluginDirectory, 'package.json'), 'utf8'), - ); - const rootPackage = JSON.parse(fs.readFileSync(path.resolve('package.json'), 'utf8')); - if (pluginPackage.engines?.node !== '>=22.12.0') { - throw new Error('IndexNow plugin must require the Node runtime supported by @netlify/blobs'); - } - for (const [name, version] of Object.entries(expectedDependencies)) { - if (pluginPackage.dependencies?.[name] !== version || rootPackage.dependencies?.[name] !== version) { - throw new Error(`IndexNow dependency ${name} must be pinned to ${version} in both package files`); - } - } - const receiptSchema = JSON.parse( - fs.readFileSync(path.join(pluginDirectory, 'receipt-schema.json'), 'utf8'), - ); - const testContract = JSON.parse( - fs.readFileSync(path.join(pluginDirectory, 'contract-tests.json'), 'utf8'), - ); - if (receiptSchema.properties?.schema_version?.const !== contract.receipt_schema_version || - testContract.schema_version !== contract.contract_tests_schema_version || - testContract.receipt_schema_version !== contract.receipt_schema_version || - JSON.stringify(testContract.plugin_entry_events) !== JSON.stringify(['onPreBuild', 'onSuccess'])) { - throw new Error('IndexNow receipt or test schema does not match the vendor contract'); - } - - for (const [filename, expected] of Object.entries(contract.files)) { - if (path.basename(filename) !== filename || !/^[0-9a-f]{64}$/.test(expected)) { - throw new Error(`Invalid IndexNow vendor contract entry: ${filename}`); - } - const actual = crypto - .createHash('sha256') - .update(fs.readFileSync(path.join(pluginDirectory, filename))) - .digest('hex'); - if (actual !== expected) { - throw new Error(`${filename} does not match the website-owned vendor contract`); - } - } - - console.log(`Verified IndexNow vendor contract (${filenames.length} files).`); -} catch (error) { - console.error(`IndexNow vendor verification failed: ${error.message}`); - process.exitCode = 1; -} diff --git a/src/seo/deploymentIntegrations.test.js b/src/seo/deploymentIntegrations.test.js index 2be80abb6..f48ef8975 100644 --- a/src/seo/deploymentIntegrations.test.js +++ b/src/seo/deploymentIntegrations.test.js @@ -81,25 +81,17 @@ describe('shared deployment integrations', () => { expect(docusaurus.themeConfig.navbar.logo.target).toBe('_self'); }); - it('uses the complete website-owned IndexNow schema-2 contract', () => { - const contract = require('../../plugins/netlify-plugin-indexnow/vendor-checksums.json'); - expect(contract).toMatchObject({ + it('installs the website-owned IndexNow schema-2 plugin from npm', () => { + const plugin = '@netdata/netlify-plugin-indexnow'; + expect(packageJson.devDependencies).toHaveProperty(plugin); + for (const relative of ['static.toml', 'netlify.toml']) { + const config = fs.readFileSync(path.join(root, relative), 'utf8'); + expect(config).toContain(`package = "${plugin}"`); + } + expect(require(`${plugin}/contract-tests.json`)).toMatchObject({ schema_version: 2, - owner: 'netdata/website', - algorithm: 'sha256', - dependencies: {'@netlify/blobs': '11.1.4', saxes: '6.0.0'}, receipt_schema_version: 2, - contract_tests_schema_version: 2, }); - expect(Object.keys(contract.files).sort()).toEqual([ - 'contract-tests.json', - 'core.js', - 'index.js', - 'manifest.yml', - 'package.json', - 'receipt-schema.json', - ]); - expect(packageJson.dependencies).toMatchObject(contract.dependencies); }); it('installs exactly one manual Cloudflare module beacon through shared configuration', () => { diff --git a/static.toml b/static.toml index ebab52b69..19314c3b6 100644 --- a/static.toml +++ b/static.toml @@ -6,7 +6,7 @@ environment = { NPM_VERSION = "10.9.9", NODE_VERSION = "22.23.3", NODE_OPTIONS = "--max-old-space-size=3072", DOCUSAURUS_SSR_CONCURRENCY = "4"} [[plugins]] - package = "./plugins/netlify-plugin-indexnow" + package = "@netdata/netlify-plugin-indexnow" [plugins.inputs] host = "learn.netdata.cloud" diff --git a/tests/indexnow_plugin.test.js b/tests/indexnow_plugin.test.js index 2518f9d48..5a1eddf8d 100644 --- a/tests/indexnow_plugin.test.js +++ b/tests/indexnow_plugin.test.js @@ -5,9 +5,9 @@ const os = require('node:os'); const path = require('node:path'); const test = require('node:test'); -const plugin = require('../plugins/netlify-plugin-indexnow'); -const core = require('../plugins/netlify-plugin-indexnow/core.js'); -const contractTests = require('../plugins/netlify-plugin-indexnow/contract-tests.json'); +const plugin = require('@netdata/netlify-plugin-indexnow'); +const core = require('@netdata/netlify-plugin-indexnow/core.js'); +const contractTests = require('@netdata/netlify-plugin-indexnow/contract-tests.json'); const HOST = 'learn.netdata.cloud'; const KEY = '69893e732658435694bf647d5a6fed1d'; diff --git a/yarn.lock b/yarn.lock index c9b254c99..8ce8b9915 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3176,6 +3176,14 @@ dependencies: "@tybys/wasm-util" "^0.10.3" +"@netdata/netlify-plugin-indexnow@^1.0.0": + version "1.0.0" + resolved "https://registry.npmjs.org/@netdata/netlify-plugin-indexnow/-/netlify-plugin-indexnow-1.0.0.tgz#b6a67116ba547be46fd70608ab700197b82134f5" + integrity sha512-k9TaaHsIEopPpgoXhz5kzihITcsxEiYsj7kon/e4TNR4juFG9S9IDCJKX9nwwCvfEUYXAjusNrRfEIsgJMu9Yw== + dependencies: + "@netlify/blobs" "^11.1.4" + saxes "^6.0.0" + "@netdata/site-build-gate@^10.1.0": version "10.1.0" resolved "https://registry.npmjs.org/@netdata/site-build-gate/-/site-build-gate-10.1.0.tgz#08c5aa479c00652762a86670444c12a72945b372" @@ -3185,7 +3193,7 @@ parse5 "^8.0.1" saxes "^6.0.0" -"@netlify/blobs@11.1.4": +"@netlify/blobs@^11.1.4": version "11.1.4" resolved "https://registry.npmjs.org/@netlify/blobs/-/blobs-11.1.4.tgz#74e4cd58b8f6a92c4b7254d644065d8610e8783d" integrity sha512-rtvtXG0NQkmIdpdI8AI/LKesf3TdZJwNP0VaxodPpb/yYzME62YAaqYVmB2MC5gO5Wxls1aIzcCK2xJFRNRhoQ== @@ -11880,7 +11888,7 @@ sax@^1.2.4, sax@^1.5.0: resolved "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz#4c23cf608c0b693ab54b4b5888e92cfe977b9843" integrity sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q== -saxes@6.0.0, saxes@^6.0.0: +saxes@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" integrity sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==