diff --git a/README.md b/README.md index ac70e43..98b8cda 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ The `agent-executor` (and the provided brand-profile agent) rely on the Azure Op | `AZURE_OPENAI_KEY` | API key for the Azure OpenAI resource | | `AZURE_API_VERSION` | API version used for the chat completions | | `AZURE_COMPLETION_DEPLOYMENT` | Deployment/model name (e.g., `gpt-4o`) | +| `BRAND_PROFILE_ENTITY_VALIDATION_LLM` | Enable the LLM leg of the Class-B entity-validation gate. Defaults **on**; set to `false` to disable it (a non-matching entity then holds as `entity_validation: unverified` instead of being LLM-triaged to a hard mismatch). The deterministic P856 domain-match always runs regardless. | When invoking the integration test, you can also set `BRAND_PROFILE_TEST_BASE_URL` to control which site is analyzed and `BRAND_PROFILE_IT_FULL=1` to print the complete agent response (otherwise the preview is truncated for readability). - To lint code: diff --git a/src/agents/brand-profile/index.js b/src/agents/brand-profile/index.js index eaff791..97716a8 100644 --- a/src/agents/brand-profile/index.js +++ b/src/agents/brand-profile/index.js @@ -176,7 +176,7 @@ async function run(context, env, log) { // name is the unknown-brand sentinel (a name search would contaminate). const needsWikipedia = !hasText(sitemapUrl) || !(Array.isArray(llmoCompetitors) && llmoCompetitors.length > 0); - const brandWiki = (brandName !== UNKNOWN_BRAND && needsWikipedia) + let brandWiki = (brandName !== UNKNOWN_BRAND && needsWikipedia) ? await wikipediaService.resolveBrand(brandName) : { wikidataId: null, @@ -187,6 +187,45 @@ async function run(context, env, log) { discardReason: 'skipped', }; + // Phase 3.5: Validate the resolved Wikidata entity ONCE, before any QID-derived + // context is consumed. The QID->article guard cannot catch a QID resolved to the + // wrong same-named entity (capella.edu -> Q12970, the star). Validating here - + // ahead of competitor inference, personas AND product extraction - ensures a wrong + // entity's Wikipedia summary/text never contaminates ANY downstream output (not + // just products). Only a 'verified' entity (authoritative P856 domain match) is + // trusted; 'mismatch'/'unverified' withhold its context fleet-wide. + let entityValidation = null; + if (brandWiki.wikidataId) { + const domain = new URL(baseURL).hostname; + entityValidation = await productService.validateEntity({ + wikidataId: brandWiki.wikidataId, brandName, domain, industry, + }); + log.info('brand-profile: entity-validation-outcome', { + metric: 'entity_validation_outcome', + wikidataId: brandWiki.wikidataId, + brand: brandName, + domain, + status: entityValidation.status, + method: entityValidation.method, + failOpen: entityValidation.failOpen === true, + }); + if (entityValidation.failOpen) { + log.warn('brand-profile: entity-validation-fail-open', { + metric: 'entity_validation_fail_open', + wikidataId: brandWiki.wikidataId, + brand: brandName, + domain, + method: entityValidation.method, + }); + } + if (entityValidation.status !== 'verified') { + log.warn(`brand-profile: entity ${brandWiki.wikidataId} for "${brandName}" (${baseURL}) is ${entityValidation.status} via ${entityValidation.method} - withholding Wikidata/Wikipedia context from all consumers`); + brandWiki = { + ...brandWiki, wikidataId: null, fullText: '', summary: '', + }; + } + } + // Phase 2: Infer region from URL log.info('brand-profile: inferring region from URL'); const regionInference = await regionalService.inferRegionFromUrl(baseURL); @@ -251,9 +290,15 @@ async function run(context, env, log) { productsResult = await productService.extractFromSitemap(sitemapUrl, brandName); } else { // Use Wikidata + QID-anchored Wikipedia extraction (resolved once, above). + // entityValidation was already computed in Phase 3.5 and brandWiki gated on it, + // so pass the verdict through: the extractor records it and skips re-validating + // (no second LLM call). A non-verified entity already has wikidataId/text nulled. productsResult = await productService.extractProducts(brandName, { wikidataId: brandWiki.wikidataId, wikipediaText: brandWiki.fullText, + domain: new URL(baseURL).hostname, + industry, + entityValidation, }); } diff --git a/src/agents/brand-profile/services/product-extractor.js b/src/agents/brand-profile/services/product-extractor.js index 3e3f927..c5c6cbc 100644 --- a/src/agents/brand-profile/services/product-extractor.js +++ b/src/agents/brand-profile/services/product-extractor.js @@ -23,7 +23,7 @@ import { AzureOpenAIClient } from '@adobe/spacecat-shared-gpt-client'; import { readPromptFile, renderTemplate } from '../../base.js'; -import { findWikidataId, resolveBrandWikipedia } from './wikipedia.js'; +import { findWikidataId, resolveBrandWikipedia, validateEntityMatchesBrand } from './wikipedia.js'; const USER_AGENT = 'SpaceCat/1.0 (https://github.com/adobe/spacecat; spacecat@adobe.com)'; const WIKIDATA_SPARQL = 'https://query.wikidata.org/sparql'; @@ -513,8 +513,80 @@ export async function extractProducts(brandName, wikipediaContext, gpt, log) { }, }; - // Step 1: Find brand's Wikidata ID (reuse the caller's when provided) - const wikidataId = ctxQid || await findWikidataId(brandName, log); + // Step 1: Find brand's Wikidata ID (reuse the caller's when provided). + // When the caller passes a precomputed entityValidation verdict (index.js), it has + // ALREADY decided the QID (nulled if not verified) - respect it and do NOT + // name-search a replacement, because that name search is itself a contamination + // vector (it can resolve the wrong same-named entity all over again). + let wikidataId = ctx.entityValidation + ? ctxQid + : (ctxQid || await findWikidataId(brandName, log)); + + // Step 1b: Validate the resolved entity actually is this brand before trusting + // any of its data. The QID->article guard cannot catch a QID that was itself + // resolved to the wrong same-named entity (e.g. capella.edu -> Q12970, the star + // Capella). Only a 'verified' outcome (authoritative P856 domain match) may + // publish; both 'mismatch' (wrong entity) and 'unverified' (couldn't confirm / + // infra error) withhold the QID-derived data. Requires the brand domain; + // industry is optional (it only sharpens the LLM branch), so a missing industry + // no longer skips the check and lets contamination through. When no domain is + // supplied the check is skipped and behaviour is unchanged. + let entityStatus = null; + if (ctx.entityValidation) { + // Validation was already performed upstream (index.js Phase 3.5) BEFORE any + // consumer saw the QID context, so a wrong same-named entity never reached + // competitor/persona inference either. Reuse it verbatim - don't re-pay for the + // LLM call or re-emit the metric (index.js owns that for the hoisted path). + result.metadata.entity_validation = ctx.entityValidation; + entityStatus = ctx.entityValidation.status; + if (entityStatus !== 'verified') { + wikidataId = null; + } + } else if (wikidataId && ctx.domain) { + // Standalone path (direct callers / tests): validate here. The deterministic + // P856 domain-match always runs (free); the LLM fallback is the only token cost + // and only fires when P856 doesn't match. Flag off (env + // BRAND_PROFILE_ENTITY_VALIDATION_LLM=false) → no verifier → a non-matching + // entity holds as 'unverified' rather than being LLM-triaged to 'mismatch'. + const validationGpt = ctx.entityValidationUseLlm === false ? null : gpt; + const validation = await validateEntityMatchesBrand( + { + wikidataId, brandName, domain: ctx.domain, industry: ctx.industry || '', + }, + validationGpt, + log, + ); + result.metadata.entity_validation = validation; + entityStatus = validation.status; + + // Alarmable fail-open-rate metric. This repo surfaces observability through + // structured log lines (CloudWatch metric filters), not a metrics SDK - emit + // one line per validation so a filter can compute fail-open / total, plus a + // distinct warn line for the fail-open subset that was previously silent. + log.info('brand-profile: entity-validation-outcome', { + metric: 'entity_validation_outcome', + wikidataId, + brand: brandName, + domain: ctx.domain, + status: validation.status, + method: validation.method, + failOpen: validation.failOpen === true, + }); + if (validation.failOpen) { + log.warn('brand-profile: entity-validation-fail-open', { + metric: 'entity_validation_fail_open', + wikidataId, + brand: brandName, + domain: ctx.domain, + method: validation.method, + }); + } + + if (validation.status !== 'verified') { + log.warn(`brand-profile: entity ${wikidataId} for brand "${brandName}" (${ctx.domain}) is ${validation.status} via ${validation.method} - withholding Wikidata/Wikipedia sources`); + wikidataId = null; + } + } if (wikidataId) { result.metadata.brand_wikidata_id = wikidataId; @@ -536,7 +608,16 @@ export async function extractProducts(brandName, wikipediaContext, gpt, log) { log.info(`Wikidata returned ${result.products.length} products (threshold: ${MIN_PRODUCTS_THRESHOLD}), trying Wikipedia fallback`); let wikiText; - if (hasProvidedText) { + if (entityStatus && entityStatus !== 'verified') { + // The resolved entity did not authoritatively verify - never use its article + // text, even if the caller passed some (it would be the wrong/unconfirmed + // entity's article). 'mismatch' = known-wrong; 'unverified' = hold for review. + wikiText = ''; + result.metadata.wikipedia_verified = false; + result.metadata.wikipedia_discard_reason = entityStatus === 'mismatch' + ? 'entity-brand-mismatch' + : 'entity-unverified'; + } else if (hasProvidedText) { // Caller supplied the exact article text. Trust assumption: the caller is // responsible for having QID-anchored/verified this text (index.js resolves // it via resolveBrandWikipedia and only passes non-empty text when verified). @@ -630,6 +711,7 @@ export function formatProductsForPrompt(extractionResult) { */ export function createProductExtractorService(env, log) { const gpt = AzureOpenAIClient.createFrom({ env, log }); + const entityValidationUseLlm = env.BRAND_PROFILE_ENTITY_VALIDATION_LLM !== 'false'; return { extractFromSitemap: (sitemapUrl, brandName) => ( @@ -638,6 +720,19 @@ export function createProductExtractorService(env, log) { extractProducts: (brandName, wikipediaContext) => ( extractProducts(brandName, wikipediaContext, gpt, log) ), + // Validate a resolved Wikidata entity up front so a wrong same-named QID can be + // withheld from EVERY consumer (competitors, personas, products), not just + // product extraction. Uses the shared gpt + the LLM flag; returns the + // {status, method, reason, failOpen} verdict for the caller to act on. + validateEntity: ({ + wikidataId, brandName, domain, industry, + }) => validateEntityMatchesBrand( + { + wikidataId, brandName, domain, industry: industry || '', + }, + entityValidationUseLlm ? gpt : null, + log, + ), formatProductsForPrompt, }; } diff --git a/src/agents/brand-profile/services/wikipedia.js b/src/agents/brand-profile/services/wikipedia.js index 7299c1e..93478e5 100644 --- a/src/agents/brand-profile/services/wikipedia.js +++ b/src/agents/brand-profile/services/wikipedia.js @@ -226,6 +226,192 @@ export async function resolveBrandWikipedia(brandName, opts, log) { } } +// Two-level public suffixes, so registrableDomain keeps the org label +// (e.g. prudential.com.au, not com.au). Since a P856 domain match is now the only +// publish-authoritative signal, keep this reasonably complete to avoid FALSE +// matches (e.g. foo.org.uk and bar.org.uk both collapsing to org.uk). Still not a +// full Public Suffix List - a PSL library is the robust follow-up if the fleet +// needs exotic suffixes. +const TWO_LEVEL_SUFFIXES = new Set([ + // UK + 'co.uk', 'org.uk', 'me.uk', 'ac.uk', 'gov.uk', 'net.uk', 'sch.uk', 'ltd.uk', 'plc.uk', + // AU + 'com.au', 'net.au', 'org.au', 'edu.au', 'gov.au', 'asn.au', 'id.au', + // JP + 'co.jp', 'ne.jp', 'or.jp', 'ac.jp', 'go.jp', 'gr.jp', + // NZ + 'co.nz', 'net.nz', 'org.nz', 'ac.nz', 'govt.nz', 'geek.nz', + // IN + 'co.in', 'net.in', 'org.in', 'gen.in', 'firm.in', 'ac.in', 'edu.in', 'gov.in', + // ZA + 'co.za', 'org.za', 'net.za', 'gov.za', 'ac.za', + // BR + 'com.br', 'net.br', 'org.br', 'gov.br', 'edu.br', + // Other ccTLD second-levels seen across the customer fleet + 'com.sg', 'com.hk', 'com.mx', 'com.tr', 'co.kr', 'com.cn', 'com.co', + 'co.id', 'com.ph', 'com.my', 'co.th', 'com.vn', 'co.il', 'com.tw', 'com.ar', +]); + +/** + * Best-effort registrable domain (eTLD+1) for a host or URL. Lowercased, `www.` + * stripped, subdomains dropped. Handles the common two-level suffixes above so + * `brand.toyota.com` and `toyota.com` both reduce to `toyota.com`. + * @param {string} input - A hostname or URL + * @returns {string|null} registrable domain, or null when it can't be derived + */ +export function registrableDomain(input) { + if (!input) { + return null; + } + let host = String(input).trim().toLowerCase(); + try { + host = new URL(host.includes('://') ? host : `https://${host}`).hostname; + } catch { + // not a parseable URL - fall through and treat `host` as a bare hostname + } + host = host.replace(/^www\./, ''); + const labels = host.split('.').filter(Boolean); + if (labels.length <= 2) { + return labels.length ? labels.join('.') : null; + } + const lastTwo = labels.slice(-2).join('.'); + return TWO_LEVEL_SUFFIXES.has(lastTwo) ? labels.slice(-3).join('.') : lastTwo; +} + +/** + * Fetch a Wikidata entity's official website(s) (P856) and English description. + * @param {string} wikidataId - Wikidata entity ID + * @param {object} log - Logger instance + * @returns {Promise<{description: string, officialWebsites: string[]}|null>} + */ +export async function fetchWikidataEntityMeta(wikidataId, log) { + try { + const params = new URLSearchParams({ + action: 'wbgetentities', + ids: wikidataId, + props: 'claims|descriptions', + languages: 'en', + format: 'json', + }); + const resp = await fetch(`${WIKIDATA_API}?${params}`, { + headers: { 'User-Agent': USER_AGENT }, + }); + if (!resp.ok) { + throw new Error(`wbgetentities failed: ${resp.status}`); + } + const data = await resp.json(); + const entity = data.entities?.[wikidataId] || {}; + const description = entity.descriptions?.en?.value || ''; + const officialWebsites = (entity.claims?.P856 || []) + .map((claim) => claim?.mainsnak?.datavalue?.value) + .filter(Boolean); + return { description, officialWebsites }; + } catch (e) { + log.error(`brand-profile: fetchWikidataEntityMeta failed for ${wikidataId}: ${e.message}`); + return null; + } +} + +/** + * Verify that a resolved Wikidata entity actually corresponds to the brand, and + * classify the outcome into one of three publish-relevant states. + * + * The QID guard in resolveBrandWikipedia only proves the *article* matches the + * *QID* - it cannot catch a QID that was itself resolved to the wrong same-named + * entity (e.g. capella.edu -> Q12970, the star Capella). This closes that gap. + * + * Outcome (`status`): + * - 'verified' - the deterministic P856 official-website registrable-domain + * match succeeded. This is the ONLY authoritative pass: it is + * self-owned, tamper-resistant evidence, and the only status a + * caller may publish on. + * - 'mismatch' - a definitive LLM "no": a different same-named organization. + * The caller drops the QID (no SPARQL, no Wikipedia). + * - 'unverified' - could not confirm: no P856 domain match AND either no + * verifier, an inconclusive/positive-only LLM answer, or an + * infrastructure error. NON-PUBLISHABLE (hold for review), + * NOT trusted. The LLM branch may only flag/downgrade, never + * auto-pass-to-publish - it reads attacker-editable third-party + * text - so even an LLM "yes" lands here rather than verifying. + * + * `failOpen: true` marks the subset of 'unverified' caused by an infrastructure + * error (Wikidata or LLM). Before this change those silently "failed open" to + * trusted; they are now held AND counted, so a Wikidata/LLM degradation window is + * alarmable during a large regeneration batch (see the fail-open metric emitted by + * the caller in product-extractor). + * + * @param {{wikidataId: string, brandName: string, domain: string, industry: string}} args + * @param {object} gpt - AzureOpenAIClient instance (or null) + * @param {object} log - Logger instance + * @returns {Promise<{status: ('verified'|'mismatch'|'unverified'), method: string, + * reason: string, failOpen: boolean}>} classification of the entity match + */ +export async function validateEntityMatchesBrand({ + wikidataId, brandName, domain, industry, +}, gpt, log) { + const brandDomain = registrableDomain(domain); + const meta = await fetchWikidataEntityMeta(wikidataId, log); + if (!meta) { + // Infra error - cannot verify. Hold (non-publishable) and count as fail-open. + return { + status: 'unverified', method: 'error-meta-unavailable', reason: 'entity-meta-unavailable', failOpen: true, + }; + } + + // 1) Authoritative: official website (P856) registrable-domain match. This is + // the only self-owned, tamper-resistant signal, and the only path that passes. + const websiteHit = meta.officialWebsites.find( + (site) => brandDomain && registrableDomain(site) === brandDomain, + ); + if (websiteHit) { + return { + status: 'verified', method: 'official_website', reason: websiteHit, failOpen: false, + }; + } + + // 2) LLM check. It may only DOWNGRADE (a definitive "no" -> mismatch); it can + // never elevate to 'verified' (it consumes attacker-editable third-party + // text). Without a verifier we cannot confirm -> hold as unverified (not an + // error, so not counted as fail-open). + if (!gpt || typeof gpt.fetchChatCompletion !== 'function') { + return { + status: 'unverified', method: 'no-verifier', reason: meta.description || '', failOpen: false, + }; + } + try { + const prompt = `You verify whether a Wikipedia/Wikidata entity is the same organization as a website. +Website domain: ${domain} +Brand name: ${brandName} +Industry: ${industry || 'unknown'} +Candidate entity (${wikidataId}) description: "${meta.description || 'n/a'}" +Is the candidate entity the SAME organization that operates that website? Consider the industry and domain. Answer with a single word: yes or no.`; + const resp = await gpt.fetchChatCompletion(prompt, { temperature: 0, maxTokens: 3 }); + // Only an EXACT yes/no is definitive. Strip surrounding punctuation/quotes so + // "No." / "yes" parse, but keep ambiguous replies ("not sure", "no idea", + // "unclear") OUT of the definitive branches - they must hold as unverified, + // never discard the QID on ambiguity. + const answer = (resp?.choices?.[0]?.message?.content || '').trim().toLowerCase().replace(/[^a-z]/g, ''); + if (answer === 'no' || answer === 'n') { + // Definitive "no" - the only action the LLM is trusted to take: discard. + return { + status: 'mismatch', method: 'llm', reason: meta.description || '', failOpen: false, + }; + } + // "yes" or any non-definitive / unparseable answer -> not authoritative, hold. + return { + status: 'unverified', + method: (answer === 'yes' || answer === 'y') ? 'llm-unconfirmed' : 'llm-inconclusive', + reason: meta.description || '', + failOpen: false, + }; + } catch (e) { + log.warn(`brand-profile: entity LLM validation failed for ${wikidataId}: ${e.message} - holding as unverified`); + return { + status: 'unverified', method: 'error-llm', reason: 'llm-error', failOpen: true, + }; + } +} + /** * Create a Wikipedia service instance. * @param {object} log - Logger instance diff --git a/test/agents/brand-profile/index.test.js b/test/agents/brand-profile/index.test.js index bb1f045..059f0d5 100644 --- a/test/agents/brand-profile/index.test.js +++ b/test/agents/brand-profile/index.test.js @@ -64,6 +64,7 @@ describe('agents/brand-profile', () => { }, '../../../src/agents/brand-profile/services/product-extractor.js': { createProductExtractorService: () => ({ + validateEntity: sb.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), extractFromSitemap: sb.stub().resolves({ products: [], services: [], @@ -209,6 +210,7 @@ describe('agents/brand-profile', () => { }; const mockProductService = { + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), extractProducts: sandbox.stub().resolves({ products: [{ name: 'Product1', category: 'Software' }], services: [], @@ -277,6 +279,88 @@ describe('agents/brand-profile', () => { expect(result.products.items).to.have.length(1); }); + it('run() withholds a non-verified (fail-open) entity\'s Wikipedia context from ALL consumers, not just products', async () => { + const fetchChatCompletion = sandbox.stub().resolves({ + choices: [{ + message: { + content: JSON.stringify({ + main_profile: { brand_name: 'Capella' }, + competitive_context: { industry: 'Education' }, + }), + }, + }], + }); + const createFrom = sandbox.stub().returns({ fetchChatCompletion }); + + const mockCompetitorService = { + inferCompetitors: sandbox.stub().resolves({ competitors: [], source: 'llm_inferred' }), + }; + const mockProductService = { + // Entity could not be verified due to an infra error (fail-open path): must be + // held AND withheld from every consumer, and the fail-open metric emitted. + validateEntity: sandbox.stub().resolves({ status: 'unverified', method: 'error-meta-unavailable', failOpen: true }), + extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + }; + const mockWikipediaService = { + resolveBrand: sandbox.stub().resolves({ + wikidataId: 'Q12970', + title: 'Capella', + fullText: 'the brightest star in Auriga', + summary: 'a star in the constellation Auriga', + verified: true, + discardReason: null, + }), + }; + + const mod = await esmock('../../../src/agents/brand-profile/index.js', { + '@adobe/spacecat-shared-gpt-client': { AzureOpenAIClient: { createFrom } }, + '../../../src/agents/base.js': { + readPromptFile: sandbox.stub().returns('PROMPT'), + renderTemplate: sandbox.stub().returns('RENDERED'), + }, + '../../../src/agents/brand-profile/services/regional-context.js': { + createRegionalContextService: () => ({ + inferRegionFromUrl: sandbox.stub().resolves({ country_code: 'US' }), + inferRegionalContext: sandbox.stub().resolves({ languages: [], primary_language: 'en' }), + }), + }, + '../../../src/agents/brand-profile/services/competitor-inference.js': { + createCompetitorInferenceService: () => mockCompetitorService, + }, + '../../../src/agents/brand-profile/services/persona-inference.js': { + createPersonaInferenceService: () => ({ + inferPersonas: sandbox.stub().resolves({ personas: [], source: 'llm_inferred' }), + }), + }, + '../../../src/agents/brand-profile/services/product-extractor.js': { + createProductExtractorService: () => mockProductService, + }, + '../../../src/agents/brand-profile/services/wikipedia.js': { + createWikipediaService: () => mockWikipediaService, + }, + }); + + await mod.default.run( + { baseURL: 'https://capella.edu', params: { enhance: true } }, + env, + log, + ); + + // The wrong entity was validated up front... + expect(mockProductService.validateEntity).to.have.been.calledOnce; + // ...and its Wikipedia summary must NOT reach competitor inference. + const compArg = mockCompetitorService.inferCompetitors.firstCall.args[0]; + expect(compArg.wikipediaSummary).to.equal(''); + // ...and product extraction gets the precomputed verdict + a nulled QID. + const prodCtx = mockProductService.extractProducts.firstCall.args[1]; + expect(prodCtx.entityValidation.status).to.equal('unverified'); + expect(prodCtx.entityValidation.failOpen).to.equal(true); + expect(prodCtx.wikidataId).to.equal(null); + expect(prodCtx.wikipediaText).to.equal(''); + // ...and the alarmable fail-open metric was emitted. + expect(log.warn).to.have.been.calledWith('brand-profile: entity-validation-fail-open', sinon.match.object); + }); + it('run() uses sitemapUrl when provided for product extraction', async () => { const fetchChatCompletion = sandbox.stub().resolves({ choices: [{ @@ -291,6 +375,7 @@ describe('agents/brand-profile', () => { const createFrom = sandbox.stub().returns({ fetchChatCompletion }); const mockProductService = { + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), extractFromSitemap: sandbox.stub().resolves({ products: [{ name: 'SitemapProduct' }], services: [], @@ -407,6 +492,7 @@ describe('agents/brand-profile', () => { '../../../src/agents/brand-profile/services/product-extractor.js': { createProductExtractorService: () => ({ extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), }), }, '../../../src/agents/brand-profile/services/wikipedia.js': { @@ -473,6 +559,7 @@ describe('agents/brand-profile', () => { '../../../src/agents/brand-profile/services/product-extractor.js': { createProductExtractorService: () => ({ extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), }), }, '../../../src/agents/brand-profile/services/wikipedia.js': { @@ -539,6 +626,7 @@ describe('agents/brand-profile', () => { '../../../src/agents/brand-profile/services/product-extractor.js': { createProductExtractorService: () => ({ extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), }), }, '../../../src/agents/brand-profile/services/wikipedia.js': { @@ -607,6 +695,7 @@ describe('agents/brand-profile', () => { '../../../src/agents/brand-profile/services/product-extractor.js': { createProductExtractorService: () => ({ extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), }), }, '../../../src/agents/brand-profile/services/wikipedia.js': { @@ -970,6 +1059,7 @@ describe('agents/brand-profile', () => { productService: { extractFromSitemap: sandbox.stub().resolves({ products: [], metadata: {} }), extractProducts: sandbox.stub().resolves({ products: [], metadata: {} }), + validateEntity: sandbox.stub().resolves({ status: 'verified', method: 'official_website', failOpen: false }), }, competitorService: { inferCompetitors: sandbox.stub().resolves({ competitors: [] }), diff --git a/test/agents/brand-profile/services/product-extractor.test.js b/test/agents/brand-profile/services/product-extractor.test.js index fd22c2d..ecc37c2 100644 --- a/test/agents/brand-profile/services/product-extractor.test.js +++ b/test/agents/brand-profile/services/product-extractor.test.js @@ -485,6 +485,162 @@ describe('services/product-extractor', () => { expect(result.products).to.have.length(1); }); + it('discards a wrong same-named entity (validation fails) and emits no wikidata/wikipedia data', async () => { + // Entity metadata for validation: no official website, star description. + fetchStub.resolves({ + ok: true, + json: () => Promise.resolve({ + entities: { Q12970: { claims: {}, descriptions: { en: { value: 'bright star in the constellation Auriga' } } } }, + }), + }); + // LLM says the entity is NOT the brand. + gpt.fetchChatCompletion.resolves({ choices: [{ message: { content: 'no' } }] }); + + const result = await extractProducts( + 'Capella', + { + wikidataId: 'Q12970', + wikipediaText: 'Capella is the brightest star in Auriga.', + domain: 'capella.edu', + industry: 'Education', + }, + gpt, + log, + ); + + expect(result.metadata.entity_validation.status).to.equal('mismatch'); + expect(result.metadata.entity_validation.method).to.equal('llm'); + expect(result.metadata.brand_wikidata_id).to.equal(null); + expect(result.metadata.wikipedia_discard_reason).to.equal('entity-brand-mismatch'); + expect(result.products).to.have.length(0); + // The wrong QID must not reach the SPARQL products query. + const urls = fetchStub.getCalls().map((c) => c.args[0]); + expect(urls.some((u) => u.includes('query.wikidata.org'))).to.equal(false); + }); + + it('withholds product data as unverified on an infra error (fail-open held, not trusted)', async () => { + // Wikidata entity-meta fetch fails -> validation cannot confirm -> unverified. + fetchStub.resolves({ ok: false, status: 503 }); + // A verifier is present but must never be consulted once meta is unavailable. + gpt.fetchChatCompletion.resolves({ choices: [{ message: { content: 'yes' } }] }); + + const result = await extractProducts( + 'Behr', + { + wikidataId: 'Q20689046', + wikipediaText: 'Behr is a type of Iberian ham.', + domain: 'behr.com', + industry: 'Paint', + }, + gpt, + log, + ); + + expect(result.metadata.entity_validation.status).to.equal('unverified'); + expect(result.metadata.entity_validation.failOpen).to.equal(true); + expect(result.metadata.brand_wikidata_id).to.equal(null); + expect(result.metadata.wikipedia_discard_reason).to.equal('entity-unverified'); + expect(result.products).to.have.length(0); + // Held entity must not reach the SPARQL products query either. + const urls = fetchStub.getCalls().map((c) => c.args[0]); + expect(urls.some((u) => u.includes('query.wikidata.org'))).to.equal(false); + }); + + it('proceeds when the entity validates via official website match', async () => { + // Entity metadata: official website matches the brand domain -> deterministic pass. + fetchStub.resolves({ + ok: true, + json: () => Promise.resolve({ + entities: { Q123: { claims: { P856: [{ mainsnak: { datavalue: { value: 'https://www.capella.edu' } } }] }, descriptions: { en: { value: 'online university' } } } }, + }), + }); + // Only the product-extraction LLM call should happen (no validation LLM). + gpt.fetchChatCompletion.resolves({ + choices: [{ + message: { + content: JSON.stringify({ + products: [{ name: 'BSc Nursing' }], services: [], sub_brands: [], discontinued: [], + }), + }, + }], + }); + + const result = await extractProducts( + 'Capella University', + { + wikidataId: 'Q123', + wikipediaText: 'Capella University is an online university.', + domain: 'capella.edu', + industry: 'Education', + }, + gpt, + log, + ); + + expect(result.metadata.entity_validation.status).to.equal('verified'); + expect(result.metadata.entity_validation.method).to.equal('official_website'); + expect(result.metadata.brand_wikidata_id).to.equal('Q123'); + expect(result.products).to.have.length(1); + }); + + it('reuses a precomputed entityValidation verdict (no re-validation) and withholds on mismatch', async () => { + // Upstream (index.js) already validated and gated the entity; extractProducts + // must reuse the verdict verbatim, never re-call Wikidata/LLM. + const result = await extractProducts( + 'Capella', + { + wikidataId: null, // index.js nulls the QID for a non-verified entity + wikipediaText: '', + domain: 'capella.edu', + industry: 'Education', + entityValidation: { status: 'mismatch', method: 'llm', failOpen: false }, + }, + gpt, + log, + ); + + expect(result.metadata.entity_validation).to.deep.equal({ status: 'mismatch', method: 'llm', failOpen: false }); + expect(result.metadata.brand_wikidata_id).to.equal(null); + expect(result.metadata.wikipedia_discard_reason).to.equal('entity-brand-mismatch'); + expect(result.products).to.have.length(0); + // No validation LLM call and no Wikidata/SPARQL fetch happened. + expect(gpt.fetchChatCompletion).to.not.have.been.called; + expect(fetchStub).to.not.have.been.called; + }); + + it('with the LLM flag off, makes no validation LLM call and holds a non-matching entity as unverified', async () => { + // No official website -> P856 cannot confirm. With the LLM gated off, the + // entity must be HELD (unverified/no-verifier), never LLM-triaged, and no + // validation token cost is incurred. + fetchStub.resolves({ + ok: true, + json: () => Promise.resolve({ + entities: { Q12970: { claims: {}, descriptions: { en: { value: 'bright star in the constellation Auriga' } } } }, + }), + }); + + const result = await extractProducts( + 'Capella', + { + wikidataId: 'Q12970', + wikipediaText: 'Capella is the brightest star in Auriga.', + domain: 'capella.edu', + industry: 'Education', + entityValidationUseLlm: false, + }, + gpt, + log, + ); + + expect(result.metadata.entity_validation.status).to.equal('unverified'); + expect(result.metadata.entity_validation.method).to.equal('no-verifier'); + expect(result.metadata.entity_validation.failOpen).to.equal(false); + expect(result.metadata.brand_wikidata_id).to.equal(null); + expect(result.metadata.wikipedia_discard_reason).to.equal('entity-unverified'); + // No LLM call was made for validation (gpt.fetchChatCompletion untouched). + expect(gpt.fetchChatCompletion).to.not.have.been.called; + }); + it('handles SPARQL query failure gracefully', async () => { // Mock Wikidata ID search fetchStub.onFirstCall().resolves({ @@ -1058,9 +1214,37 @@ describe('services/product-extractor', () => { expect(service).to.have.property('extractFromSitemap'); expect(service).to.have.property('extractProducts'); + expect(service).to.have.property('validateEntity'); expect(service).to.have.property('formatProductsForPrompt'); }); + it('validateEntity resolves a verified entity via P856 without an LLM call', async () => { + const env = { + AZURE_OPENAI_ENDPOINT: 'https://example.openai.azure.com', + AZURE_OPENAI_KEY: 'test-key', + AZURE_API_VERSION: '2023-05-15', + AZURE_COMPLETION_DEPLOYMENT: 'gpt-4', + }; + // Wikidata meta whose P856 official website matches the brand domain -> verified. + fetchStub.resolves({ + ok: true, + json: () => Promise.resolve({ + entities: { + Q1: { + claims: { P856: [{ mainsnak: { datavalue: { value: 'https://www.lovesac.com' } } }] }, + descriptions: { en: { value: 'furniture company' } }, + }, + }, + }), + }); + const service = createProductExtractorService(env, log); + const v = await service.validateEntity({ + wikidataId: 'Q1', brandName: 'Lovesac', domain: 'lovesac.com', industry: 'Furniture', + }); + expect(v.status).to.equal('verified'); + expect(v.method).to.equal('official_website'); + }); + it('service methods can be called', async () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://example.openai.azure.com', diff --git a/test/agents/brand-profile/services/wikipedia.test.js b/test/agents/brand-profile/services/wikipedia.test.js index af6ceb5..ad03b09 100644 --- a/test/agents/brand-profile/services/wikipedia.test.js +++ b/test/agents/brand-profile/services/wikipedia.test.js @@ -481,4 +481,134 @@ describe('services/wikipedia', () => { expect(svc).to.have.property('resolveBrand').that.is.a('function'); }); }); + + describe('registrableDomain', () => { + const cases = [ + ['https://www.lovesac.com', 'lovesac.com'], + ['brand.toyota.com', 'toyota.com'], + ['https://prudential.com.au/foo', 'prudential.com.au'], + ['aia.com.hk', 'aia.com.hk'], + ['capella.edu', 'capella.edu'], + ['WWW.Example.CO.UK', 'example.co.uk'], + // unparseable even with the https:// prefix (space in authority) -> bare-hostname fallthrough + ['bad host.com', 'bad host.com'], + ['', null], + [null, null], + ]; + cases.forEach(([input, expected]) => { + it(`reduces ${JSON.stringify(input)} -> ${JSON.stringify(expected)}`, async () => { + const mod = await esmock('../../../../src/agents/brand-profile/services/wikipedia.js', {}); + expect(mod.registrableDomain(input)).to.equal(expected); + }); + }); + }); + + describe('validateEntityMatchesBrand', () => { + const load = () => esmock('../../../../src/agents/brand-profile/services/wikipedia.js', {}); + const metaResp = (claims, description) => { + const descriptions = description ? { en: { value: description } } : {}; + return { + ok: true, + json: () => Promise.resolve({ entities: { Q1: { claims, descriptions } } }), + }; + }; + + it('verifies deterministically when the official website domain matches (only status that publishes)', async () => { + fetchStub.resolves(metaResp( + { P856: [{ mainsnak: { datavalue: { value: 'https://www.lovesac.com/' } } }] }, + 'furniture company', + )); + const gpt = { fetchChatCompletion: sandbox.stub() }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'Lovesac', domain: 'lovesac.com', industry: 'Furniture', + }, gpt, log); + expect(r.status).to.equal('verified'); + expect(r.method).to.equal('official_website'); + expect(r.failOpen).to.equal(false); + expect(gpt.fetchChatCompletion).to.not.have.been.called; + }); + + it('classifies mismatch when no website match and the LLM says no', async () => { + fetchStub.resolves(metaResp({}, 'bright star in the constellation Auriga')); + const gpt = { fetchChatCompletion: sandbox.stub().resolves({ choices: [{ message: { content: 'no' } }] }) }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'Capella', domain: 'capella.edu', industry: 'Education', + }, gpt, log); + expect(r.status).to.equal('mismatch'); + expect(r.method).to.equal('llm'); + expect(r.failOpen).to.equal(false); + }); + + it('holds as unverified on an LLM "yes" (LLM may never auto-pass-to-publish)', async () => { + fetchStub.resolves(metaResp({}, 'an online university')); + const gpt = { fetchChatCompletion: sandbox.stub().resolves({ choices: [{ message: { content: 'Yes.' } }] }) }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'Capella', domain: 'capella.edu', industry: 'Education', + }, gpt, log); + expect(r.status).to.equal('unverified'); + expect(r.method).to.equal('llm-unconfirmed'); + expect(r.failOpen).to.equal(false); + }); + + it('holds as unverified on a garbled/empty LLM answer (never discards on ambiguity)', async () => { + fetchStub.resolves(metaResp({}, 'some description')); + const gpt = { fetchChatCompletion: sandbox.stub().resolves({ choices: [{ message: { content: '' } }] }) }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'X', domain: 'x.com', industry: 'Tech', + }, gpt, log); + expect(r.status).to.equal('unverified'); + expect(r.method).to.equal('llm-inconclusive'); + expect(r.failOpen).to.equal(false); + }); + + it('does NOT treat a hedged "not sure" as a definitive no (holds, not discards)', async () => { + fetchStub.resolves(metaResp({}, 'some description')); + const gpt = { fetchChatCompletion: sandbox.stub().resolves({ choices: [{ message: { content: 'Not sure' } }] }) }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'X', domain: 'x.com', industry: 'Tech', + }, gpt, log); + expect(r.status).to.equal('unverified'); // must NOT be 'mismatch' just because it starts with "n" + expect(r.method).to.equal('llm-inconclusive'); + }); + + it('holds as unverified AND flags fail-open when entity metadata cannot be fetched', async () => { + fetchStub.resolves({ ok: false, status: 500 }); + const gpt = { fetchChatCompletion: sandbox.stub() }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'X', domain: 'x.com', industry: 'Tech', + }, gpt, log); + expect(r.status).to.equal('unverified'); + expect(r.method).to.equal('error-meta-unavailable'); + expect(r.failOpen).to.equal(true); + }); + + it('holds as unverified AND flags fail-open when the LLM call throws', async () => { + fetchStub.resolves(metaResp({}, 'some description')); + const gpt = { fetchChatCompletion: sandbox.stub().rejects(new Error('llm down')) }; + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'X', domain: 'x.com', industry: 'Tech', + }, gpt, log); + expect(r.status).to.equal('unverified'); + expect(r.method).to.equal('error-llm'); + expect(r.failOpen).to.equal(true); + }); + + it('holds as unverified (not fail-open) when no verifier is available and no website matches', async () => { + fetchStub.resolves(metaResp({}, 'some description')); + const mod = await load(); + const r = await mod.validateEntityMatchesBrand({ + wikidataId: 'Q1', brandName: 'X', domain: 'x.com', industry: 'Tech', + }, null, log); + expect(r.status).to.equal('unverified'); + expect(r.method).to.equal('no-verifier'); + expect(r.failOpen).to.equal(false); + }); + }); });