fix(brand-profile): hold entity-validation failures as unverified, not fail-open (#3268, re-opens #361)

README.md

@@ -50,6 +50,7 @@ The `agent-executor` (and the provided brand-profile agent) rely on the Azure Op
 | `AZURE_OPENAI_KEY` | API key for the Azure OpenAI resource |
 | `AZURE_API_VERSION` | API version used for the chat completions |
 | `AZURE_COMPLETION_DEPLOYMENT` | Deployment/model name (e.g., `gpt-4o`) |
+| `BRAND_PROFILE_ENTITY_VALIDATION_LLM` | Enable the LLM leg of the Class-B entity-validation gate. Defaults **on**; set to `false` to disable it (a non-matching entity then holds as `entity_validation: unverified` instead of being LLM-triaged to a hard mismatch). The deterministic P856 domain-match always runs regardless. |
 
 When invoking the integration test, you can also set `BRAND_PROFILE_TEST_BASE_URL` to control which site is analyzed and `BRAND_PROFILE_IT_FULL=1` to print the complete agent response (otherwise the preview is truncated for readability).
 - To lint code:

src/agents/brand-profile/index.js

@@ -176,7 +176,7 @@ async function run(context, env, log) {
   // name is the unknown-brand sentinel (a name search would contaminate).
   const needsWikipedia = !hasText(sitemapUrl)
     || !(Array.isArray(llmoCompetitors) && llmoCompetitors.length > 0);
-  const brandWiki = (brandName !== UNKNOWN_BRAND && needsWikipedia)
+  let brandWiki = (brandName !== UNKNOWN_BRAND && needsWikipedia)
     ? await wikipediaService.resolveBrand(brandName)
     : {
       wikidataId: null,
@@ -187,6 +187,45 @@ async function run(context, env, log) {
       discardReason: 'skipped',
     };
 
+  // Phase 3.5: Validate the resolved Wikidata entity ONCE, before any QID-derived
+  // context is consumed. The QID->article guard cannot catch a QID resolved to the
+  // wrong same-named entity (capella.edu -> Q12970, the star). Validating here -
+  // ahead of competitor inference, personas AND product extraction - ensures a wrong
+  // entity's Wikipedia summary/text never contaminates ANY downstream output (not
+  // just products). Only a 'verified' entity (authoritative P856 domain match) is
+  // trusted; 'mismatch'/'unverified' withhold its context fleet-wide.
+  let entityValidation = null;
+  if (brandWiki.wikidataId) {
+    const domain = new URL(baseURL).hostname;
+    entityValidation = await productService.validateEntity({
+      wikidataId: brandWiki.wikidataId, brandName, domain, industry,
+    });
+    log.info('brand-profile: entity-validation-outcome', {
+      metric: 'entity_validation_outcome',
+      wikidataId: brandWiki.wikidataId,
+      brand: brandName,
+      domain,
+      status: entityValidation.status,
+      method: entityValidation.method,
+      failOpen: entityValidation.failOpen === true,
+    });
+    if (entityValidation.failOpen) {
+      log.warn('brand-profile: entity-validation-fail-open', {
+        metric: 'entity_validation_fail_open',
+        wikidataId: brandWiki.wikidataId,
+        brand: brandName,
+        domain,
+        method: entityValidation.method,
+      });
+    }
+    if (entityValidation.status !== 'verified') {
+      log.warn(`brand-profile: entity ${brandWiki.wikidataId} for "${brandName}" (${baseURL}) is ${entityValidation.status} via ${entityValidation.method} - withholding Wikidata/Wikipedia context from all consumers`);
+      brandWiki = {
+        ...brandWiki, wikidataId: null, fullText: '', summary: '',
+      };
+    }
+  }
+
   // Phase 2: Infer region from URL
   log.info('brand-profile: inferring region from URL');
   const regionInference = await regionalService.inferRegionFromUrl(baseURL);
@@ -251,9 +290,15 @@ async function run(context, env, log) {
     productsResult = await productService.extractFromSitemap(sitemapUrl, brandName);
   } else {
     // Use Wikidata + QID-anchored Wikipedia extraction (resolved once, above).
+    // entityValidation was already computed in Phase 3.5 and brandWiki gated on it,
+    // so pass the verdict through: the extractor records it and skips re-validating
+    // (no second LLM call). A non-verified entity already has wikidataId/text nulled.
     productsResult = await productService.extractProducts(brandName, {
       wikidataId: brandWiki.wikidataId,
       wikipediaText: brandWiki.fullText,
+      domain: new URL(baseURL).hostname,
+      industry,
+      entityValidation,
     });
   }
 

src/agents/brand-profile/services/product-extractor.js

@@ -23,7 +23,7 @@
 
 import { AzureOpenAIClient } from '@adobe/spacecat-shared-gpt-client';
 import { readPromptFile, renderTemplate } from '../../base.js';
-import { findWikidataId, resolveBrandWikipedia } from './wikipedia.js';
+import { findWikidataId, resolveBrandWikipedia, validateEntityMatchesBrand } from './wikipedia.js';
 
 const USER_AGENT = 'SpaceCat/1.0 (https://github.com/adobe/spacecat; spacecat@adobe.com)';
 const WIKIDATA_SPARQL = 'https://query.wikidata.org/sparql';
@@ -513,8 +513,80 @@ export async function extractProducts(brandName, wikipediaContext, gpt, log) {
     },
   };
 
-  // Step 1: Find brand's Wikidata ID (reuse the caller's when provided)
-  const wikidataId = ctxQid || await findWikidataId(brandName, log);
+  // Step 1: Find brand's Wikidata ID (reuse the caller's when provided).
+  // When the caller passes a precomputed entityValidation verdict (index.js), it has
+  // ALREADY decided the QID (nulled if not verified) - respect it and do NOT
+  // name-search a replacement, because that name search is itself a contamination
+  // vector (it can resolve the wrong same-named entity all over again).
+  let wikidataId = ctx.entityValidation
+    ? ctxQid
+    : (ctxQid || await findWikidataId(brandName, log));
+
+  // Step 1b: Validate the resolved entity actually is this brand before trusting
+  // any of its data. The QID->article guard cannot catch a QID that was itself
+  // resolved to the wrong same-named entity (e.g. capella.edu -> Q12970, the star
+  // Capella). Only a 'verified' outcome (authoritative P856 domain match) may
+  // publish; both 'mismatch' (wrong entity) and 'unverified' (couldn't confirm /
+  // infra error) withhold the QID-derived data. Requires the brand domain;
+  // industry is optional (it only sharpens the LLM branch), so a missing industry
+  // no longer skips the check and lets contamination through. When no domain is
+  // supplied the check is skipped and behaviour is unchanged.
+  let entityStatus = null;
+  if (ctx.entityValidation) {
+    // Validation was already performed upstream (index.js Phase 3.5) BEFORE any
+    // consumer saw the QID context, so a wrong same-named entity never reached
+    // competitor/persona inference either. Reuse it verbatim - don't re-pay for the
+    // LLM call or re-emit the metric (index.js owns that for the hoisted path).
+    result.metadata.entity_validation = ctx.entityValidation;
+    entityStatus = ctx.entityValidation.status;
+    if (entityStatus !== 'verified') {
+      wikidataId = null;
+    }
+  } else if (wikidataId && ctx.domain) {
+    // Standalone path (direct callers / tests): validate here. The deterministic
+    // P856 domain-match always runs (free); the LLM fallback is the only token cost
+    // and only fires when P856 doesn't match. Flag off (env
+    // BRAND_PROFILE_ENTITY_VALIDATION_LLM=false) → no verifier → a non-matching
+    // entity holds as 'unverified' rather than being LLM-triaged to 'mismatch'.
+    const validationGpt = ctx.entityValidationUseLlm === false ? null : gpt;
+    const validation = await validateEntityMatchesBrand(
+      {
+        wikidataId, brandName, domain: ctx.domain, industry: ctx.industry || '',
+      },
+      validationGpt,
+      log,
+    );
+    result.metadata.entity_validation = validation;
+    entityStatus = validation.status;
+
+    // Alarmable fail-open-rate metric. This repo surfaces observability through
+    // structured log lines (CloudWatch metric filters), not a metrics SDK - emit
+    // one line per validation so a filter can compute fail-open / total, plus a
+    // distinct warn line for the fail-open subset that was previously silent.
+    log.info('brand-profile: entity-validation-outcome', {
+      metric: 'entity_validation_outcome',
+      wikidataId,
+      brand: brandName,
+      domain: ctx.domain,
+      status: validation.status,
+      method: validation.method,
+      failOpen: validation.failOpen === true,
+    });
+    if (validation.failOpen) {
+      log.warn('brand-profile: entity-validation-fail-open', {
+        metric: 'entity_validation_fail_open',
+        wikidataId,
+        brand: brandName,
+        domain: ctx.domain,
+        method: validation.method,
+      });
+    }
+
+    if (validation.status !== 'verified') {
+      log.warn(`brand-profile: entity ${wikidataId} for brand "${brandName}" (${ctx.domain}) is ${validation.status} via ${validation.method} - withholding Wikidata/Wikipedia sources`);
+      wikidataId = null;
+    }
+  }
 
   if (wikidataId) {
     result.metadata.brand_wikidata_id = wikidataId;
@@ -536,7 +608,16 @@ export async function extractProducts(brandName, wikipediaContext, gpt, log) {
     log.info(`Wikidata returned ${result.products.length} products (threshold: ${MIN_PRODUCTS_THRESHOLD}), trying Wikipedia fallback`);
 
     let wikiText;
-    if (hasProvidedText) {
+    if (entityStatus && entityStatus !== 'verified') {
+      // The resolved entity did not authoritatively verify - never use its article
+      // text, even if the caller passed some (it would be the wrong/unconfirmed
+      // entity's article). 'mismatch' = known-wrong; 'unverified' = hold for review.
+      wikiText = '';
+      result.metadata.wikipedia_verified = false;
+      result.metadata.wikipedia_discard_reason = entityStatus === 'mismatch'
+        ? 'entity-brand-mismatch'
+        : 'entity-unverified';
+    } else if (hasProvidedText) {
       // Caller supplied the exact article text. Trust assumption: the caller is
       // responsible for having QID-anchored/verified this text (index.js resolves
       // it via resolveBrandWikipedia and only passes non-empty text when verified).
@@ -630,6 +711,7 @@ export function formatProductsForPrompt(extractionResult) {
  */
 export function createProductExtractorService(env, log) {
   const gpt = AzureOpenAIClient.createFrom({ env, log });
+  const entityValidationUseLlm = env.BRAND_PROFILE_ENTITY_VALIDATION_LLM !== 'false';
 
   return {
     extractFromSitemap: (sitemapUrl, brandName) => (
@@ -638,6 +720,19 @@ export function createProductExtractorService(env, log) {
     extractProducts: (brandName, wikipediaContext) => (
       extractProducts(brandName, wikipediaContext, gpt, log)
     ),
+    // Validate a resolved Wikidata entity up front so a wrong same-named QID can be
+    // withheld from EVERY consumer (competitors, personas, products), not just
+    // product extraction. Uses the shared gpt + the LLM flag; returns the
+    // {status, method, reason, failOpen} verdict for the caller to act on.
+    validateEntity: ({
+      wikidataId, brandName, domain, industry,
+    }) => validateEntityMatchesBrand(
+      {
+        wikidataId, brandName, domain, industry: industry || '',
+      },
+      entityValidationUseLlm ? gpt : null,
+      log,
+    ),
     formatProductsForPrompt,
   };
 }