feat: map octolens mentions from ten platforms per the ingestion spec

backend/src/database/migrations/V1792094459__add_octolens_mention_activity_types.sql

@@ -0,0 +1,25 @@
+-- The table has no unique key, so skip rows that already exist (e.g. mention/twitter).
+INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description, "label")
+SELECT v."activityType", v.platform, false, false, v.description, v."label"
+FROM (VALUES
+    ('mention', 'github', 'Mentioned the project on GitHub', 'Mentioned the project'),
+    ('mention', 'twitter', 'Mentioned the project on X/Twitter', 'Mentioned the project'),
+    ('mention', 'linkedin', 'Mentioned the project on LinkedIn', 'Mentioned the project'),
+    ('mention', 'reddit', 'Mentioned the project on Reddit', 'Mentioned the project'),
+    ('mention', 'devto', 'Mentioned the project on DEV', 'Mentioned the project'),
+    ('mention', 'hackernews', 'Mentioned the project on Hacker News', 'Mentioned the project'),
+    ('mention', 'stackoverflow', 'Mentioned the project on Stack Overflow', 'Mentioned the project'),
+    ('mention', 'medium', 'Mentioned the project on Medium', 'Mentioned the project'),
+    ('mention', 'producthunt', 'Mentioned the project on Product Hunt', 'Mentioned the project'),
+    ('mention', 'facebook', 'Mentioned the project on Facebook', 'Mentioned the project'),
+    ('mention', 'youtube', 'Mentioned the project on YouTube', 'Mentioned the project'),
+    ('mention', 'bluesky', 'Mentioned the project on Bluesky', 'Mentioned the project'),
+    ('mention', 'tiktok', 'Mentioned the project on TikTok', 'Mentioned the project'),
+    ('mention', 'instagram', 'Mentioned the project on Instagram', 'Mentioned the project')
+) AS v("activityType", platform, description, "label")
+WHERE NOT EXISTS (
+    SELECT 1
+    FROM "activityTypes" a
+    WHERE a."activityType" = v."activityType"
+      AND a.platform = v.platform
+);

services/apps/data_sink_worker/src/service/activity.service.ts

@@ -619,7 +619,10 @@ export default class ActivityService extends LoggerBase {
         continue
       }
 
-      if (payload.platform === PlatformType.GITLAB || payload.platform === PlatformType.GITHUB) {
+      if (
+        !payload.skipRepositorySegmentLookup &&
+        (payload.platform === PlatformType.GITLAB || payload.platform === PlatformType.GITHUB)
+      ) {
         repoPayloads.push(payload)
       } else if (!payload.segmentId) {
         resultMap.set(payload.resultId, {
@@ -2144,6 +2147,7 @@ interface IActivityProcessData {
   platform: PlatformType
   activity: IActivityData
   segmentId: string
+  skipRepositorySegmentLookup?: boolean
 
   dbMember?: IDbMember
   dbObjectMember?: IDbMember

services/apps/data_sink_worker/src/service/dataSink.service.ts

@@ -31,7 +31,10 @@ import {
 import { WORKER_SETTINGS } from '../conf'
 import ActivityService from './activity.service'
 import MemberService from './member.service'
-import { resolveActivityResultPlatform } from './resolveActivityResultPlatform'
+import {
+  resolveActivityResultPlatform,
+  skipsRepositorySegmentLookup,
+} from './resolveActivityResultPlatform'
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
@@ -351,7 +354,11 @@ export default class DataSinkService extends LoggerBase {
       if (type === IntegrationResultType.ACTIVITY) {
         const results = groupedByType.get(type)
 
+        const keepsResultSegment = new Set<string>()
         for (const result of results) {
+          if (skipsRepositorySegmentLookup(result.platform)) {
+            keepsResultSegment.add(result.id)
+          }
           result.platform = resolveActivityResultPlatform(
             result.platform,
             (result.data.data as IActivityData | undefined)?.platform as PlatformType | undefined,
@@ -393,6 +400,7 @@ export default class DataSinkService extends LoggerBase {
                   platform: r.platform,
                   activity: r.data.data as IActivityData,
                   segmentId: r.data.segmentId ?? r.segmentId,
+                  skipRepositorySegmentLookup: keepsResultSegment.has(r.id),
                 }
               }),
               toProcess.some((r) => r.onboarding === true),

services/apps/data_sink_worker/src/service/resolveActivityResultPlatform.test.ts

@@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest'
 
 import { PlatformType } from '@crowd/types'
 
-import { resolveActivityResultPlatform } from './resolveActivityResultPlatform'
+import {
+  resolveActivityResultPlatform,
+  skipsRepositorySegmentLookup,
+} from './resolveActivityResultPlatform'
 
 describe('resolveActivityResultPlatform', () => {
   it('prefers the activity platform over the integration platform when both are set', () => {
@@ -29,3 +32,16 @@ describe('resolveActivityResultPlatform', () => {
     expect(resolveActivityResultPlatform(null, undefined)).toBeNull()
   })
 })
+
+describe('skipsRepositorySegmentLookup', () => {
+  it('skips the repositories lookup for activities from an Octolens integration', () => {
+    expect(skipsRepositorySegmentLookup(PlatformType.OCTOLENS)).toBe(true)
+  })
+
+  it.each([PlatformType.GITHUB, PlatformType.GITHUB_NANGO, PlatformType.GITLAB, null])(
+    'keeps the lookup for %s integrations',
+    (platform) => {
+      expect(skipsRepositorySegmentLookup(platform)).toBe(false)
+    },
+  )
+})

services/apps/data_sink_worker/src/service/resolveActivityResultPlatform.ts

@@ -9,3 +9,8 @@ export function resolveActivityResultPlatform(
   const platform = activityPlatform ?? integrationPlatform
   return platform === PlatformType.GITHUB_NANGO ? PlatformType.GITHUB : platform
 }
+
+// Octolens routes each mention to its keyword's project, never through public.repositories.
+export function skipsRepositorySegmentLookup(integrationPlatform: PlatformType | null): boolean {
+  return integrationPlatform === PlatformType.OCTOLENS
+}

services/apps/organizations_enrichment_worker/src/sources/lfx-internal-api/service.ts

@@ -20,6 +20,13 @@ import {
   IOrganizationEnrichmentDataInternalAPIResponse,
 } from './types'
 
+// Never stored before these PlatformType values existed; storing them needs its own decision.
+const IGNORED_IDENTITY_PLATFORMS = new Set<PlatformType>([
+  PlatformType.BLUESKY,
+  PlatformType.TIKTOK,
+  PlatformType.INSTAGRAM,
+])
+
 export default class EnrichmentServiceLFXInternalAPI
   extends LoggerBase
   implements IOrganizationEnrichmentService
@@ -126,6 +133,10 @@ export default class EnrichmentServiceLFXInternalAPI
             this.log.warn({ key }, 'Unknown platform in identities, skipping')
             return []
           }
+          if (IGNORED_IDENTITY_PLATFORMS.has(platform)) {
+            this.log.warn({ key }, 'Ignored platform in identities, skipping')
+            return []
+          }
 
           return (Array.isArray(values) ? values : [values])
             .filter((v) => v && typeof v === 'string')