feat: resolve member contributors for unknown ecosystems via llm repo inference

backend/src/api/public/v1/akrites-external/openapi.yaml

@@ -117,6 +117,27 @@ components:
             message:
               type: string
 
+    ConflictError:
+      type: object
+      required: [error]
+      properties:
+        error:
+          type: object
+          required: [code, message]
+          properties:
+            code:
+              type: string
+              example: CONFLICT
+            message:
+              type: string
+            context:
+              type: object
+              properties:
+                jobId:
+                  type: string
+                  format: uuid
+                  description: The job whose result is still readable; poll it.
+
     Ecosystem:
       type: string
       example: npm
@@ -957,6 +978,24 @@ components:
           type: string
           example: Repository not found on GitHub
 
+    MemberContributorsJobUnresolvable:
+      type: object
+      required: [jobId, status, reason, repoUrl]
+      properties:
+        jobId:
+          type: string
+          format: uuid
+        status:
+          type: string
+          enum: [unresolvable]
+        reason:
+          type: string
+          example: LLM confidence 0.4 below threshold 0.7
+        repoUrl:
+          type: string
+          nullable: true
+          description: Repository the job attempted before giving up, when there was one.
+
     MemberContributorsJobDone:
       allOf:
         - $ref: "#/components/schemas/MemberContributorsBody"
@@ -970,17 +1009,29 @@ components:
               type: string
               enum: [done]
               description: Job status; the collected/not_collected status of the sync endpoint is not included in this body.
+            repoUrlSource:
+              type: string
+              enum: [llm_inferred]
+              description: >
+                Present only for jobs on ecosystems CDP does not track: the
+                repository was inferred by a language model and verified to
+                exist on GitHub. repoMappingConfidence is null on such jobs,
+                maintainers.registry is empty, and the body is available for a
+                limited time after completion; afterwards the poll answers
+                failed with a resubmit reason.
 
     MemberContributorsJob:
       oneOf:
         - $ref: "#/components/schemas/MemberContributorsJobPending"
         - $ref: "#/components/schemas/MemberContributorsJobFailed"
+        - $ref: "#/components/schemas/MemberContributorsJobUnresolvable"
         - $ref: "#/components/schemas/MemberContributorsJobDone"
       discriminator:
         propertyName: status
         mapping:
           pending: "#/components/schemas/MemberContributorsJobPending"
           failed: "#/components/schemas/MemberContributorsJobFailed"
+          unresolvable: "#/components/schemas/MemberContributorsJobUnresolvable"
           done: "#/components/schemas/MemberContributorsJobDone"
 
     MemberContributorEntry:
@@ -1281,8 +1332,12 @@ paths:
         submission also makes sure the job's workflow is running. Packages that already have
         collected data are refused with 409, unless the purl's most recent job
         failed, in which case a new job is accepted and re-collects the
-        repository; ecosystems CDP does not collect
-        are refused with 400. Rate-limited independently of the read endpoints.
+        repository. Purls from ecosystems CDP does not collect are accepted and
+        routed to a workflow that infers the repository with a language model;
+        those jobs end done with repoUrlSource llm_inferred or unresolvable,
+        store no contributor data, and a resubmission while the previous result
+        is still readable is refused with 409 whose error.context.jobId names
+        the job to poll. Rate-limited independently of the read endpoints.
         Requires read:akrites-maintainers, or read:maintainer-roles from the old
         Self Serve client as a fallback. A done job whose repository reports no
         human contributors on GitHub leaves the package not_collected;
@@ -1307,7 +1362,7 @@ paths:
               schema:
                 $ref: "#/components/schemas/MemberContributorsJobPending"
         "400":
-          description: Malformed purl, or an ecosystem CDP does not collect.
+          description: Malformed purl.
           content:
             application/json:
               schema:
@@ -1331,11 +1386,11 @@ paths:
               schema:
                 $ref: "#/components/schemas/Error"
         "409":
-          description: Member contributors are already collected for this purl and its most recent job did not fail; use the sync endpoint.
+          description: Member contributors already collected for this purl (stored data on tracked ecosystems, or a still-readable result on inferred-repo jobs; error.context.jobId names it); use the sync endpoint or poll that job.
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Error"
+                $ref: "#/components/schemas/ConflictError"
         "429":
           description: Too many requests — rate-limited independently of the other endpoints.
           content:
@@ -1348,9 +1403,10 @@ paths:
       operationId: getMemberContributorsJob
       summary: Poll an on-demand member-contributors collection job
       description: >
-        Returns pending until the job finishes, failed with a reason, or done
-        with the same body the sync endpoint returns for the purl plus jobId
-        and status.
+        Returns pending until the job finishes, failed with a reason,
+        unresolvable with a reason when no repository could be inferred, or
+        done with the same body the sync endpoint returns for the purl plus
+        jobId and status.
       tags: [Contacts]
       security:
         - M2MBearer:

backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts

@@ -19,7 +19,8 @@ vi.mock('@/db/packagesDb', () => ({
   getPackagesQx: vi.fn().mockResolvedValue({}),
 }))
 
-vi.mock('@crowd/data-access-layer', () => ({
+vi.mock('@crowd/data-access-layer', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@crowd/data-access-layer')>()),
   findMemberContributorsPackage,
   findAffiliatedMemberIdentities,
 }))

backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts

@@ -7,12 +7,10 @@ import { NotFoundError } from '@crowd/common'
 import {
   findAffiliatedMemberIdentities,
   findMemberContributorsPackage,
-} from '@crowd/data-access-layer'
-
-import {
   groupAffiliatedMemberIdentities,
   toAkritesExternalMemberContributors,
-} from './akritesExternalMemberContributors'
+} from '@crowd/data-access-layer'
+
 import { purlQuerySchema } from './purl'
 
 export async function getAkritesExternalMemberContributors(

backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts

@@ -8,18 +8,42 @@ import type {
   MemberContributorsPackageRow,
 } from '@crowd/data-access-layer'
 import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers'
+import { WorkflowFailedError, WorkflowNotFoundError } from '@crowd/temporal'
 
 import { getMemberContributorsJob } from './getMemberContributorsJob'
 
-const { findMemberContributorJob, findMemberContributorsPackage, findAffiliatedMemberIdentities } =
-  vi.hoisted(() => ({
-    findMemberContributorJob: vi.fn(),
-    findMemberContributorsPackage: vi.fn(),
-    findAffiliatedMemberIdentities: vi.fn(),
-  }))
+const {
+  getHandle,
+  result,
+  findMemberContributorJob,
+  findMemberContributorsPackage,
+  findAffiliatedMemberIdentities,
+} = vi.hoisted(() => ({
+  getHandle: vi.fn(),
+  result: vi.fn(),
+  findMemberContributorJob: vi.fn(),
+  findMemberContributorsPackage: vi.fn(),
+  findAffiliatedMemberIdentities: vi.fn(),
+}))
 
+vi.mock('@/db/packagesTemporal', () => ({
+  getPackagesTemporalClient: vi.fn().mockResolvedValue({ workflow: { getHandle } }),
+}))
+vi.mock('@crowd/temporal', () => {
+  class MockWorkflowNotFoundError extends Error {
+    name = 'WorkflowNotFoundError'
+  }
+  class MockWorkflowFailedError extends Error {
+    name = 'WorkflowFailedError'
+  }
+  return {
+    WorkflowNotFoundError: MockWorkflowNotFoundError,
+    WorkflowFailedError: MockWorkflowFailedError,
+  }
+})
 vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}) }))
-vi.mock('@crowd/data-access-layer', () => ({
+vi.mock('@crowd/data-access-layer', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@crowd/data-access-layer')>()),
   findMemberContributorJob,
   findMemberContributorsPackage,
   findAffiliatedMemberIdentities,
@@ -31,10 +55,14 @@ function jobRow(overrides: Partial<MemberContributorJobRow> = {}): MemberContrib
   return {
     id: JOB_ID,
     purl: 'pkg:npm/request-oauth',
+    ecosystem: 'npm',
+    collectionPath: 'github',
     status: 'pending',
     error: null,
     packageId: '2008420',
     repoId: '590455',
+    repoUrl: null,
+    repoUrlSource: null,
     ...overrides,
   }
 }
@@ -81,6 +109,7 @@ function mockReqRes(params: Record<string, unknown>) {
 
 beforeEach(() => {
   vi.clearAllMocks()
+  getHandle.mockReturnValue({ result })
 })
 
 describe('getMemberContributorsJob', () => {
@@ -163,3 +192,104 @@ describe('getMemberContributorsJob', () => {
     await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError)
   })
 })
+
+describe('getMemberContributorsJob, llm repo resolution path', () => {
+  const llmJob = (overrides: Partial<MemberContributorJobRow>) =>
+    jobRow({ collectionPath: 'llm_repo_resolution', ...overrides })
+  const workflowId = `member-contributors-job/${JOB_ID}`
+  const inferredBody = {
+    purl: 'pkg:hackage/aeson',
+    name: 'aeson',
+    ecosystem: 'hackage',
+    repoUrl: 'https://github.com/haskell/aeson',
+    repoUrlSource: 'llm_inferred',
+    repoMappingConfidence: null,
+    contributors: [],
+    maintainers: { registry: [], governanceFiles: [] },
+  }
+
+  it.each([
+    {
+      name: 'reports unresolvable with the reason and attempted repo url, without the workflow',
+      arrange: () =>
+        findMemberContributorJob.mockResolvedValue(
+          llmJob({
+            status: 'unresolvable',
+            error: 'LLM confidence 0.4 below threshold 0.7',
+            repoUrl: null,
+          }),
+        ),
+      response: {
+        jobId: JOB_ID,
+        status: 'unresolvable',
+        reason: 'LLM confidence 0.4 below threshold 0.7',
+        repoUrl: null,
+      },
+      workflowIds: [],
+      packageReads: [],
+    },
+    {
+      name: 'returns the workflow result as the done body',
+      arrange: () => {
+        findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' }))
+        result.mockResolvedValue(inferredBody)
+      },
+      response: { ...inferredBody, jobId: JOB_ID, status: 'done' },
+      workflowIds: [[workflowId]],
+      packageReads: [],
+    },
+    {
+      name: 'reports failed with the resubmit reason when the workflow result is gone',
+      arrange: () => {
+        findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' }))
+        result.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined))
+      },
+      response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' },
+      workflowIds: [[workflowId]],
+      packageReads: [],
+    },
+    {
+      name: 'reports failed with the resubmit reason when the workflow ended abnormally',
+      arrange: () => {
+        findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' }))
+        result.mockRejectedValue(
+          new WorkflowFailedError('terminated', undefined, 'NON_RETRYABLE_FAILURE'),
+        )
+      },
+      response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' },
+      workflowIds: [[workflowId]],
+      packageReads: [],
+    },
+    {
+      name: 'keeps reading github path done jobs from the tables, not the workflow',
+      arrange: () => {
+        findMemberContributorJob.mockResolvedValue(jobRow({ status: 'done' }))
+        findMemberContributorsPackage.mockResolvedValue(packageRow())
+        findAffiliatedMemberIdentities.mockResolvedValue([identityRow()])
+      },
+      response: expect.objectContaining({
+        jobId: JOB_ID,
+        status: 'done',
+        purl: 'pkg:npm/request-oauth',
+      }),
+      workflowIds: [],
+      packageReads: [[{}, 'pkg:npm/request-oauth']],
+    },
+  ])('$name', async ({ arrange, response, workflowIds, packageReads }) => {
+    arrange()
+    const { req, res, json } = mockReqRes({ jobId: JOB_ID })
+    await getMemberContributorsJob(req, res)
+    expect({
+      responses: json.mock.calls,
+      workflowIds: getHandle.mock.calls,
+      packageReads: findMemberContributorsPackage.mock.calls,
+    }).toEqual({ responses: [[response]], workflowIds, packageReads })
+  })
+
+  it('throws an internal error when a done workflow returned no body', async () => {
+    findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' }))
+    result.mockResolvedValue(undefined)
+    const { req, res } = mockReqRes({ jobId: JOB_ID })
+    await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError)
+  })
+})