diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index c241337ff6..7a7aec93ee 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -117,6 +117,27 @@ components: message: type: string + ConflictError: + type: object + required: [error] + properties: + error: + type: object + required: [code, message] + properties: + code: + type: string + example: CONFLICT + message: + type: string + context: + type: object + properties: + jobId: + type: string + format: uuid + description: The job whose result is still readable; poll it. + Ecosystem: type: string example: npm @@ -957,6 +978,24 @@ components: type: string example: Repository not found on GitHub + MemberContributorsJobUnresolvable: + type: object + required: [jobId, status, reason, repoUrl] + properties: + jobId: + type: string + format: uuid + status: + type: string + enum: [unresolvable] + reason: + type: string + example: LLM confidence 0.4 below threshold 0.7 + repoUrl: + type: string + nullable: true + description: Repository the job attempted before giving up, when there was one. + MemberContributorsJobDone: allOf: - $ref: "#/components/schemas/MemberContributorsBody" @@ -970,17 +1009,29 @@ components: type: string enum: [done] description: Job status; the collected/not_collected status of the sync endpoint is not included in this body. + repoUrlSource: + type: string + enum: [llm_inferred] + description: > + Present only for jobs on ecosystems CDP does not track: the + repository was inferred by a language model and verified to + exist on GitHub. repoMappingConfidence is null on such jobs, + maintainers.registry is empty, and the body is available for a + limited time after completion; afterwards the poll answers + failed with a resubmit reason. MemberContributorsJob: oneOf: - $ref: "#/components/schemas/MemberContributorsJobPending" - $ref: "#/components/schemas/MemberContributorsJobFailed" + - $ref: "#/components/schemas/MemberContributorsJobUnresolvable" - $ref: "#/components/schemas/MemberContributorsJobDone" discriminator: propertyName: status mapping: pending: "#/components/schemas/MemberContributorsJobPending" failed: "#/components/schemas/MemberContributorsJobFailed" + unresolvable: "#/components/schemas/MemberContributorsJobUnresolvable" done: "#/components/schemas/MemberContributorsJobDone" MemberContributorEntry: @@ -1281,8 +1332,12 @@ paths: submission also makes sure the job's workflow is running. Packages that already have collected data are refused with 409, unless the purl's most recent job failed, in which case a new job is accepted and re-collects the - repository; ecosystems CDP does not collect - are refused with 400. Rate-limited independently of the read endpoints. + repository. Purls from ecosystems CDP does not collect are accepted and + routed to a workflow that infers the repository with a language model; + those jobs end done with repoUrlSource llm_inferred or unresolvable, + store no contributor data, and a resubmission while the previous result + is still readable is refused with 409 whose error.context.jobId names + the job to poll. Rate-limited independently of the read endpoints. Requires read:akrites-maintainers, or read:maintainer-roles from the old Self Serve client as a fallback. A done job whose repository reports no human contributors on GitHub leaves the package not_collected; @@ -1307,7 +1362,7 @@ paths: schema: $ref: "#/components/schemas/MemberContributorsJobPending" "400": - description: Malformed purl, or an ecosystem CDP does not collect. + description: Malformed purl. content: application/json: schema: @@ -1331,11 +1386,11 @@ paths: schema: $ref: "#/components/schemas/Error" "409": - description: Member contributors are already collected for this purl and its most recent job did not fail; use the sync endpoint. + description: Member contributors already collected for this purl (stored data on tracked ecosystems, or a still-readable result on inferred-repo jobs; error.context.jobId names it); use the sync endpoint or poll that job. content: application/json: schema: - $ref: "#/components/schemas/Error" + $ref: "#/components/schemas/ConflictError" "429": description: Too many requests — rate-limited independently of the other endpoints. content: @@ -1348,9 +1403,10 @@ paths: operationId: getMemberContributorsJob summary: Poll an on-demand member-contributors collection job description: > - Returns pending until the job finishes, failed with a reason, or done - with the same body the sync endpoint returns for the purl plus jobId - and status. + Returns pending until the job finishes, failed with a reason, + unresolvable with a reason when no repository could be inferred, or + done with the same body the sync endpoint returns for the purl plus + jobId and status. tags: [Contacts] security: - M2MBearer: diff --git a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts index e3c6abb001..7ff262d040 100644 --- a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts +++ b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts @@ -19,7 +19,8 @@ vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}), })) -vi.mock('@crowd/data-access-layer', () => ({ +vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ + ...(await importOriginal()), findMemberContributorsPackage, findAffiliatedMemberIdentities, })) diff --git a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts index f406196201..76ccc38844 100644 --- a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts +++ b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts @@ -7,12 +7,10 @@ import { NotFoundError } from '@crowd/common' import { findAffiliatedMemberIdentities, findMemberContributorsPackage, -} from '@crowd/data-access-layer' - -import { groupAffiliatedMemberIdentities, toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' +} from '@crowd/data-access-layer' + import { purlQuerySchema } from './purl' export async function getAkritesExternalMemberContributors( diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index 456154046e..732aaf7722 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -8,18 +8,42 @@ import type { MemberContributorsPackageRow, } from '@crowd/data-access-layer' import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' +import { WorkflowFailedError, WorkflowNotFoundError } from '@crowd/temporal' import { getMemberContributorsJob } from './getMemberContributorsJob' -const { findMemberContributorJob, findMemberContributorsPackage, findAffiliatedMemberIdentities } = - vi.hoisted(() => ({ - findMemberContributorJob: vi.fn(), - findMemberContributorsPackage: vi.fn(), - findAffiliatedMemberIdentities: vi.fn(), - })) +const { + getHandle, + result, + findMemberContributorJob, + findMemberContributorsPackage, + findAffiliatedMemberIdentities, +} = vi.hoisted(() => ({ + getHandle: vi.fn(), + result: vi.fn(), + findMemberContributorJob: vi.fn(), + findMemberContributorsPackage: vi.fn(), + findAffiliatedMemberIdentities: vi.fn(), +})) +vi.mock('@/db/packagesTemporal', () => ({ + getPackagesTemporalClient: vi.fn().mockResolvedValue({ workflow: { getHandle } }), +})) +vi.mock('@crowd/temporal', () => { + class MockWorkflowNotFoundError extends Error { + name = 'WorkflowNotFoundError' + } + class MockWorkflowFailedError extends Error { + name = 'WorkflowFailedError' + } + return { + WorkflowNotFoundError: MockWorkflowNotFoundError, + WorkflowFailedError: MockWorkflowFailedError, + } +}) vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}) })) -vi.mock('@crowd/data-access-layer', () => ({ +vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ + ...(await importOriginal()), findMemberContributorJob, findMemberContributorsPackage, findAffiliatedMemberIdentities, @@ -31,10 +55,14 @@ function jobRow(overrides: Partial = {}): MemberContrib return { id: JOB_ID, purl: 'pkg:npm/request-oauth', + ecosystem: 'npm', + collectionPath: 'github', status: 'pending', error: null, packageId: '2008420', repoId: '590455', + repoUrl: null, + repoUrlSource: null, ...overrides, } } @@ -81,6 +109,7 @@ function mockReqRes(params: Record) { beforeEach(() => { vi.clearAllMocks() + getHandle.mockReturnValue({ result }) }) describe('getMemberContributorsJob', () => { @@ -163,3 +192,104 @@ describe('getMemberContributorsJob', () => { await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError) }) }) + +describe('getMemberContributorsJob, llm repo resolution path', () => { + const llmJob = (overrides: Partial) => + jobRow({ collectionPath: 'llm_repo_resolution', ...overrides }) + const workflowId = `member-contributors-job/${JOB_ID}` + const inferredBody = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + + it.each([ + { + name: 'reports unresolvable with the reason and attempted repo url, without the workflow', + arrange: () => + findMemberContributorJob.mockResolvedValue( + llmJob({ + status: 'unresolvable', + error: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }), + ), + response: { + jobId: JOB_ID, + status: 'unresolvable', + reason: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }, + workflowIds: [], + packageReads: [], + }, + { + name: 'returns the workflow result as the done body', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockResolvedValue(inferredBody) + }, + response: { ...inferredBody, jobId: JOB_ID, status: 'done' }, + workflowIds: [[workflowId]], + packageReads: [], + }, + { + name: 'reports failed with the resubmit reason when the workflow result is gone', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) + }, + response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' }, + workflowIds: [[workflowId]], + packageReads: [], + }, + { + name: 'reports failed with the resubmit reason when the workflow ended abnormally', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockRejectedValue( + new WorkflowFailedError('terminated', undefined, 'NON_RETRYABLE_FAILURE'), + ) + }, + response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' }, + workflowIds: [[workflowId]], + packageReads: [], + }, + { + name: 'keeps reading github path done jobs from the tables, not the workflow', + arrange: () => { + findMemberContributorJob.mockResolvedValue(jobRow({ status: 'done' })) + findMemberContributorsPackage.mockResolvedValue(packageRow()) + findAffiliatedMemberIdentities.mockResolvedValue([identityRow()]) + }, + response: expect.objectContaining({ + jobId: JOB_ID, + status: 'done', + purl: 'pkg:npm/request-oauth', + }), + workflowIds: [], + packageReads: [[{}, 'pkg:npm/request-oauth']], + }, + ])('$name', async ({ arrange, response, workflowIds, packageReads }) => { + arrange() + const { req, res, json } = mockReqRes({ jobId: JOB_ID }) + await getMemberContributorsJob(req, res) + expect({ + responses: json.mock.calls, + workflowIds: getHandle.mock.calls, + packageReads: findMemberContributorsPackage.mock.calls, + }).toEqual({ responses: [[response]], workflowIds, packageReads }) + }) + + it('throws an internal error when a done workflow returned no body', async () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockResolvedValue(undefined) + const { req, res } = mockReqRes({ jobId: JOB_ID }) + await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError) + }) +}) diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts index 0f0704ec87..87f9681751 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts @@ -2,27 +2,33 @@ import type { Request, Response } from 'express' import { z } from 'zod' import { getPackagesQx } from '@/db/packagesDb' +import { getPackagesTemporalClient } from '@/db/packagesTemporal' import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' import { InternalError, NotFoundError } from '@crowd/common' import { + AkritesExternalMemberContributors, + InferredRepoMemberContributorsBody, MemberContributorJobRow, findAffiliatedMemberIdentities, findMemberContributorJob, findMemberContributorsPackage, + groupAffiliatedMemberIdentities, + toAkritesExternalMemberContributors, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { WorkflowFailedError, WorkflowNotFoundError } from '@crowd/temporal' -import { - AkritesExternalMemberContributors, - groupAffiliatedMemberIdentities, - toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' import { AkritesExternalMemberContributorsJob, + AkritesExternalMemberContributorsJobDone, + AkritesExternalMemberContributorsJobFailed, + RESULT_EXPIRED_REASON, + memberContributorsJobWorkflowId, toDoneMemberContributorsJob, toFailedMemberContributorsJob, toPendingMemberContributorsJob, + toUnresolvableMemberContributorsJob, } from './memberContributorsJob' const paramsSchema = z.object({ jobId: z.uuid() }) @@ -43,17 +49,62 @@ async function readCollectedBody( ) } +function assertNever(value: never): never { + throw new InternalError(`Unhandled job status ${String(value)}`) +} + +async function readInferredRepoBody( + jobId: string, +): Promise { + const packagesTemporal = await getPackagesTemporalClient() + let body: InferredRepoMemberContributorsBody | undefined + try { + body = await packagesTemporal.workflow + .getHandle<() => Promise>( + memberContributorsJobWorkflowId(jobId), + ) + .result() + } catch (err) { + if (err instanceof WorkflowNotFoundError || err instanceof WorkflowFailedError) { + return null + } + throw err + } + if (!body) { + throw new InternalError(`Done member contributors job ${jobId} has no workflow result`) + } + return body +} + +async function toDoneResponse( + qx: QueryExecutor, + job: MemberContributorJobRow, +): Promise { + if (job.collectionPath !== 'llm_repo_resolution') { + return toDoneMemberContributorsJob(job.id, await readCollectedBody(qx, job.purl)) + } + const body = await readInferredRepoBody(job.id) + return body + ? toDoneMemberContributorsJob(job.id, body) + : toFailedMemberContributorsJob(job.id, RESULT_EXPIRED_REASON) +} + async function toJobResponse( qx: QueryExecutor, job: MemberContributorJobRow, ): Promise { switch (job.status) { case 'done': - return toDoneMemberContributorsJob(job.id, await readCollectedBody(qx, job.purl)) + return toDoneResponse(qx, job) case 'failed': return toFailedMemberContributorsJob(job.id, job.error) - default: + case 'unresolvable': + return toUnresolvableMemberContributorsJob(job.id, job.error, job.repoUrl) + case 'pending': + case 'running': return toPendingMemberContributorsJob(job.id) + default: + return assertNever(job.status) } } diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index 60a50cf6bf..194599eb9a 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest' import { MEMBER_CONTRIBUTOR_COLLECTION_PATHS, collectionPathForPurlType, + isWellFormedPurl, purlType, } from './memberContributorCollectionPaths' @@ -28,13 +29,27 @@ describe('collectionPathForPurlType', () => { expect(Object.keys(MEMBER_CONTRIBUTOR_COLLECTION_PATHS)).toHaveLength(8) }) - it('returns null for purl types we do not collect', () => { - expect(collectionPathForPurlType('hackage')).toBeNull() - expect(collectionPathForPurlType('conan')).toBeNull() - expect(collectionPathForPurlType('')).toBeNull() - expect(collectionPathForPurlType('constructor')).toBeNull() - expect(collectionPathForPurlType('go')).toBeNull() - expect(collectionPathForPurlType('packagist')).toBeNull() - expect(collectionPathForPurlType('rubygems')).toBeNull() + it.each(['hackage', 'conan', 'hasOwnProperty'])( + 'routes the %s purl type outside the map to llm repo resolution', + (type) => { + expect(collectionPathForPurlType(type)).toBe('llm_repo_resolution') + }, + ) +}) + +describe('isWellFormedPurl', () => { + it.each([ + ['pkg:hackage/aeson', true], + ['pkg:conan/openssl.org/openssl', true], + ['pkg:Hackage/aeson', true], + ['pkg:npm/%40babel/core', true], + ['pkg:', false], + ['pkg:hackage', false], + ['pkg:hackage/', false], + ['pkg:/x', false], + ['pkg:hackage/aeson%ZZ', false], + ['pkg:hackage/aeson%FF', false], + ])('requires a type, a name segment and valid percent encoding: %s is %s', (purl, wellFormed) => { + expect(isWellFormedPurl(purl)).toBe(wellFormed) }) }) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts index 6ec1450618..0ec826d758 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts @@ -19,8 +19,25 @@ export function purlType(purl: string): string { return type.toLowerCase() } -export function collectionPathForPurlType(type: string): MemberContributorCollectionPath | null { +const WELL_FORMED_PURL = /^pkg:[a-z][a-z0-9.+-]*\/[^/?#]+/i + +function isDecodable(value: string): boolean { + try { + decodeURIComponent(value) + return true + } catch { + return false + } +} + +export function isWellFormedPurl(purl: string): boolean { + return WELL_FORMED_PURL.test(purl) && isDecodable(purl.slice('pkg:'.length)) +} + +export const LLM_REPO_RESOLUTION_PATH: MemberContributorCollectionPath = 'llm_repo_resolution' + +export function collectionPathForPurlType(type: string): MemberContributorCollectionPath { return Object.hasOwn(MEMBER_CONTRIBUTOR_COLLECTION_PATHS, type) ? MEMBER_CONTRIBUTOR_COLLECTION_PATHS[type] - : null + : LLM_REPO_RESOLUTION_PATH } diff --git a/backend/src/api/public/v1/packages/memberContributorsJob.ts b/backend/src/api/public/v1/packages/memberContributorsJob.ts index 82a36af8b6..00e73ffbd6 100644 --- a/backend/src/api/public/v1/packages/memberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/memberContributorsJob.ts @@ -1,9 +1,14 @@ +import type { + AkritesExternalMemberContributors, + InferredRepoMemberContributorsBody, +} from '@crowd/data-access-layer' import { TemporalWorkflowId } from '@crowd/types' -import type { AkritesExternalMemberContributors } from './akritesExternalMemberContributors' - export const MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE = 'security-contacts-worker' export const MEMBER_CONTRIBUTORS_JOB_WORKFLOW = 'collectMemberContributors' +export const MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW = + 'collectMemberContributorsFromInferredRepo' +export const RESULT_EXPIRED_REASON = 'Job result expired, resubmit the job' export interface AkritesExternalMemberContributorsJobPending { jobId: string @@ -16,10 +21,18 @@ export interface AkritesExternalMemberContributorsJobFailed { reason: string } -export type AkritesExternalMemberContributorsJobDone = Omit< - AkritesExternalMemberContributors, - 'status' -> & { +export interface AkritesExternalMemberContributorsJobUnresolvable { + jobId: string + status: 'unresolvable' + reason: string + repoUrl: string | null +} + +export type MemberContributorsDoneBody = + | Omit + | InferredRepoMemberContributorsBody + +export type AkritesExternalMemberContributorsJobDone = MemberContributorsDoneBody & { jobId: string status: 'done' } @@ -27,6 +40,7 @@ export type AkritesExternalMemberContributorsJobDone = Omit< export type AkritesExternalMemberContributorsJob = | AkritesExternalMemberContributorsJobPending | AkritesExternalMemberContributorsJobFailed + | AkritesExternalMemberContributorsJobUnresolvable | AkritesExternalMemberContributorsJobDone export function memberContributorsJobWorkflowId(jobId: string): string { @@ -46,9 +60,22 @@ export function toFailedMemberContributorsJob( return { jobId, status: 'failed', reason: reason ?? 'Job failed' } } +export function toUnresolvableMemberContributorsJob( + jobId: string, + reason: string | null, + repoUrl: string | null, +): AkritesExternalMemberContributorsJobUnresolvable { + return { + jobId, + status: 'unresolvable', + reason: reason ?? 'Repository could not be resolved', + repoUrl, + } +} + export function toDoneMemberContributorsJob( jobId: string, - body: AkritesExternalMemberContributors, + body: MemberContributorsDoneBody, ): AkritesExternalMemberContributorsJobDone { return { ...body, jobId, status: 'done' } } diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index fa33422bec..b893143258 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -15,6 +15,8 @@ const { findMemberContributorsPackage, findActiveMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, + findLatestMemberContributorJobByPurl, + incrementMemberContributorJobSubmissions, createMemberContributorJob, failMemberContributorJob, } = vi.hoisted(() => ({ @@ -24,6 +26,8 @@ const { findMemberContributorsPackage: vi.fn(), findActiveMemberContributorJobByPurl: vi.fn(), findLatestMemberContributorJobStatusByPurl: vi.fn(), + findLatestMemberContributorJobByPurl: vi.fn(), + incrementMemberContributorJobSubmissions: vi.fn(), createMemberContributorJob: vi.fn(), failMemberContributorJob: vi.fn(), })) @@ -50,6 +54,8 @@ vi.mock('@crowd/data-access-layer', () => ({ findMemberContributorsPackage, findActiveMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, + findLatestMemberContributorJobByPurl, + incrementMemberContributorJobSubmissions, createMemberContributorJob, failMemberContributorJob, })) @@ -102,6 +108,8 @@ beforeEach(() => { findMemberContributorsPackage.mockResolvedValue(packageRow()) findActiveMemberContributorJobByPurl.mockResolvedValue(null) findLatestMemberContributorJobStatusByPurl.mockResolvedValue(null) + findLatestMemberContributorJobByPurl.mockResolvedValue(null) + incrementMemberContributorJobSubmissions.mockResolvedValue(undefined) createMemberContributorJob.mockImplementation(async (_qx, input) => input.id) failMemberContributorJob.mockResolvedValue(undefined) }) @@ -170,16 +178,6 @@ describe('submitMemberContributorsJob', () => { expect(status).toHaveBeenCalledWith(202) }) - it('rejects an ecosystem we do not collect with 400 before any database read', async () => { - const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) - const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) - expect(error).toBeInstanceOf(BadRequestError) - expect((error as BadRequestError).message).toBe('Unsupported ecosystem: hackage') - expect(findMemberContributorsPackage).not.toHaveBeenCalled() - expect(createMemberContributorJob).not.toHaveBeenCalled() - expect(start).not.toHaveBeenCalled() - }) - it('rejects an unknown purl with 404', async () => { findMemberContributorsPackage.mockResolvedValue(null) const { req, res } = mockReqRes({ purl: 'pkg:npm/nope' }) @@ -262,6 +260,7 @@ describe('submitMemberContributorsJob', () => { await submitMemberContributorsJob(req, res) expect(createMemberContributorJob).not.toHaveBeenCalled() + expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'existing-job') expectRedispatched('existing-job') expect(status).toHaveBeenCalledWith(202) expect(json).toHaveBeenCalledWith({ jobId: 'existing-job', status: 'pending' }) @@ -365,3 +364,138 @@ describe('submitMemberContributorsJob', () => { expect(status).not.toHaveBeenCalled() }) }) + +describe('submitMemberContributorsJob, unknown ecosystem', () => { + const INFERRED_WORKFLOW = 'collectMemberContributorsFromInferredRepo' + + function inferredStart(jobId: string) { + return [ + INFERRED_WORKFLOW, + { + taskQueue: 'security-contacts-worker', + workflowId: `member-contributors-job/${jobId}`, + workflowIdConflictPolicy: 'USE_EXISTING', + workflowIdReusePolicy: 'REJECT_DUPLICATE', + retry: { maximumAttempts: 1 }, + args: [{ jobId }], + }, + ] + } + + function latestJob(id: string, latestStatus: 'done' | 'failed' | 'unresolvable') { + findLatestMemberContributorJobByPurl.mockResolvedValue({ + id, + status: latestStatus, + collectionPath: 'llm_repo_resolution', + }) + } + + it.each(['pkg:hackage/aeson', 'pkg:Hackage/aeson'])( + 'creates an llm repo resolution job for %s without a package lookup and starts the inferred-repo workflow', + async (purl) => { + const { req, res, status, json } = mockReqRes({ purl }) + + await submitMemberContributorsJob(req, res) + + const [, input] = createMemberContributorJob.mock.calls[0] + expect(createMemberContributorJob.mock.calls).toEqual([ + [ + {}, + { + id: expect.stringMatching(UUID), + purl, + ecosystem: 'hackage', + collectionPath: 'llm_repo_resolution', + packageId: null, + repoId: null, + }, + ], + ]) + expect(start.mock.calls).toEqual([inferredStart(input.id)]) + expect(findMemberContributorsPackage).not.toHaveBeenCalled() + expect(status).toHaveBeenCalledWith(202) + expect(json).toHaveBeenCalledWith({ jobId: input.id, status: 'pending' }) + }, + ) + + it.each(['pkg:', 'pkg:hackage', 'pkg:/x', 'pkg:hackage/aeson%ZZ'])( + 'rejects the malformed purl %s with 400 before any database call', + async (purl) => { + const { req, res } = mockReqRes({ purl }) + const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) + expect(error).toBeInstanceOf(BadRequestError) + expect((error as BadRequestError).message).toBe('Malformed purl') + expect( + [ + findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, + createMemberContributorJob, + start, + ].map((fn) => fn.mock.calls.length), + ).toEqual([0, 0, 0, 0]) + }, + ) + + it('returns the active job, counts the submission and re-dispatches the inferred-repo workflow', async () => { + findActiveMemberContributorJobByPurl.mockResolvedValue({ id: 'active-1' }) + const { req, res, status, json } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'active-1']]) + expect(start.mock.calls).toEqual([inferredStart('active-1')]) + expect(createMemberContributorJob).not.toHaveBeenCalled() + expect(status).toHaveBeenCalledWith(202) + expect(json).toHaveBeenCalledWith({ jobId: 'active-1', status: 'pending' }) + }) + + it.each(['COMPLETED', 'RUNNING'])( + 'refuses with 409 carrying the previous done jobId while its workflow is %s', + async (workflowStatus) => { + latestJob('done-1', 'done') + describeWorkflow.mockResolvedValue({ status: { name: workflowStatus } }) + const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(ConflictError) + expect((error as ConflictError).context).toEqual({ jobId: 'done-1' }) + expect(getHandle.mock.calls).toEqual([['member-contributors-job/done-1']]) + expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'done-1']]) + expect(createMemberContributorJob).not.toHaveBeenCalled() + }, + ) + + it.each([ + { + latestStatus: 'done' as const, + workflowStatus: 'not found', + describedWorkflows: [['member-contributors-job/old-1']], + }, + { + latestStatus: 'done' as const, + workflowStatus: 'FAILED', + describedWorkflows: [['member-contributors-job/old-1']], + }, + { latestStatus: 'failed' as const, workflowStatus: 'not found', describedWorkflows: [] }, + { latestStatus: 'unresolvable' as const, workflowStatus: 'not found', describedWorkflows: [] }, + ])( + 'accepts a new job when the latest job is $latestStatus with its workflow $workflowStatus', + async ({ latestStatus, workflowStatus, describedWorkflows }) => { + latestJob('old-1', latestStatus) + if (workflowStatus !== 'not found') { + describeWorkflow.mockResolvedValue({ status: { name: workflowStatus } }) + } + const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect({ + describedWorkflows: getHandle.mock.calls, + created: createMemberContributorJob.mock.calls.length, + submissionsCounted: incrementMemberContributorJobSubmissions.mock.calls, + statuses: status.mock.calls, + }).toEqual({ describedWorkflows, created: 1, submissionsCounted: [], statuses: [[202]] }) + }, + ) +}) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts index 28120ed5e5..d937b630c5 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts @@ -10,8 +10,10 @@ import { createMemberContributorJob, failMemberContributorJob, findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, findMemberContributorsPackage, + incrementMemberContributorJobSubmissions, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { @@ -19,10 +21,17 @@ import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy, WorkflowIdReusePolicy, + WorkflowNotFoundError, } from '@crowd/temporal' -import { collectionPathForPurlType, purlType } from './memberContributorCollectionPaths' import { + LLM_REPO_RESOLUTION_PATH, + collectionPathForPurlType, + isWellFormedPurl, + purlType, +} from './memberContributorCollectionPaths' +import { + MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE, MEMBER_CONTRIBUTORS_JOB_WORKFLOW, memberContributorsJobWorkflowId, @@ -37,11 +46,7 @@ function resolveCollectionPath(purl: string): { collectionPath: MemberContributorCollectionPath } { const type = purlType(purl) - const collectionPath = collectionPathForPurlType(type) - if (collectionPath === null) { - throw new BadRequestError(`Unsupported ecosystem: ${type}`) - } - return { purlType: type, collectionPath } + return { purlType: type, collectionPath: collectionPathForPurlType(type) } } async function findLinkedPackage(qx: QueryExecutor, purl: string): Promise { @@ -88,13 +93,10 @@ function jobWorkflowStartOptions(jobId: string) { } } -async function ensureJobWorkflowStarted(jobId: string): Promise { +async function ensureJobWorkflowStarted(jobId: string, workflowType: string): Promise { const packagesTemporal = await getPackagesTemporalClient() try { - await packagesTemporal.workflow.start( - MEMBER_CONTRIBUTORS_JOB_WORKFLOW, - jobWorkflowStartOptions(jobId), - ) + await packagesTemporal.workflow.start(workflowType, jobWorkflowStartOptions(jobId)) } catch (err) { if (!(err instanceof WorkflowExecutionAlreadyStartedError)) { throw err @@ -111,14 +113,15 @@ async function workflowExists(packagesTemporal: Client, jobId: string): Promise< } } -async function startJobWorkflow(qx: QueryExecutor, jobId: string): Promise { +async function startJobWorkflow( + qx: QueryExecutor, + jobId: string, + workflowType: string, +): Promise { let packagesTemporal: Client | undefined try { packagesTemporal = await getPackagesTemporalClient() - await packagesTemporal.workflow.start( - MEMBER_CONTRIBUTORS_JOB_WORKFLOW, - jobWorkflowStartOptions(jobId), - ) + await packagesTemporal.workflow.start(workflowType, jobWorkflowStartOptions(jobId)) } catch (err) { if (err instanceof WorkflowExecutionAlreadyStartedError) { return @@ -132,17 +135,99 @@ async function startJobWorkflow(qx: QueryExecutor, jobId: string): Promise } } +const READABLE_DONE_JOB_WORKFLOW_STATUSES = new Set(['COMPLETED', 'RUNNING']) + +async function doneJobWorkflowResultReadable(jobId: string): Promise { + const packagesTemporal = await getPackagesTemporalClient() + try { + const description = await packagesTemporal.workflow + .getHandle(memberContributorsJobWorkflowId(jobId)) + .describe() + return READABLE_DONE_JOB_WORKFLOW_STATUSES.has(description.status.name) + } catch (err) { + if (err instanceof WorkflowNotFoundError) { + return false + } + throw err + } +} + +async function assertInferredRepoResultNotReadable(qx: QueryExecutor, purl: string): Promise { + const latest = await findLatestMemberContributorJobByPurl(qx, purl) + if (!latest || latest.status !== 'done' || latest.collectionPath !== LLM_REPO_RESOLUTION_PATH) { + return + } + if (!(await doneJobWorkflowResultReadable(latest.id))) { + return + } + await incrementMemberContributorJobSubmissions(qx, latest.id) + throw new ConflictError('Member contributors already collected for this purl', { + jobId: latest.id, + }) +} + +async function respondWithActiveJob( + qx: QueryExecutor, + jobId: string, + workflowType: string, + res: Response, +): Promise { + await incrementMemberContributorJobSubmissions(qx, jobId) + await ensureJobWorkflowStarted(jobId, workflowType) + res.status(202).json(toPendingMemberContributorsJob(jobId)) +} + +async function submitInferredRepoJob( + qx: QueryExecutor, + purl: string, + ecosystem: string, + res: Response, +): Promise { + if (!isWellFormedPurl(purl)) { + throw new BadRequestError('Malformed purl') + } + const active = await findActiveMemberContributorJobByPurl(qx, purl) + if (active) { + await respondWithActiveJob(qx, active.id, MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, res) + return + } + await assertInferredRepoResultNotReadable(qx, purl) + const created = await createMemberContributorJob(qx, { + id: generateUUIDv4(), + purl, + ecosystem, + collectionPath: LLM_REPO_RESOLUTION_PATH, + packageId: null, + repoId: null, + }) + if (created === null) { + await respondWithActiveJob( + qx, + await findActiveJobIdOrThrow(qx, purl), + MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, + res, + ) + return + } + await startJobWorkflow(qx, created, MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW) + res.status(202).json(toPendingMemberContributorsJob(created)) +} + export async function submitMemberContributorsJob(req: Request, res: Response): Promise { const { purl } = validateOrThrow(purlBodySchema, req.body) - const { collectionPath } = resolveCollectionPath(purl) + const { purlType: type, collectionPath } = resolveCollectionPath(purl) const qx = await getPackagesQx() + if (collectionPath === LLM_REPO_RESOLUTION_PATH) { + await submitInferredRepoJob(qx, purl, type, res) + return + } + const pkg = await findLinkedPackage(qx, purl) const active = await findActiveMemberContributorJobByPurl(qx, purl) if (active) { - await ensureJobWorkflowStarted(active.id) - res.status(202).json(toPendingMemberContributorsJob(active.id)) + await respondWithActiveJob(qx, active.id, MEMBER_CONTRIBUTORS_JOB_WORKFLOW, res) return } @@ -157,12 +242,15 @@ export async function submitMemberContributorsJob(req: Request, res: Response): repoId: pkg.repoId, }) if (created === null) { - const activeJobId = await findActiveJobIdOrThrow(qx, purl) - await ensureJobWorkflowStarted(activeJobId) - res.status(202).json(toPendingMemberContributorsJob(activeJobId)) + await respondWithActiveJob( + qx, + await findActiveJobIdOrThrow(qx, purl), + MEMBER_CONTRIBUTORS_JOB_WORKFLOW, + res, + ) return } - await startJobWorkflow(qx, created) + await startJobWorkflow(qx, created, MEMBER_CONTRIBUTORS_JOB_WORKFLOW) res.status(202).json(toPendingMemberContributorsJob(created)) } diff --git a/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql b/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql new file mode 100644 index 0000000000..01e620a170 --- /dev/null +++ b/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql @@ -0,0 +1,14 @@ +ALTER TABLE member_contributor_jobs + DROP CONSTRAINT member_contributor_jobs_collection_path_check, + ADD CONSTRAINT member_contributor_jobs_collection_path_check + CHECK (collection_path IN ('github', 'llm_repo_resolution')), + DROP CONSTRAINT member_contributor_jobs_status_check, + ADD CONSTRAINT member_contributor_jobs_status_check + CHECK (status IN ('pending', 'running', 'done', 'failed', 'unresolvable')), + ADD COLUMN repo_url TEXT, + ADD COLUMN repo_url_source TEXT CHECK (repo_url_source IN ('llm_inferred')), + ADD COLUMN llm_confidence NUMERIC(3, 2), + ADD COLUMN llm_input_tokens INTEGER, + ADD COLUMN llm_output_tokens INTEGER, + ADD COLUMN llm_cost_usd NUMERIC(10, 6), + ADD COLUMN submission_count INTEGER NOT NULL DEFAULT 1; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a21a17b1cc..3d9441fd53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1672,6 +1672,9 @@ importers: unzipper: specifier: ^0.12.3 version: 0.12.3 + zod: + specifier: 'catalog:' + version: 4.3.6 devDependencies: '@types/js-yaml': specifier: ^4.0.9 diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index 236bee949c..5aeb3f9ac9 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -107,7 +107,8 @@ "tar": "^7.5.20", "tsx": "catalog:", "undici": "^7.29.1", - "unzipper": "^0.12.3" + "unzipper": "^0.12.3", + "zod": "catalog:" }, "devDependencies": { "@types/js-yaml": "^4.0.9", diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts index 0a90ccf540..9e23e1cbe1 100644 --- a/services/apps/packages_worker/src/activities.ts +++ b/services/apps/packages_worker/src/activities.ts @@ -70,13 +70,18 @@ export { blastRadiusReport, } from './blast-radius/activities' export { + buildInferredRepoMemberContributors, collectGithubRepoContributors, completeMemberContributorsJob, failMemberContributorsJob, + fetchGithubRepoContributors, + markMemberContributorsJobUnresolvable, mergeIdentityAffiliations, prepareIdentityAffiliationsRun, resolveIdentityAffiliationsChunk, resolveNewIdentityAffiliations, + resolveRepoViaLlm, + startInferredRepoMemberContributorsJob, startMemberContributorsJob, syncGitActivityContributors, syncGithubContributorsBatch, diff --git a/services/apps/packages_worker/src/config.ts b/services/apps/packages_worker/src/config.ts index de5268b4e6..ece7e6d023 100644 --- a/services/apps/packages_worker/src/config.ts +++ b/services/apps/packages_worker/src/config.ts @@ -139,3 +139,17 @@ export function getReportingProtocolConfig() { llmSecretAccessKey: process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY, } } + +export interface BedrockCredentials { + accessKeyId: string + secretAccessKey: string +} + +export function getBedrockCredentials(): BedrockCredentials | null { + const accessKeyId = process.env.CROWD_AWS_BEDROCK_ACCESS_KEY_ID + const secretAccessKey = process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY + if (!accessKeyId || !secretAccessKey) { + return null + } + return { accessKeyId, secretAccessKey } +} diff --git a/services/apps/packages_worker/src/llm/bedrockClient.ts b/services/apps/packages_worker/src/llm/bedrockClient.ts new file mode 100644 index 0000000000..5dcfecdea9 --- /dev/null +++ b/services/apps/packages_worker/src/llm/bedrockClient.ts @@ -0,0 +1,45 @@ +import { BedrockRuntimeClient, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime' + +import type { BedrockCredentials } from '../config' + +export type BedrockInvoke = (modelId: string, body: string) => Promise + +const clients = new Map() + +export function bedrockClientFor( + region: string, + credentials: BedrockCredentials, +): BedrockRuntimeClient { + const key = `${region}:${credentials.accessKeyId}` + let client = clients.get(key) + if (!client) { + client = new BedrockRuntimeClient({ region, credentials }) + clients.set(key, client) + } + return client +} + +export function invokeBedrockModel( + region: string, + credentials: BedrockCredentials, + timeoutMs: number, +): BedrockInvoke { + return async (modelId, body) => { + const controller = new AbortController() + const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs) + try { + const response = await bedrockClientFor(region, credentials).send( + new InvokeModelCommand({ + modelId, + accept: 'application/json', + contentType: 'application/json', + body, + }), + { abortSignal: controller.signal }, + ) + return response.body.transformToString() + } finally { + clearTimeout(timeoutHandle) + } + } +} diff --git a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts index c234641413..77a64d5c9a 100644 --- a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts +++ b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts @@ -6,6 +6,7 @@ import type { } from '../git-activity/syncGitActivityContributors' import { collectMemberContributors, + collectMemberContributorsFromInferredRepo, resolveIdentityAffiliations, syncRepoContributorsFromGitActivity, } from '../workflows' @@ -36,6 +37,11 @@ const h = vi.hoisted(() => { resolveNewIdentityAffiliations: vi.fn(), completeMemberContributorsJob: vi.fn(), failMemberContributorsJob: vi.fn(), + startInferredRepoMemberContributorsJob: vi.fn(), + resolveRepoViaLlm: vi.fn(), + fetchGithubRepoContributors: vi.fn(), + buildInferredRepoMemberContributors: vi.fn(), + markMemberContributorsJobUnresolvable: vi.fn(), } const proxyCalls: Array<{ options: Record; activities: string[] }> = [] return { @@ -275,4 +281,113 @@ describe('activity proxy options', () => { retry: { initialInterval: '30 seconds', backoffCoefficient: 2, maximumAttempts: 3 }, }) }) + + it('bounds the llm resolution to two attempts and retries the inferred repo finisher forever', () => { + const retry = { initialInterval: '30 seconds', backoffCoefficient: 2, maximumAttempts: 3 } + expect({ + start: optionsFor('startInferredRepoMemberContributorsJob'), + resolve: optionsFor('resolveRepoViaLlm'), + fetch: optionsFor('fetchGithubRepoContributors'), + build: optionsFor('buildInferredRepoMemberContributors'), + unresolvable: optionsFor('markMemberContributorsJobUnresolvable'), + }).toEqual({ + start: { startToCloseTimeout: '5 minutes', retry }, + resolve: { startToCloseTimeout: '3 minutes', retry: { ...retry, maximumAttempts: 2 } }, + fetch: { startToCloseTimeout: '1 hour', heartbeatTimeout: '5 minutes', retry }, + build: { startToCloseTimeout: '5 minutes', retry }, + unresolvable: { + startToCloseTimeout: '1 minute', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumInterval: '5 minutes', + }, + }, + }) + }) +}) + +describe('collectMemberContributorsFromInferredRepo', () => { + const job = { jobId: 'job-1' } + const repoUrl = 'https://github.com/haskell/aeson' + const body = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl, + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + + beforeEach(() => { + h.acts.startInferredRepoMemberContributorsJob.mockResolvedValue({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + h.acts.resolveRepoViaLlm.mockResolvedValue({ outcome: 'resolved', repoUrl, confidence: 0.95 }) + h.acts.fetchGithubRepoContributors.mockResolvedValue({ status: 'fetched', logins: ['octo'] }) + h.acts.resolveNewIdentityAffiliations.mockResolvedValue({ + identities: 1, + inserted: 1, + counts: {}, + }) + h.acts.buildInferredRepoMemberContributors.mockResolvedValue(body) + }) + + it('resolves the repo, collects and affiliates its contributors, then completes with the body', async () => { + await expect(collectMemberContributorsFromInferredRepo(job)).resolves.toEqual(body) + + const sequence = [ + h.acts.startInferredRepoMemberContributorsJob, + h.acts.resolveRepoViaLlm, + h.acts.fetchGithubRepoContributors, + h.acts.resolveNewIdentityAffiliations, + h.acts.buildInferredRepoMemberContributors, + h.acts.completeMemberContributorsJob, + ] + expect(sequence.map((fn) => fn.mock.calls)).toEqual([ + [[job]], + [[{ jobId: 'job-1', purl: 'pkg:hackage/aeson' }]], + [[{ repoUrl }]], + [[{ logins: ['octo'] }]], + [[{ purl: 'pkg:hackage/aeson', ecosystem: 'hackage', repoUrl, logins: ['octo'] }]], + [[job]], + ]) + const order = sequence.map((fn) => fn.mock.invocationCallOrder[0]) + expect(order).toEqual([...order].sort((a, b) => a - b)) + expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() + expect(h.acts.markMemberContributorsJobUnresolvable).not.toHaveBeenCalled() + }) + + it('marks the job unresolvable with the reason and attempted url, returning nothing', async () => { + h.acts.resolveRepoViaLlm.mockResolvedValue({ + outcome: 'unresolvable', + reason: 'LLM returned no repository', + repoUrl: null, + }) + + await expect(collectMemberContributorsFromInferredRepo(job)).resolves.toBeUndefined() + + expect(h.acts.markMemberContributorsJobUnresolvable.mock.calls).toEqual([ + [{ jobId: 'job-1', reason: 'LLM returned no repository', repoUrl: null }], + ]) + expect(h.acts.fetchGithubRepoContributors).not.toHaveBeenCalled() + expect(h.acts.completeMemberContributorsJob).not.toHaveBeenCalled() + expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() + }) + + it('fails the job non-cancellably with the activity cause and rethrows', async () => { + const cause = new Error('Bedrock credentials are not configured') + h.acts.resolveRepoViaLlm.mockRejectedValue(new h.ActivityFailure('Activity failed', cause)) + + await expect(collectMemberContributorsFromInferredRepo(job)).rejects.toThrow('Activity failed') + + expect(h.acts.failMemberContributorsJob.mock.calls).toEqual([ + [{ jobId: 'job-1', reason: 'Bedrock credentials are not configured' }], + ]) + expect(h.nonCancellable).toHaveBeenCalledTimes(1) + expect(h.acts.completeMemberContributorsJob).not.toHaveBeenCalled() + }) }) diff --git a/services/apps/packages_worker/src/member-contributors/activities.ts b/services/apps/packages_worker/src/member-contributors/activities.ts index 80dfb86b9d..fa0dd8a6ce 100644 --- a/services/apps/packages_worker/src/member-contributors/activities.ts +++ b/services/apps/packages_worker/src/member-contributors/activities.ts @@ -6,4 +6,12 @@ export { syncGovernanceContributors } from './governance/syncGovernanceContribut export { startMemberContributorsJob } from './job/start' export { collectGithubRepoContributors } from './job/collectGithubRepoContributors' export { resolveNewIdentityAffiliations } from './job/resolveNewIdentityAffiliations' -export { completeMemberContributorsJob, failMemberContributorsJob } from './job/finish' +export { + completeMemberContributorsJob, + failMemberContributorsJob, + markMemberContributorsJobUnresolvable, +} from './job/finish' +export { startInferredRepoMemberContributorsJob } from './job/startInferredRepo' +export { resolveRepoViaLlm } from './job/resolveRepoViaLlm' +export { fetchGithubRepoContributors } from './job/fetchGithubRepoContributors' +export { buildInferredRepoMemberContributors } from './job/buildInferredRepoBody' diff --git a/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts b/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts index 7e142dec88..43123e4622 100644 --- a/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts +++ b/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts @@ -19,17 +19,21 @@ function contributorLogin(contributor: GithubContributor): string | null { return login.toLowerCase() } -export function toGithubContributorRows( - contributors: GithubContributor[], - repoId: string, - observedAt: Date, -): RepoContributorRow[] { +export function toGithubContributorLogins(contributors: GithubContributor[]): string[] { const logins = new Set() for (const contributor of contributors) { const login = contributorLogin(contributor) if (login !== null) logins.add(login) } - return [...logins].map((login) => ({ + return [...logins] +} + +export function toGithubContributorRows( + contributors: GithubContributor[], + repoId: string, + observedAt: Date, +): RepoContributorRow[] { + return toGithubContributorLogins(contributors).map((login) => ({ repoId, source: GITHUB_SOURCE, role: GIT_ACTIVITY_ROLE, diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts new file mode 100644 index 0000000000..a8dc3fe616 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' +import * as affiliationsDal from '@crowd/data-access-layer/src/osspckgs/identityAffiliations' + +import { buildInferredRepoMemberContributors, purlPackageName } from '../buildInferredRepoBody' + +const qx = vi.hoisted(() => ({})) +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('@crowd/data-access-layer/src/osspckgs/identityAffiliations', async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@crowd/data-access-layer/src/osspckgs/identityAffiliations') + >()), + findAffiliationsForIdentityKeys: vi.fn(), +})) + +const find = vi.mocked(affiliationsDal.findAffiliationsForIdentityKeys) + +beforeEach(() => vi.clearAllMocks()) + +describe('purlPackageName', () => { + it.each([ + ['pkg:hackage/aeson', 'aeson'], + ['pkg:conan/boost', 'boost'], + ['pkg:hackage/aeson@2.2.3.0?arch=x#sub', 'aeson'], + ['pkg:npm/%40babel/core', '@babel/core'], + ['pkg:huggingface/distilbert-base-uncased', 'distilbert-base-uncased'], + ])('%s → %s', (purl, name) => { + expect(purlPackageName(purl)).toBe(name) + }) +}) + +describe('buildInferredRepoMemberContributors', () => { + const input = { + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + logins: ['octo', 'nobody'], + } + const emptyBody = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + + it('builds the body from affiliated logins with llm provenance', async () => { + find.mockResolvedValue([ + { + person: 'cdp-1', + source: 'github', + role: null, + roleKind: 'contributor', + identityType: 'github-login', + identityValue: 'octo', + endedAt: null, + lastSeenAt: null, + registryUsername: null, + registryDisplayName: null, + akritesMember: 'Google', + cdpOrganizationId: 'org-google', + confidence: 'cdp', + confidenceScore: '1.0', + }, + ]) + + const body = await buildInferredRepoMemberContributors(input) + + expect(find.mock.calls).toEqual([ + [ + qx, + [ + { identityType: 'github-login', identityValue: 'octo' }, + { identityType: 'github-login', identityValue: 'nobody' }, + ], + ], + ]) + expect(body).toEqual({ + ...emptyBody, + contributors: [ + { + githubLogin: 'octo', + displayName: null, + emails: [], + registryUsernames: [], + sources: ['github'], + organization: { + id: canonicalCdpOrganizationIdByMemberName().get('Google'), + name: 'Google', + confidence: 'cdp', + confidenceScore: 1, + }, + }, + ], + }) + }) + + it('returns an empty body without querying affiliations when there are no logins', async () => { + await expect(buildInferredRepoMemberContributors({ ...input, logins: [] })).resolves.toEqual( + emptyBody, + ) + expect(find).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts new file mode 100644 index 0000000000..9e3d65e000 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' + +import { canonicalGithubUrl } from '../canonicalGithubUrl' + +const aeson = { + kind: 'github', + url: 'https://github.com/haskell/aeson', + owner: 'haskell', + name: 'aeson', +} + +describe('canonicalGithubUrl', () => { + it.each([ + ['https://github.com/haskell/aeson', aeson], + ['https://github.com/Haskell/Aeson.git', aeson], + ['http://www.github.com/haskell/aeson/', aeson], + ['https://github.com/haskell/aeson/tree/main/src', aeson], + ['github.com/haskell/aeson', aeson], + ['https://gitlab.com/a/b', { kind: 'other', host: 'gitlab.com' }], + ['https://github.com/haskell', { kind: 'invalid' }], + ['not a url', { kind: 'invalid' }], + ['https://github.com/', { kind: 'invalid' }], + ])('classifies %s', (raw, expected) => { + expect(canonicalGithubUrl(raw)).toEqual(expected) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts new file mode 100644 index 0000000000..a98ab72fb4 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts @@ -0,0 +1,95 @@ +import { heartbeat } from '@temporalio/activity' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { githubApiGet } from '../../../security-contacts/githubToken' +import { fetchGithubRepoContributors } from '../fetchGithubRepoContributors' + +vi.mock('@temporalio/activity', async (importOriginal) => ({ + ...(await importOriginal()), + heartbeat: vi.fn(), +})) +vi.mock('../../../config', () => ({ getGithubAppConfig: vi.fn(() => ({ appId: '1' })) })) +vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) + +const repo = { repoUrl: 'https://github.com/a/b' } + +function contributorsJson(entries: Array<{ login: string; type?: string }>): string { + return JSON.stringify(entries.map((e) => ({ type: 'User', ...e }))) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('fetchGithubRepoContributors', () => { + it('returns the lowercased human logins of the first contributors page', async () => { + vi.mocked(githubApiGet).mockResolvedValue({ + status: 200, + text: contributorsJson([ + { login: 'Octo' }, + { login: 'dependabot[bot]', type: 'Bot' }, + { login: 'ada' }, + ]), + }) + await expect( + fetchGithubRepoContributors({ repoUrl: 'https://github.com/haskell/aeson' }), + ).resolves.toEqual({ status: 'fetched', logins: ['octo', 'ada'] }) + expect(vi.mocked(githubApiGet).mock.calls).toEqual([ + ['/repos/haskell/aeson/contributors?per_page=100&page=1', 15000, { extraOkStatuses: [204] }], + ]) + }) + + it.each([ + ['an empty repository', { status: 204, text: null }], + [ + 'only bot contributors', + { status: 200, text: contributorsJson([{ login: 'x[bot]', type: 'Bot' }]) }, + ], + ])('returns empty for %s', async (_label, response) => { + vi.mocked(githubApiGet).mockResolvedValue(response) + await expect(fetchGithubRepoContributors(repo)).resolves.toEqual({ + status: 'empty', + logins: [], + }) + }) + + it.each([ + [ + 'without retry when the repo is missing', + () => vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }), + { message: 'Repository not found on GitHub', nonRetryable: true }, + ], + [ + 'retryably when the fetch errors', + () => vi.mocked(githubApiGet).mockRejectedValue(new Error('socket hang up')), + { + message: 'GitHub contributors fetch failed for https://github.com/a/b', + nonRetryable: false, + }, + ], + ])('fails %s', async (_label, arrange, expected) => { + arrange() + const error = (await fetchGithubRepoContributors(repo).catch((e: unknown) => e)) as Error & { + nonRetryable?: boolean + } + expect({ message: error.message, nonRetryable: error.nonRetryable === true }).toEqual(expected) + }) + + it('heartbeats while the fetch is in flight and stops once it settles', async () => { + vi.useFakeTimers() + vi.mocked(githubApiGet).mockImplementation( + () => + new Promise((resolve) => setTimeout(() => resolve({ status: 204, text: null }), 45_000)), + ) + const pending = fetchGithubRepoContributors(repo) + await vi.advanceTimersByTimeAsync(30_000) + expect(heartbeat).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(15_000) + await expect(pending).resolves.toEqual({ status: 'empty', logins: [] }) + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts new file mode 100644 index 0000000000..39fac38405 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts @@ -0,0 +1,154 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { inferRepositoryCandidates } from '../llmRepoInference' + +const { log } = vi.hoisted(() => ({ log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })) + +vi.mock('@crowd/logging', () => ({ getServiceChildLogger: () => log })) + +type ResponseBlock = { type: 'thinking'; thinking: string } | { type: 'text'; text: string } + +const usage = { inputTokens: 300, outputTokens: 80, costUsd: 300 * 0.000002 + 80 * 0.00001 } +const twoCandidates = [ + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, +] +const fourCandidates = [1, 2, 3, 4].map((n) => ({ + repositoryUrl: `https://github.com/o/r${n}`, + confidence: 0.9, +})) + +function bedrockResponse(text: string | null, stopReason = 'end_turn'): string { + const content: ResponseBlock[] = [{ type: 'thinking', thinking: '' }] + if (text !== null) content.push({ type: 'text', text }) + return JSON.stringify({ + content, + usage: { input_tokens: 300, output_tokens: 80 }, + stop_reason: stopReason, + }) +} + +function answer(candidates: unknown[], rationale = ''): string { + return JSON.stringify({ candidates, rationale }) +} + +function invokeReturning(response: string) { + return vi.fn().mockResolvedValue(response) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('inferRepositoryCandidates', () => { + it('sends the purl to sonnet 5.5, parses the first text block and logs the call', async () => { + const invoke = invokeReturning( + bedrockResponse(answer(twoCandidates, 'aeson moved to the haskell org')), + ) + + const result = await inferRepositoryCandidates('pkg:hackage/aeson', invoke) + + expect(invoke).toHaveBeenCalledTimes(1) + const [modelId, body] = invoke.mock.calls[0] + expect(modelId).toBe('us.anthropic.claude-sonnet-5-5') + expect(JSON.parse(body)).toEqual({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 2048, + output_config: { effort: 'low' }, + system: expect.stringContaining('canonical source repository'), + messages: [{ role: 'user', content: [{ type: 'text', text: 'pkg:hackage/aeson' }] }], + }) + expect(result).toEqual({ status: 'answered', candidates: twoCandidates, usage }) + expect(log.info.mock.calls).toEqual([ + [ + { + modelId: 'us.anthropic.claude-sonnet-5-5', + purl: 'pkg:hackage/aeson', + stopReason: 'end_turn', + inputTokens: 300, + outputTokens: 80, + costUsd: usage.costUsd, + candidates: 2, + durationMs: expect.any(Number), + }, + 'LLM repo inference call', + ], + ]) + }) + + it.each([ + [ + 'a fenced answer, capped at three candidates', + bedrockResponse('```json\n' + answer(fourCandidates) + '\n```'), + { status: 'answered', candidates: fourCandidates.slice(0, 3), usage }, + ], + [ + 'an empty candidate list', + bedrockResponse(answer([], 'unknown')), + { status: 'answered', candidates: [], usage }, + ], + [ + 'a response without a text block', + bedrockResponse(null), + { status: 'unparseable', reason: 'LLM answer had no text content', usage }, + ], + [ + 'an answer that is not JSON', + bedrockResponse('I think it is github.com/haskell/aeson'), + { status: 'unparseable', reason: 'LLM answer was not valid JSON', usage }, + ], + [ + 'an answer outside the schema', + bedrockResponse(answer([{ repositoryUrl: 'https://github.com/a/b', confidence: 1.7 }])), + { status: 'unparseable', reason: 'LLM answer did not match the expected schema', usage }, + ], + [ + 'a refusal', + bedrockResponse(null, 'refusal'), + { status: 'unparseable', reason: 'LLM refused to answer', usage }, + ], + [ + 'an answer truncated at max_tokens', + bedrockResponse('{"candidates": [', 'max_tokens'), + { status: 'truncated', usage }, + ], + ])('returns and logs the result with usage for %s', async (_label, response, expected) => { + await expect( + inferRepositoryCandidates('pkg:conan/x', invokeReturning(response)), + ).resolves.toEqual(expected) + expect(log.info.mock.calls).toEqual([ + [ + expect.objectContaining({ + stopReason: JSON.parse(response).stop_reason, + inputTokens: 300, + outputTokens: 80, + }), + 'LLM repo inference call', + ], + ]) + }) + + it.each([ + ['usage is missing', undefined], + ['usage is non-numeric', { input_tokens: '300', output_tokens: 80 }], + ['usage is partial', { input_tokens: 300 }], + ])('rejects a response whose %s', async (_label, responseUsage) => { + const invoke = invokeReturning( + JSON.stringify({ + content: [{ type: 'text', text: '{}' }], + usage: responseUsage, + stop_reason: 'end_turn', + }), + ) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( + 'Bedrock response had no token usage', + ) + }) + + it('propagates transport errors', async () => { + const invoke = vi.fn().mockRejectedValue(new Error('ThrottlingException')) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( + 'ThrottlingException', + ) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts new file mode 100644 index 0000000000..1d54133889 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts @@ -0,0 +1,272 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import * as jobsDal from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { getBedrockCredentials, getGithubAppConfig } from '../../../config' +import { githubApiGet } from '../../../security-contacts/githubToken' +import { inferRepositoryCandidates } from '../llmRepoInference' +import { resolveRepoViaLlm } from '../resolveRepoViaLlm' + +const qx = vi.hoisted(() => ({})) + +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('../../../config', () => ({ + getBedrockCredentials: vi.fn(), + getGithubAppConfig: vi.fn(), +})) +vi.mock('../../../llm/bedrockClient', () => ({ invokeBedrockModel: vi.fn(() => vi.fn()) })) +vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) +vi.mock('../llmRepoInference', async (importOriginal) => ({ + ...(await importOriginal()), + inferRepositoryCandidates: vi.fn(), +})) +vi.mock('@crowd/data-access-layer/src/osspckgs/memberContributorJobs', () => ({ + recordMemberContributorJobInferredRepo: vi.fn(), + recordMemberContributorJobLlmUsage: vi.fn(), +})) + +const usage = { inputTokens: 300, outputTokens: 80, costUsd: 0.0014 } +const input = { jobId: 'job-1', purl: 'pkg:hackage/aeson' } +const recordRepo = vi.mocked(jobsDal.recordMemberContributorJobInferredRepo) +const recordUsage = vi.mocked(jobsDal.recordMemberContributorJobLlmUsage) +const found = { status: 200, text: '{}' } +const absent = { status: 404, text: null } + +type Inference = Awaited> +type GithubResult = Awaited> + +function answered(candidates: Array<{ repositoryUrl: string; confidence: number }>): Inference { + return { status: 'answered', candidates, usage } +} + +function githubCheck(path: string) { + return [path, 15000] +} + +function expectUsageRecordedOnceBeforeGithub() { + expect(recordUsage.mock.calls).toEqual([[qx, 'job-1', usage]]) + const githubCalls = vi.mocked(githubApiGet).mock.invocationCallOrder + if (githubCalls.length > 0) { + expect(recordUsage.mock.invocationCallOrder[0]).toBeLessThan(githubCalls[0]) + } +} + +async function resolveWith(inference: Inference, githubResults: GithubResult[]) { + vi.mocked(inferRepositoryCandidates).mockResolvedValue(inference) + for (const githubResult of githubResults) { + vi.mocked(githubApiGet).mockResolvedValueOnce(githubResult) + } + const result = await resolveRepoViaLlm(input) + expectUsageRecordedOnceBeforeGithub() + return { + result, + githubCalls: vi.mocked(githubApiGet).mock.calls, + recordedRepos: recordRepo.mock.calls, + } +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(getBedrockCredentials).mockReturnValue({ accessKeyId: 'k', secretAccessKey: 's' }) + vi.mocked(getGithubAppConfig).mockReturnValue( + {} as unknown as ReturnType, + ) + vi.mocked(githubApiGet).mockResolvedValue(found) +}) + +describe('resolveRepoViaLlm', () => { + it.each([ + { + name: 'canonicalises the first existing candidate', + inference: answered([ + { repositoryUrl: 'https://github.com/Haskell/aeson.git', confidence: 0.95 }, + ]), + github: [found], + checked: [githubCheck('/repos/haskell/aeson')], + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.95, + }, + { + name: 'skips an absent candidate for the next existing one', + inference: answered([ + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.8 }, + ]), + github: [absent, found], + checked: [githubCheck('/repos/bos/aeson'), githubCheck('/repos/haskell/aeson')], + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.8, + }, + ])( + '$name and records it on the job', + async ({ inference, github, checked, repoUrl, confidence }) => { + await expect(resolveWith(inference, github)).resolves.toEqual({ + result: { outcome: 'resolved', repoUrl, confidence }, + githubCalls: checked, + recordedRepos: [[qx, 'job-1', { repoUrl, confidence }]], + }) + }, + ) + + it.each([ + { + name: 'every candidate is absent, keeping the last attempted url', + inference: answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.8 }, + ]), + github: [absent, absent], + checked: [githubCheck('/repos/a/one'), githubCheck('/repos/a/two')], + reason: 'Repository not found on GitHub: https://github.com/a/two', + repoUrl: 'https://github.com/a/two', + }, + { + name: 'the only candidate is below the confidence threshold', + inference: answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }]), + github: [], + checked: [], + reason: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }, + { + name: 'all candidates are below the threshold, quoting the highest confidence', + inference: answered([ + { repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }, + { repositoryUrl: 'https://github.com/a/c', confidence: 0.6 }, + ]), + github: [], + checked: [], + reason: 'LLM confidence 0.6 below threshold 0.7', + repoUrl: null, + }, + { + name: 'the confident candidate is absent and lower-ranked ones are below the threshold', + inference: answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.2 }, + ]), + github: [absent], + checked: [githubCheck('/repos/a/one')], + reason: 'Repository not found on GitHub: https://github.com/a/one', + repoUrl: 'https://github.com/a/one', + }, + { + name: 'the candidate host is not GitHub', + inference: answered([{ repositoryUrl: 'https://gitlab.com/a/b', confidence: 0.9 }]), + github: [], + checked: [], + reason: 'Repository host not supported: gitlab.com', + repoUrl: 'https://gitlab.com/a/b', + }, + { + name: 'the candidate url is not recognised', + inference: answered([{ repositoryUrl: 'not a url', confidence: 0.9 }]), + github: [], + checked: [], + reason: 'Repository URL not recognised: not a url', + repoUrl: 'not a url', + }, + { + name: 'the LLM returns no candidate', + inference: answered([]), + github: [], + checked: [], + reason: 'LLM returned no repository', + repoUrl: null, + }, + { + name: 'the LLM answer is unparseable', + inference: { status: 'unparseable', reason: 'LLM answer was not valid JSON', usage }, + github: [], + checked: [], + reason: 'LLM answer was not valid JSON', + repoUrl: null, + }, + ] satisfies Array<{ + name: string + inference: Inference + github: GithubResult[] + checked: Array> + reason: string + repoUrl: string | null + }>)('is unresolvable when $name', async ({ inference, github, checked, reason, repoUrl }) => { + await expect(resolveWith(inference, github)).resolves.toEqual({ + result: { outcome: 'unresolvable', reason, repoUrl }, + githubCalls: checked, + recordedRepos: [], + }) + }) + + it.each([ + { + name: 'retryably when the GitHub existence check fails, after recording usage', + arrange: () => { + vi.mocked(inferRepositoryCandidates).mockResolvedValue( + answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.9 }]), + ) + vi.mocked(githubApiGet).mockRejectedValue(new Error('HTTP 502')) + }, + expected: { + message: 'HTTP 502', + nonRetryable: false, + llmCalls: 1, + usageCalls: [[qx, 'job-1', usage]], + githubCalls: 1, + }, + }, + { + name: 'retryably when the LLM answer is truncated, after recording usage', + arrange: () => { + vi.mocked(inferRepositoryCandidates).mockResolvedValue({ status: 'truncated', usage }) + }, + expected: { + message: 'LLM answer truncated at max_tokens', + nonRetryable: false, + llmCalls: 1, + usageCalls: [[qx, 'job-1', usage]], + githubCalls: 0, + }, + }, + { + name: 'before calling the LLM when the GitHub App config is missing', + arrange: () => { + vi.mocked(getGithubAppConfig).mockImplementation(() => { + throw new Error('Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY') + }) + }, + expected: { + message: 'Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY', + nonRetryable: false, + llmCalls: 0, + usageCalls: [], + githubCalls: 0, + }, + }, + { + name: 'without retry and before calling the LLM when Bedrock credentials are missing', + arrange: () => { + vi.mocked(getBedrockCredentials).mockReturnValue(null) + }, + expected: { + message: 'Bedrock credentials are not configured', + nonRetryable: true, + llmCalls: 0, + usageCalls: [], + githubCalls: 0, + }, + }, + ])('fails $name', async ({ arrange, expected }) => { + arrange() + const error = (await resolveRepoViaLlm(input).catch((e: unknown) => e)) as Error & { + nonRetryable?: boolean + } + expect({ + message: error.message, + nonRetryable: error.nonRetryable === true, + llmCalls: vi.mocked(inferRepositoryCandidates).mock.calls.length, + usageCalls: recordUsage.mock.calls, + githubCalls: vi.mocked(githubApiGet).mock.calls.length, + }).toEqual(expected) + expect(recordRepo).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts new file mode 100644 index 0000000000..eddcedb5e1 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts @@ -0,0 +1,41 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import * as jobsDal from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { startInferredRepoMemberContributorsJob } from '../startInferredRepo' + +const qx = vi.hoisted(() => ({})) +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('@crowd/data-access-layer/src/osspckgs/memberContributorJobs', () => ({ + markMemberContributorJobRunning: vi.fn(), + findMemberContributorJobPurl: vi.fn(), +})) + +const markRunning = vi.mocked(jobsDal.markMemberContributorJobRunning) +const findPurl = vi.mocked(jobsDal.findMemberContributorJobPurl) + +beforeEach(() => vi.clearAllMocks()) + +describe('startInferredRepoMemberContributorsJob', () => { + it('marks the job running before reading its purl and ecosystem', async () => { + findPurl.mockResolvedValue({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage' }) + + await expect(startInferredRepoMemberContributorsJob({ jobId: 'job-1' })).resolves.toEqual({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + expect(markRunning.mock.calls).toEqual([[qx, 'job-1']]) + expect(findPurl.mock.calls).toEqual([[qx, 'job-1']]) + expect(markRunning.mock.invocationCallOrder[0]).toBeLessThan( + findPurl.mock.invocationCallOrder[0], + ) + }) + + it('fails without retry when the job row is missing', async () => { + findPurl.mockResolvedValue(null) + const error = await startInferredRepoMemberContributorsJob({ jobId: 'job-1' }).catch( + (e: unknown) => e, + ) + expect(error).toMatchObject({ message: 'Job not found', nonRetryable: true }) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts b/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts new file mode 100644 index 0000000000..e76c0f1cb9 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts @@ -0,0 +1,48 @@ +import { + IdentityKey, + findAffiliationsForIdentityKeys, +} from '@crowd/data-access-layer/src/osspckgs/identityAffiliations' +import { + InferredRepoMemberContributorsBody, + groupAffiliatedMemberIdentities, + toInferredRepoMemberContributorsBody, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorsBody' + +import { getPackagesDb } from '../../db' + +export interface BuildInferredRepoBodyInput { + purl: string + ecosystem: string + repoUrl: string + logins: string[] +} + +export function purlPackageName(purl: string): string { + const afterScheme = purl.slice('pkg:'.length) + const afterType = afterScheme.slice(afterScheme.indexOf('/') + 1) + const withoutQualifiers = afterType.split(/[?#]/)[0] + const withoutVersion = withoutQualifiers.replace(/@[^/@]+$/, '') + return decodeURIComponent(withoutVersion) +} + +function toLoginKeys(logins: string[]): IdentityKey[] { + return logins.map((identityValue) => ({ identityType: 'github-login', identityValue })) +} + +export async function buildInferredRepoMemberContributors( + input: BuildInferredRepoBodyInput, +): Promise { + const rows = + input.logins.length === 0 + ? [] + : await findAffiliationsForIdentityKeys(await getPackagesDb(), toLoginKeys(input.logins)) + return toInferredRepoMemberContributorsBody( + { + purl: input.purl, + name: purlPackageName(input.purl), + ecosystem: input.ecosystem, + repoUrl: input.repoUrl, + }, + groupAffiliatedMemberIdentities(rows), + ) +} diff --git a/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts b/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts new file mode 100644 index 0000000000..920800e15c --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts @@ -0,0 +1,35 @@ +const GITHUB_HOSTS = new Set(['github.com', 'www.github.com']) +const SEGMENT = /^[a-z0-9._-]+$/i + +export type GithubUrlParse = + | { kind: 'github'; url: string; owner: string; name: string } + | { kind: 'other'; host: string } + | { kind: 'invalid' } + +function parseUrl(raw: string): URL | null { + const trimmed = raw.trim() + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}` + try { + return new URL(withScheme) + } catch { + return null + } +} + +export function canonicalGithubUrl(raw: string): GithubUrlParse { + const url = parseUrl(raw) + if (!url) return { kind: 'invalid' } + const host = url.hostname.toLowerCase() + if (!GITHUB_HOSTS.has(host)) return { kind: 'other', host } + const [owner, repo] = url.pathname.split('/').filter(Boolean) + const name = repo?.replace(/\.git$/i, '') + if (!owner || !name || !SEGMENT.test(owner) || !SEGMENT.test(name)) return { kind: 'invalid' } + const lowerOwner = owner.toLowerCase() + const lowerName = name.toLowerCase() + return { + kind: 'github', + url: `https://github.com/${lowerOwner}/${lowerName}`, + owner: lowerOwner, + name: lowerName, + } +} diff --git a/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts b/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts new file mode 100644 index 0000000000..4ffbc91e6b --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts @@ -0,0 +1,44 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { getGithubAppConfig } from '../../config' +import { githubApiGet } from '../../security-contacts/githubToken' +import { RepoContributorsFetch, fetchRepoContributors } from '../github/fetchRepoContributors' +import { toGithubContributorLogins } from '../github/mapContributors' +import { HEARTBEAT_INTERVAL_MS, heartbeatSafely } from '../github/syncGithubContributorsBatch' + +const FETCH_TIMEOUT_MS = 15_000 + +export interface FetchGithubRepoContributorsInput { + repoUrl: string +} + +export interface GithubRepoContributorsFetch { + status: 'fetched' | 'empty' + logins: string[] +} + +async function fetchWithHeartbeat(repoUrl: string): Promise { + const heartbeatTimer = setInterval(() => heartbeatSafely(), HEARTBEAT_INTERVAL_MS) + try { + return await fetchRepoContributors(repoUrl, (path) => + githubApiGet(path, FETCH_TIMEOUT_MS, { extraOkStatuses: [204] }), + ) + } finally { + clearInterval(heartbeatTimer) + } +} + +export async function fetchGithubRepoContributors( + input: FetchGithubRepoContributorsInput, +): Promise { + getGithubAppConfig() + const fetched = await fetchWithHeartbeat(input.repoUrl) + if (fetched.status === 'missing') { + throw ApplicationFailure.nonRetryable('Repository not found on GitHub') + } + if (fetched.status === 'error') { + throw new Error(`GitHub contributors fetch failed for ${input.repoUrl}`) + } + const logins = toGithubContributorLogins(fetched.contributors) + return { status: logins.length === 0 ? 'empty' : 'fetched', logins } +} diff --git a/services/apps/packages_worker/src/member-contributors/job/finish.ts b/services/apps/packages_worker/src/member-contributors/job/finish.ts index 1563832313..a7c62d27bd 100644 --- a/services/apps/packages_worker/src/member-contributors/job/finish.ts +++ b/services/apps/packages_worker/src/member-contributors/job/finish.ts @@ -1,6 +1,7 @@ import { completeMemberContributorJob, failMemberContributorJob, + markMemberContributorJobUnresolvable, } from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' import { getPackagesDb } from '../../db' @@ -21,3 +22,19 @@ export async function failMemberContributorsJob( ): Promise { await failMemberContributorJob(await getPackagesDb(), input.jobId, input.reason) } + +export interface MarkUnresolvableInput extends MemberContributorsJobRef { + reason: string + repoUrl: string | null +} + +export async function markMemberContributorsJobUnresolvable( + input: MarkUnresolvableInput, +): Promise { + await markMemberContributorJobUnresolvable( + await getPackagesDb(), + input.jobId, + input.reason, + input.repoUrl, + ) +} diff --git a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts new file mode 100644 index 0000000000..ebe4ba635c --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts @@ -0,0 +1,159 @@ +import { z } from 'zod' + +import { parseLlmJson } from '@crowd/common' +import type { LlmUsage } from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' +import { getServiceChildLogger } from '@crowd/logging' +import { LlmModelType, estimateLlmCostUsd } from '@crowd/types' + +import type { BedrockInvoke } from '../../llm/bedrockClient' + +const log = getServiceChildLogger('member-contributors:llm-repo-inference') + +export const REPO_INFERENCE_MODEL_ID = LlmModelType.CLAUDE_SONNET_5_5 +export const MAX_REPO_CANDIDATES = 3 +const MAX_TOKENS = 2048 +export const TRUNCATED_ANSWER = 'LLM answer truncated at max_tokens' + +const candidateSchema = z.object({ + repositoryUrl: z.string().min(1), + confidence: z.number().min(0).max(1), +}) +const answerSchema = z.object({ + candidates: z.array(candidateSchema), + rationale: z.string().optional(), +}) + +const ANSWER_JSON_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['candidates', 'rationale'], + properties: { + candidates: { + type: 'array', + maxItems: MAX_REPO_CANDIDATES, + items: { + type: 'object', + additionalProperties: false, + required: ['repositoryUrl', 'confidence'], + properties: { + repositoryUrl: { type: 'string' }, + confidence: { type: 'number', minimum: 0, maximum: 1 }, + }, + }, + }, + rationale: { type: 'string' }, + }, +} + +const SYSTEM_PROMPT = `You resolve a software package, given as a package URL (purl, e.g. pkg:hackage/aeson or pkg:conan/boost), to its canonical source repository. +Rules: +- Answer only from what you know. Never invent a URL you have not seen. +- Return up to ${MAX_REPO_CANDIDATES} candidate GitHub repository URLs ranked by likelihood, each with a confidence between 0 and 1 that it is the package's canonical upstream source repository. +- Prefer the upstream project over forks, mirrors, packaging repositories and distribution recipes. +- If you do not know the repository, return an empty candidates list. +- rationale: one short sentence. +Respond ONLY with a single JSON object matching this JSON schema, no prose: +${JSON.stringify(ANSWER_JSON_SCHEMA)}` + +export interface RepoCandidate { + repositoryUrl: string + confidence: number +} + +export type RepoInference = + | { status: 'answered'; candidates: RepoCandidate[]; usage: LlmUsage } + | { status: 'unparseable'; reason: string; usage: LlmUsage } + | { status: 'truncated'; usage: LlmUsage } + +interface BedrockMessageResponse { + content?: Array<{ type?: string; text?: string }> + usage?: { input_tokens?: unknown; output_tokens?: unknown } + stop_reason?: string +} + +function requestBody(purl: string): string { + return JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: MAX_TOKENS, + output_config: { effort: 'low' }, + system: SYSTEM_PROMPT, + messages: [{ role: 'user', content: [{ type: 'text', text: purl }] }], + }) +} + +function usageOf(response: BedrockMessageResponse): LlmUsage { + const inputTokens = response.usage?.input_tokens + const outputTokens = response.usage?.output_tokens + if (typeof inputTokens !== 'number' || typeof outputTokens !== 'number') { + throw new Error('Bedrock response had no token usage') + } + return { + inputTokens, + outputTokens, + costUsd: estimateLlmCostUsd(REPO_INFERENCE_MODEL_ID, inputTokens, outputTokens), + } +} + +function firstText(response: BedrockMessageResponse): string | null { + const block = response.content?.find( + (entry) => entry.type === 'text' && typeof entry.text === 'string', + ) + return block?.text ?? null +} + +function candidatesOf(response: BedrockMessageResponse): RepoCandidate[] | string { + if (response.stop_reason === 'max_tokens') { + return TRUNCATED_ANSWER + } + if (response.stop_reason === 'refusal') { + return 'LLM refused to answer' + } + const text = firstText(response) + return text === null ? 'LLM answer had no text content' : parseAnswer(text) +} + +function parseAnswer(text: string): RepoCandidate[] | string { + let raw: unknown + try { + raw = parseLlmJson(text) + } catch { + return 'LLM answer was not valid JSON' + } + const parsed = answerSchema.safeParse(raw) + if (!parsed.success) { + return 'LLM answer did not match the expected schema' + } + return parsed.data.candidates.slice(0, MAX_REPO_CANDIDATES) +} + +export async function inferRepositoryCandidates( + purl: string, + invoke: BedrockInvoke, +): Promise { + const startedAt = Date.now() + const response = JSON.parse( + await invoke(REPO_INFERENCE_MODEL_ID, requestBody(purl)), + ) as BedrockMessageResponse + const usage = usageOf(response) + const candidates = candidatesOf(response) + log.info( + { + modelId: REPO_INFERENCE_MODEL_ID, + purl, + stopReason: response.stop_reason, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + costUsd: usage.costUsd, + candidates: typeof candidates === 'string' ? 0 : candidates.length, + durationMs: Date.now() - startedAt, + }, + 'LLM repo inference call', + ) + if (candidates === TRUNCATED_ANSWER) { + return { status: 'truncated', usage } + } + if (typeof candidates === 'string') { + return { status: 'unparseable', reason: candidates, usage } + } + return { status: 'answered', candidates, usage } +} diff --git a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts new file mode 100644 index 0000000000..e3405e58b2 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts @@ -0,0 +1,109 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { + recordMemberContributorJobInferredRepo, + recordMemberContributorJobLlmUsage, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' +import { LLM_MODEL_REGION_MAP } from '@crowd/types' + +import { getBedrockCredentials, getGithubAppConfig } from '../../config' +import { getPackagesDb } from '../../db' +import { invokeBedrockModel } from '../../llm/bedrockClient' +import { githubApiGet } from '../../security-contacts/githubToken' +import { canonicalGithubUrl } from './canonicalGithubUrl' +import { + REPO_INFERENCE_MODEL_ID, + RepoCandidate, + TRUNCATED_ANSWER, + inferRepositoryCandidates, +} from './llmRepoInference' + +export const MIN_REPO_CONFIDENCE = 0.7 +const LLM_TIMEOUT_MS = 60_000 +const GITHUB_TIMEOUT_MS = 15_000 + +export interface ResolveRepoViaLlmInput { + jobId: string + purl: string +} + +export type ResolveRepoViaLlmResult = + | { outcome: 'resolved'; repoUrl: string; confidence: number } + | { outcome: 'unresolvable'; reason: string; repoUrl: string | null } + +type Unresolvable = Extract + +function unresolvable(reason: string, repoUrl: string | null = null): Unresolvable { + return { outcome: 'unresolvable', reason, repoUrl } +} + +async function repoExistsOnGithub(owner: string, name: string): Promise { + const result = await githubApiGet(`/repos/${owner}/${name}`, GITHUB_TIMEOUT_MS) + return result.status === 200 +} + +async function pickExistingCandidate( + candidates: RepoCandidate[], +): Promise { + const confident = candidates.filter((candidate) => candidate.confidence >= MIN_REPO_CONFIDENCE) + if (confident.length === 0) { + const highest = Math.max(...candidates.map((candidate) => candidate.confidence)) + return unresolvable(`LLM confidence ${highest} below threshold ${MIN_REPO_CONFIDENCE}`) + } + let last: Unresolvable = unresolvable('LLM returned no repository') + for (const candidate of confident) { + const parsed = canonicalGithubUrl(candidate.repositoryUrl) + if (parsed.kind === 'other') { + last = unresolvable(`Repository host not supported: ${parsed.host}`, candidate.repositoryUrl) + continue + } + if (parsed.kind === 'invalid') { + last = unresolvable( + `Repository URL not recognised: ${candidate.repositoryUrl}`, + candidate.repositoryUrl, + ) + continue + } + if (await repoExistsOnGithub(parsed.owner, parsed.name)) { + return { outcome: 'resolved', repoUrl: parsed.url, confidence: candidate.confidence } + } + last = unresolvable(`Repository not found on GitHub: ${parsed.url}`, parsed.url) + } + return last +} + +export async function resolveRepoViaLlm( + input: ResolveRepoViaLlmInput, +): Promise { + const credentials = getBedrockCredentials() + if (!credentials) { + throw ApplicationFailure.nonRetryable('Bedrock credentials are not configured') + } + getGithubAppConfig() + const invoke = invokeBedrockModel( + LLM_MODEL_REGION_MAP[REPO_INFERENCE_MODEL_ID], + credentials, + LLM_TIMEOUT_MS, + ) + const inference = await inferRepositoryCandidates(input.purl, invoke) + const pkgsQx = await getPackagesDb() + await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) + + if (inference.status === 'truncated') { + throw new Error(TRUNCATED_ANSWER) + } + if (inference.status === 'unparseable') { + return unresolvable(inference.reason) + } + if (inference.candidates.length === 0) { + return unresolvable('LLM returned no repository') + } + const result = await pickExistingCandidate(inference.candidates) + if (result.outcome === 'resolved') { + await recordMemberContributorJobInferredRepo(pkgsQx, input.jobId, { + repoUrl: result.repoUrl, + confidence: result.confidence, + }) + } + return result +} diff --git a/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts b/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts new file mode 100644 index 0000000000..95e9e67083 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts @@ -0,0 +1,22 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { + MemberContributorJobPurl, + findMemberContributorJobPurl, + markMemberContributorJobRunning, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { getPackagesDb } from '../../db' +import { MemberContributorsJobRef } from './start' + +export async function startInferredRepoMemberContributorsJob( + input: MemberContributorsJobRef, +): Promise { + const pkgsQx = await getPackagesDb() + await markMemberContributorJobRunning(pkgsQx, input.jobId) + const job = await findMemberContributorJobPurl(pkgsQx, input.jobId) + if (!job) { + throw ApplicationFailure.nonRetryable('Job not found') + } + return job +} diff --git a/services/apps/packages_worker/src/member-contributors/workflows.ts b/services/apps/packages_worker/src/member-contributors/workflows.ts index c3ed23bbc2..1d0c77ecb2 100644 --- a/services/apps/packages_worker/src/member-contributors/workflows.ts +++ b/services/apps/packages_worker/src/member-contributors/workflows.ts @@ -9,6 +9,8 @@ import { workflowInfo, } from '@temporalio/workflow' +import type { InferredRepoMemberContributorsBody } from '@crowd/data-access-layer/src/osspckgs/memberContributorsBody' + import type * as memberContributorActivities from './activities' import { addChunkCounts, emptyChunkCounts } from './affiliations/counts' import type { IdentityAffiliationsRunCounts, IdentityAffiliationsState } from './affiliations/state' @@ -91,6 +93,38 @@ const { completeMemberContributorsJob, failMemberContributorsJob } = proxyActivi retry: UNBOUNDED_RETRY, }) +const { startInferredRepoMemberContributorsJob } = proxyActivities< + typeof memberContributorActivities +>({ + startToCloseTimeout: '5 minutes', + retry: RETRY, +}) + +const { resolveRepoViaLlm } = proxyActivities({ + startToCloseTimeout: '3 minutes', + retry: { ...RETRY, maximumAttempts: 2 }, +}) + +const { fetchGithubRepoContributors } = proxyActivities({ + startToCloseTimeout: '1 hour', + heartbeatTimeout: '5 minutes', + retry: RETRY, +}) + +const { buildInferredRepoMemberContributors } = proxyActivities( + { + startToCloseTimeout: '5 minutes', + retry: RETRY, + }, +) + +const { markMemberContributorsJobUnresolvable } = proxyActivities< + typeof memberContributorActivities +>({ + startToCloseTimeout: '1 minute', + retry: UNBOUNDED_RETRY, +}) + const GITHUB_BATCH_SIZE = 500 const GITHUB_BATCHES_PER_RUN = 20 @@ -202,3 +236,35 @@ export async function collectMemberContributors(input: MemberContributorsJobRef) throw err } } + +export async function collectMemberContributorsFromInferredRepo( + input: MemberContributorsJobRef, +): Promise { + try { + const { purl, ecosystem } = await startInferredRepoMemberContributorsJob(input) + const resolved = await resolveRepoViaLlm({ jobId: input.jobId, purl }) + if (resolved.outcome === 'unresolvable') { + await markMemberContributorsJobUnresolvable({ + jobId: input.jobId, + reason: resolved.reason, + repoUrl: resolved.repoUrl, + }) + return undefined + } + const collected = await fetchGithubRepoContributors({ repoUrl: resolved.repoUrl }) + await resolveNewIdentityAffiliations({ logins: collected.logins }) + const body = await buildInferredRepoMemberContributors({ + purl, + ecosystem, + repoUrl: resolved.repoUrl, + logins: collected.logins, + }) + await completeMemberContributorsJob(input) + return body + } catch (err) { + await CancellationScope.nonCancellable(() => + failMemberContributorsJob({ jobId: input.jobId, reason: failureReason(err) }), + ) + throw err + } +} diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts index 231641da55..38ff6d16e4 100644 --- a/services/apps/packages_worker/src/workflows/index.ts +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -41,6 +41,7 @@ export { } from '../security-contacts/workflows' export { collectMemberContributors, + collectMemberContributorsFromInferredRepo, resolveIdentityAffiliations, syncGovernanceFileContributors, syncRepoContributorsFromGitActivity, diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 61a3ce19a8..191766abdf 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -44,4 +44,5 @@ export * from './osspckgs/api' export * from './osspckgs/akritesMembers' export * from './osspckgs/identityAffiliations' export * from './osspckgs/memberContributors' +export * from './osspckgs/memberContributorsBody' export * from './osspckgs/memberContributorJobs' diff --git a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts index f470be357b..6cbb32d4e3 100644 --- a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts @@ -5,6 +5,7 @@ import { IdentityKey, StagingAffiliationRow, fillIdentityAffiliationsStaging, + findAffiliationsForIdentityKeys, findIdentityKeysWithoutAffiliation, insertIdentityAffiliations, mergeIdentityAffiliationsStaging, @@ -314,3 +315,47 @@ describe('mergeIdentityAffiliationsStaging delete guard', () => { expect(del?.params).toEqual({ runStartedAt: RUN_STARTED_AT }) }) }) + +describe('findAffiliationsForIdentityKeys', () => { + it('returns no rows without querying for no keys', async () => { + const { qx, calls } = fakeQx() + await expect(findAffiliationsForIdentityKeys(qx, [])).resolves.toEqual([]) + expect(calls).toHaveLength(0) + }) + + it('reads only member-affiliated rows for the keys in one query, keyed by person', async () => { + const row = { + person: 'cdp-member-1', + source: 'github', + role: null, + roleKind: 'contributor', + identityType: 'github-login', + identityValue: 'octo', + endedAt: null, + lastSeenAt: null, + registryUsername: null, + registryDisplayName: null, + akritesMember: 'Google', + cdpOrganizationId: 'org-google', + confidence: 'cdp', + confidenceScore: '1.0', + } + const { qx, calls } = fakeQx({ selectRows: [row] }) + const rows = await findAffiliationsForIdentityKeys(qx, [ + { identityType: 'github-login', identityValue: 'octo' }, + { identityType: 'github-login', identityValue: 'ada' }, + ]) + expect(rows).toEqual([row]) + expect(calls).toEqual([ + { + method: 'select', + query: expect.stringContaining('WHERE a.akrites_member IS NOT NULL'), + params: { types: ['github-login', 'github-login'], values: ['octo', 'ada'] }, + inTx: false, + }, + ]) + expect(calls[0].query).toContain( + "COALESCE(a.cdp_member_id::text, a.identity_type || ':' || a.identity_value) AS person", + ) + }) +}) diff --git a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts index e592506973..0a198c96f4 100644 --- a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts +++ b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts @@ -1,4 +1,5 @@ import { QueryExecutor, formatQuery } from '../queryExecutor' +import type { AffiliatedMemberIdentityRow } from './memberContributors' export type IdentityType = 'github-login' | 'email' export type AffiliationConfidence = 'cdp' | 'email_domain' | 'github_company' @@ -160,6 +161,35 @@ export async function findIdentityKeysWithoutAffiliation( ) } +export async function findAffiliationsForIdentityKeys( + qx: QueryExecutor, + keys: IdentityKey[], +): Promise { + if (keys.length === 0) { + return [] + } + return qx.select( + ` + SELECT COALESCE(a.cdp_member_id::text, a.identity_type || ':' || a.identity_value) AS person, + 'github'::text AS source, NULL::text AS role, 'contributor'::text AS "roleKind", + a.identity_type AS "identityType", a.identity_value AS "identityValue", + NULL::timestamptz AS "endedAt", NULL::timestamptz AS "lastSeenAt", + NULL::text AS "registryUsername", NULL::text AS "registryDisplayName", + a.akrites_member AS "akritesMember", a.cdp_organization_id::text AS "cdpOrganizationId", + a.confidence, a.confidence_score AS "confidenceScore" + FROM unnest($(types)::text[], $(values)::text[]) AS k(identity_type, identity_value) + JOIN identity_affiliations a + ON a.identity_type = k.identity_type AND a.identity_value = k.identity_value + WHERE a.akrites_member IS NOT NULL + ORDER BY person, a.identity_type, a.identity_value + `, + { + types: keys.map((key) => key.identityType), + values: keys.map((key) => key.identityValue), + }, + ) +} + export async function insertIdentityAffiliations( qx: QueryExecutor, rows: StagingAffiliationRow[], diff --git a/services/libs/data-access-layer/src/osspckgs/index.ts b/services/libs/data-access-layer/src/osspckgs/index.ts index 163917b2bd..f08da9191d 100644 --- a/services/libs/data-access-layer/src/osspckgs/index.ts +++ b/services/libs/data-access-layer/src/osspckgs/index.ts @@ -8,4 +8,5 @@ export * from './stewardships' export * from './api' export * from './identityAffiliations' export * from './memberContributors' +export * from './memberContributorsBody' export * from './memberContributorJobs' diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts index d383b0080b..3b4fe152c6 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts @@ -6,10 +6,16 @@ import { createMemberContributorJob, failMemberContributorJob, findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, findMemberContributorJob, + findMemberContributorJobPurl, findMemberContributorJobRepo, + incrementMemberContributorJobSubmissions, markMemberContributorJobRunning, + markMemberContributorJobUnresolvable, + recordMemberContributorJobInferredRepo, + recordMemberContributorJobLlmUsage, } from './memberContributorJobs' interface RecordedCall { @@ -39,6 +45,10 @@ function fakeQx(selectOneOrNoneRow: object | null = null) { return { qx: executor as unknown as QueryExecutor, calls } } +function singleLine(query: string): string { + return query.replace(/\s+/g, ' ') +} + const input = { id: '11111111-1111-4111-8111-111111111111', purl: 'pkg:npm/request-oauth', @@ -146,3 +156,117 @@ describe('status transitions', () => { expect(calls[0].params).toEqual({ id: input.id, error: 'boom' }) }) }) + +describe('inferred repo job reads', () => { + it('reads the purl and ecosystem of a job by id', async () => { + const { qx, calls } = fakeQx({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage' }) + await expect(findMemberContributorJobPurl(qx, 'job-1')).resolves.toEqual({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: 'SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)', + params: { id: 'job-1' }, + }, + ]) + }) + + it('reads the newest job of a purl with its status and collection path', async () => { + const latest = { id: 'job-1', status: 'done', collectionPath: 'llm_repo_resolution' } + const { qx, calls } = fakeQx(latest) + await expect(findLatestMemberContributorJobByPurl(qx, 'pkg:hackage/aeson')).resolves.toEqual( + latest, + ) + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: expect.stringContaining('WHERE purl = $(purl)'), + params: { purl: 'pkg:hackage/aeson' }, + }, + ]) + expect(singleLine(calls[0].query)).toContain('ORDER BY created_at DESC, id DESC LIMIT 1') + }) + + it('selects the collection path and inferred repo columns of a job', async () => { + const { qx, calls } = fakeQx(null) + await findMemberContributorJob(qx, 'job-1') + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: expect.stringContaining('repo_url AS "repoUrl", repo_url_source AS "repoUrlSource"'), + params: { id: 'job-1' }, + }, + ]) + expect(calls[0].query).toContain('collection_path AS "collectionPath"') + }) +}) + +describe('inferred repo job writes', () => { + it('records the inferred repo on a running job without touching llm usage', async () => { + const { qx, calls } = fakeQx() + await recordMemberContributorJobInferredRepo(qx, 'job-1', { + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.92, + }) + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status = 'running'"), + params: { id: 'job-1', repoUrl: 'https://github.com/haskell/aeson', confidence: 0.92 }, + }, + ]) + expect(calls[0].query).toContain( + "SET repo_url = $(repoUrl), repo_url_source = 'llm_inferred', llm_confidence = $(confidence),", + ) + expect(calls[0].query).not.toMatch(/llm_(input_tokens|output_tokens|cost_usd)/) + }) + + it('accumulates llm usage on a running job, keeping the cost when it is unknown', async () => { + const { qx, calls } = fakeQx() + await recordMemberContributorJobLlmUsage(qx, 'job-1', { + inputTokens: 300, + outputTokens: 20, + costUsd: null, + }) + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status = 'running'"), + params: { id: 'job-1', inputTokens: 300, outputTokens: 20, costUsd: null }, + }, + ]) + expect(singleLine(calls[0].query)).toContain( + 'SET llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens), llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens), llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END,', + ) + expect(calls[0].query).not.toContain('repo_url') + }) + + it('marks a job unresolvable unless it is done, keeping the known repo url', async () => { + const { qx, calls } = fakeQx() + await markMemberContributorJobUnresolvable(qx, 'job-1', 'LLM returned no repository', null) + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status <> 'done'"), + params: { id: 'job-1', reason: 'LLM returned no repository', repoUrl: null }, + }, + ]) + expect(calls[0].query).toContain( + "SET status = 'unresolvable', error = $(reason), repo_url = COALESCE($(repoUrl), repo_url),", + ) + }) + + it('increments the submission count of a job', async () => { + const { qx, calls } = fakeQx() + await incrementMemberContributorJobSubmissions(qx, 'job-1') + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining('SET submission_count = submission_count + 1'), + params: { id: 'job-1' }, + }, + ]) + }) +}) diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts index 5786aa8275..bb914bfce9 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts @@ -1,24 +1,51 @@ import { QueryExecutor } from '../queryExecutor' -export type MemberContributorCollectionPath = 'github' -export type MemberContributorJobStatus = 'pending' | 'running' | 'done' | 'failed' +export type MemberContributorCollectionPath = 'github' | 'llm_repo_resolution' +export type MemberContributorJobStatus = 'pending' | 'running' | 'done' | 'failed' | 'unresolvable' +export type MemberContributorRepoUrlSource = 'llm_inferred' export interface MemberContributorJobInput { id: string purl: string ecosystem: string collectionPath: MemberContributorCollectionPath - packageId: string - repoId: string + packageId: string | null + repoId: string | null } export interface MemberContributorJobRow { id: string purl: string + ecosystem: string + collectionPath: MemberContributorCollectionPath status: MemberContributorJobStatus error: string | null packageId: string | null repoId: string | null + repoUrl: string | null + repoUrlSource: MemberContributorRepoUrlSource | null +} + +export interface MemberContributorJobPurl { + purl: string + ecosystem: string +} + +export interface LatestMemberContributorJob { + id: string + status: MemberContributorJobStatus + collectionPath: MemberContributorCollectionPath +} + +export interface LlmUsage { + inputTokens: number + outputTokens: number + costUsd: number | null +} + +export interface InferredRepoRecord { + repoUrl: string + confidence: number } export interface MemberContributorJobRepo { @@ -82,8 +109,10 @@ export async function findMemberContributorJob( ): Promise { return qx.selectOneOrNone( ` - SELECT id, purl, status, error, - package_id::text AS "packageId", repo_id::text AS "repoId" + SELECT id, purl, ecosystem, status, error, + collection_path AS "collectionPath", + package_id::text AS "packageId", repo_id::text AS "repoId", + repo_url AS "repoUrl", repo_url_source AS "repoUrlSource" FROM member_contributor_jobs WHERE id = $(id) `, @@ -145,3 +174,92 @@ export async function failMemberContributorJob( { id, error }, ) } + +export async function findMemberContributorJobPurl( + qx: QueryExecutor, + id: string, +): Promise { + return qx.selectOneOrNone( + `SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)`, + { + id, + }, + ) +} + +export async function findLatestMemberContributorJobByPurl( + qx: QueryExecutor, + purl: string, +): Promise { + return qx.selectOneOrNone( + ` + SELECT id, status, collection_path AS "collectionPath" + FROM member_contributor_jobs + WHERE purl = $(purl) + ORDER BY created_at DESC, id DESC + LIMIT 1 + `, + { purl }, + ) +} + +export async function recordMemberContributorJobInferredRepo( + qx: QueryExecutor, + id: string, + record: InferredRepoRecord, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET repo_url = $(repoUrl), repo_url_source = 'llm_inferred', llm_confidence = $(confidence), + updated_at = NOW() + WHERE id = $(id) AND status = 'running' + `, + { id, ...record }, + ) +} + +export async function recordMemberContributorJobLlmUsage( + qx: QueryExecutor, + id: string, + usage: LlmUsage, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens), + llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens), + llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END, + updated_at = NOW() + WHERE id = $(id) AND status = 'running' + `, + { id, ...usage }, + ) +} + +export async function markMemberContributorJobUnresolvable( + qx: QueryExecutor, + id: string, + reason: string, + repoUrl: string | null, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET status = 'unresolvable', error = $(reason), repo_url = COALESCE($(repoUrl), repo_url), + completed_at = NOW(), updated_at = NOW() + WHERE id = $(id) AND status <> 'done' + `, + { id, reason, repoUrl }, + ) +} + +export async function incrementMemberContributorJobSubmissions( + qx: QueryExecutor, + id: string, +): Promise { + await qx.result( + `UPDATE member_contributor_jobs SET submission_count = submission_count + 1, updated_at = NOW() WHERE id = $(id)`, + { id }, + ) +} diff --git a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts similarity index 91% rename from backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts rename to services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts index 39fa712553..ca58f6b18c 100644 --- a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts @@ -3,12 +3,13 @@ import { describe, expect, it } from 'vitest' import type { AffiliatedMemberIdentityRow, MemberContributorsPackageRow, -} from '@crowd/data-access-layer' - + MemberIdentitySource, +} from './memberContributors' import { groupAffiliatedMemberIdentities, toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' + toInferredRepoMemberContributorsBody, +} from './memberContributorsBody' const GOOGLE = { akritesMember: 'Google', @@ -431,3 +432,46 @@ describe('toAkritesExternalMemberContributors', () => { expect(result.maintainers.registry).toHaveLength(1) }) }) + +describe('toInferredRepoMemberContributorsBody', () => { + it('builds the sync body with llm provenance, null mapping confidence and no maintainers', () => { + const entry = { + githubLogin: 'octo', + displayName: null, + emails: [], + registryUsernames: [], + sources: ['github'] as MemberIdentitySource[], + organization: { + id: 'org-google', + name: 'Google', + confidence: 'cdp' as const, + confidenceScore: 1, + }, + } + const body = toInferredRepoMemberContributorsBody( + { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + }, + { + contributors: [entry], + registry: [ + { ...entry, roles: [{ role: 'maintainer', roleKind: 'maintainer', current: true }] }, + ], + governanceFiles: [], + }, + ) + expect(body).toEqual({ + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [entry], + maintainers: { registry: [], governanceFiles: [] }, + }) + }) +}) diff --git a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts similarity index 89% rename from backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts rename to services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts index c722a692e7..60656a0544 100644 --- a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts @@ -1,12 +1,14 @@ +import { canonicalCdpOrganizationIdByMemberName } from './akritesMembers' +import type { AffiliationConfidence } from './identityAffiliations' import type { AffiliatedMemberIdentityRow, - AffiliationConfidence, MemberContributorsPackageRow, MemberIdentitySource, -} from '@crowd/data-access-layer' -import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' +} from './memberContributors' -import { toNullableNumber } from './mappers' +function toNullableNumber(value: number | string | null): number | null { + return value != null ? Number(value) : null +} const CANONICAL_ORGANIZATION_IDS = canonicalCdpOrganizationIdByMemberName() @@ -276,3 +278,34 @@ export function toAkritesExternalMemberContributors( }, } } + +export interface InferredRepoMemberContributorsBody extends Omit< + AkritesExternalMemberContributors, + 'status' | 'repoMappingConfidence' +> { + repoMappingConfidence: null + repoUrlSource: 'llm_inferred' +} + +export interface InferredRepoPackage { + purl: string + name: string + ecosystem: string + repoUrl: string +} + +export function toInferredRepoMemberContributorsBody( + pkg: InferredRepoPackage, + sections: MemberContributorsSections, +): InferredRepoMemberContributorsBody { + return { + purl: pkg.purl, + name: pkg.name, + ecosystem: pkg.ecosystem, + repoUrl: pkg.repoUrl, + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: sections.contributors, + maintainers: { registry: [], governanceFiles: [] }, + } +}