From 5a8ce2167878475c2eafef95b2a9f0fa4d885c0a Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:28:08 +0000 Subject: [PATCH 01/23] feat: extend member contributor jobs for llm repo resolution Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../packages/getMemberContributorsJob.test.ts | 4 + ...r_contributor_jobs_llm_repo_resolution.sql | 14 ++ .../osspckgs/memberContributorJobs.test.ts | 108 +++++++++++++++ .../src/osspckgs/memberContributorJobs.ts | 126 +++++++++++++++++- 4 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index 456154046e..372a19668e 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -31,10 +31,14 @@ function jobRow(overrides: Partial = {}): MemberContrib return { id: JOB_ID, purl: 'pkg:npm/request-oauth', + ecosystem: 'npm', + collectionPath: 'github', status: 'pending', error: null, packageId: '2008420', repoId: '590455', + repoUrl: null, + repoUrlSource: null, ...overrides, } } diff --git a/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql b/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql new file mode 100644 index 0000000000..01e620a170 --- /dev/null +++ b/backend/src/osspckgs/migrations/V1792150054__member_contributor_jobs_llm_repo_resolution.sql @@ -0,0 +1,14 @@ +ALTER TABLE member_contributor_jobs + DROP CONSTRAINT member_contributor_jobs_collection_path_check, + ADD CONSTRAINT member_contributor_jobs_collection_path_check + CHECK (collection_path IN ('github', 'llm_repo_resolution')), + DROP CONSTRAINT member_contributor_jobs_status_check, + ADD CONSTRAINT member_contributor_jobs_status_check + CHECK (status IN ('pending', 'running', 'done', 'failed', 'unresolvable')), + ADD COLUMN repo_url TEXT, + ADD COLUMN repo_url_source TEXT CHECK (repo_url_source IN ('llm_inferred')), + ADD COLUMN llm_confidence NUMERIC(3, 2), + ADD COLUMN llm_input_tokens INTEGER, + ADD COLUMN llm_output_tokens INTEGER, + ADD COLUMN llm_cost_usd NUMERIC(10, 6), + ADD COLUMN submission_count INTEGER NOT NULL DEFAULT 1; diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts index d383b0080b..6162771e31 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts @@ -6,10 +6,16 @@ import { createMemberContributorJob, failMemberContributorJob, findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, findMemberContributorJob, + findMemberContributorJobPurl, findMemberContributorJobRepo, + incrementMemberContributorJobSubmissions, markMemberContributorJobRunning, + markMemberContributorJobUnresolvable, + recordMemberContributorJobInferredRepo, + recordMemberContributorJobLlmUsage, } from './memberContributorJobs' interface RecordedCall { @@ -146,3 +152,105 @@ describe('status transitions', () => { expect(calls[0].params).toEqual({ id: input.id, error: 'boom' }) }) }) + +describe('findMemberContributorJobPurl', () => { + it('reads purl and ecosystem by id', async () => { + const { qx, calls } = fakeQx({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage' }) + await expect(findMemberContributorJobPurl(qx, 'job-1')).resolves.toEqual({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + expect(calls[0].query).toContain('SELECT purl, ecosystem') + expect(calls[0].params).toEqual({ id: 'job-1' }) + }) +}) + +describe('findLatestMemberContributorJobByPurl', () => { + it('returns id, status and collection path of the newest job', async () => { + const { qx, calls } = fakeQx({ id: 'job-1', status: 'done', collectionPath: 'llm_repo_resolution' }) + await expect(findLatestMemberContributorJobByPurl(qx, 'pkg:hackage/aeson')).resolves.toEqual({ + id: 'job-1', + status: 'done', + collectionPath: 'llm_repo_resolution', + }) + expect(calls[0].query).toContain('ORDER BY created_at DESC, id DESC') + expect(calls[0].query).toContain('LIMIT 1') + }) +}) + +describe('recordMemberContributorJobInferredRepo', () => { + it('writes repo url, source, confidence and usage on a running job', async () => { + const { qx, calls } = fakeQx() + await recordMemberContributorJobInferredRepo(qx, 'job-1', { + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.92, + inputTokens: 300, + outputTokens: 80, + costUsd: 0.0014, + }) + expect(calls[0].method).toBe('result') + expect(calls[0].query).toContain("repo_url_source = 'llm_inferred'") + expect(calls[0].query).toContain("WHERE id = $(id) AND status = 'running'") + expect(calls[0].params).toEqual({ + id: 'job-1', + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.92, + inputTokens: 300, + outputTokens: 80, + costUsd: 0.0014, + }) + }) +}) + +describe('recordMemberContributorJobLlmUsage', () => { + it('writes only the usage columns on a running job', async () => { + const { qx, calls } = fakeQx() + await recordMemberContributorJobLlmUsage(qx, 'job-1', { + inputTokens: 300, + outputTokens: 20, + costUsd: null, + }) + expect(calls[0].query).not.toContain('repo_url') + expect(calls[0].query).toContain("status = 'running'") + expect(calls[0].params).toEqual({ + id: 'job-1', + inputTokens: 300, + outputTokens: 20, + costUsd: null, + }) + }) +}) + +describe('markMemberContributorJobUnresolvable', () => { + it('sets unresolvable with reason and repo url unless the job is done', async () => { + const { qx, calls } = fakeQx() + await markMemberContributorJobUnresolvable(qx, 'job-1', 'LLM returned no repository', null) + expect(calls[0].query).toContain("status = 'unresolvable'") + expect(calls[0].query).toContain('repo_url = COALESCE($(repoUrl), repo_url)') + expect(calls[0].query).toContain("WHERE id = $(id) AND status <> 'done'") + expect(calls[0].params).toEqual({ + id: 'job-1', + reason: 'LLM returned no repository', + repoUrl: null, + }) + }) +}) + +describe('incrementMemberContributorJobSubmissions', () => { + it('increments submission_count', async () => { + const { qx, calls } = fakeQx() + await incrementMemberContributorJobSubmissions(qx, 'job-1') + expect(calls[0].query).toContain('submission_count = submission_count + 1') + expect(calls[0].params).toEqual({ id: 'job-1' }) + }) +}) + +describe('findMemberContributorJob columns', () => { + it('selects the collection path and inferred repo columns', async () => { + const { qx, calls } = fakeQx(null) + await findMemberContributorJob(qx, 'job-1') + expect(calls[0].query).toContain('collection_path AS "collectionPath"') + expect(calls[0].query).toContain('repo_url AS "repoUrl"') + expect(calls[0].query).toContain('repo_url_source AS "repoUrlSource"') + }) +}) diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts index 5786aa8275..4d8a26e88d 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts @@ -1,24 +1,51 @@ import { QueryExecutor } from '../queryExecutor' -export type MemberContributorCollectionPath = 'github' -export type MemberContributorJobStatus = 'pending' | 'running' | 'done' | 'failed' +export type MemberContributorCollectionPath = 'github' | 'llm_repo_resolution' +export type MemberContributorJobStatus = 'pending' | 'running' | 'done' | 'failed' | 'unresolvable' +export type MemberContributorRepoUrlSource = 'llm_inferred' export interface MemberContributorJobInput { id: string purl: string ecosystem: string collectionPath: MemberContributorCollectionPath - packageId: string - repoId: string + packageId: string | null + repoId: string | null } export interface MemberContributorJobRow { id: string purl: string + ecosystem: string + collectionPath: MemberContributorCollectionPath status: MemberContributorJobStatus error: string | null packageId: string | null repoId: string | null + repoUrl: string | null + repoUrlSource: MemberContributorRepoUrlSource | null +} + +export interface MemberContributorJobPurl { + purl: string + ecosystem: string +} + +export interface LatestMemberContributorJob { + id: string + status: MemberContributorJobStatus + collectionPath: MemberContributorCollectionPath +} + +export interface LlmUsage { + inputTokens: number + outputTokens: number + costUsd: number | null +} + +export interface InferredRepoRecord extends LlmUsage { + repoUrl: string + confidence: number } export interface MemberContributorJobRepo { @@ -82,8 +109,10 @@ export async function findMemberContributorJob( ): Promise { return qx.selectOneOrNone( ` - SELECT id, purl, status, error, - package_id::text AS "packageId", repo_id::text AS "repoId" + SELECT id, purl, ecosystem, status, error, + collection_path AS "collectionPath", + package_id::text AS "packageId", repo_id::text AS "repoId", + repo_url AS "repoUrl", repo_url_source AS "repoUrlSource" FROM member_contributor_jobs WHERE id = $(id) `, @@ -145,3 +174,88 @@ export async function failMemberContributorJob( { id, error }, ) } + +export async function findMemberContributorJobPurl( + qx: QueryExecutor, + id: string, +): Promise { + return qx.selectOneOrNone(`SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)`, { + id, + }) +} + +export async function findLatestMemberContributorJobByPurl( + qx: QueryExecutor, + purl: string, +): Promise { + return qx.selectOneOrNone( + ` + SELECT id, status, collection_path AS "collectionPath" + FROM member_contributor_jobs + WHERE purl = $(purl) + ORDER BY created_at DESC, id DESC + LIMIT 1 + `, + { purl }, + ) +} + +export async function recordMemberContributorJobInferredRepo( + qx: QueryExecutor, + id: string, + record: InferredRepoRecord, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET repo_url = $(repoUrl), repo_url_source = 'llm_inferred', llm_confidence = $(confidence), + llm_input_tokens = $(inputTokens), llm_output_tokens = $(outputTokens), llm_cost_usd = $(costUsd), + updated_at = NOW() + WHERE id = $(id) AND status = 'running' + `, + { id, ...record }, + ) +} + +export async function recordMemberContributorJobLlmUsage( + qx: QueryExecutor, + id: string, + usage: LlmUsage, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET llm_input_tokens = $(inputTokens), llm_output_tokens = $(outputTokens), llm_cost_usd = $(costUsd), + updated_at = NOW() + WHERE id = $(id) AND status = 'running' + `, + { id, ...usage }, + ) +} + +export async function markMemberContributorJobUnresolvable( + qx: QueryExecutor, + id: string, + reason: string, + repoUrl: string | null, +): Promise { + await qx.result( + ` + UPDATE member_contributor_jobs + SET status = 'unresolvable', error = $(reason), repo_url = COALESCE($(repoUrl), repo_url), + completed_at = NOW(), updated_at = NOW() + WHERE id = $(id) AND status <> 'done' + `, + { id, reason, repoUrl }, + ) +} + +export async function incrementMemberContributorJobSubmissions( + qx: QueryExecutor, + id: string, +): Promise { + await qx.result( + `UPDATE member_contributor_jobs SET submission_count = submission_count + 1, updated_at = NOW() WHERE id = $(id)`, + { id }, + ) +} From d08a68ed993d59549b1c15924c1bacf757d70811 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:30:10 +0000 Subject: [PATCH 02/23] feat: look up affiliations for a list of identity keys Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../src/osspckgs/identityAffiliations.test.ts | 45 +++++++++++++++++++ .../src/osspckgs/identityAffiliations.ts | 30 +++++++++++++ 2 files changed, 75 insertions(+) diff --git a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts index f470be357b..81892a7c46 100644 --- a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts @@ -5,6 +5,7 @@ import { IdentityKey, StagingAffiliationRow, fillIdentityAffiliationsStaging, + findAffiliationsForIdentityKeys, findIdentityKeysWithoutAffiliation, insertIdentityAffiliations, mergeIdentityAffiliationsStaging, @@ -314,3 +315,47 @@ describe('mergeIdentityAffiliationsStaging delete guard', () => { expect(del?.params).toEqual({ runStartedAt: RUN_STARTED_AT }) }) }) + +describe('findAffiliationsForIdentityKeys', () => { + it('returns nothing without a query for no keys', async () => { + const { qx, calls } = fakeQx() + await expect(findAffiliationsForIdentityKeys(qx, [])).resolves.toEqual([]) + expect(calls).toHaveLength(0) + }) + + it('reads affiliated rows for the given keys in one query shaped like the sync rows', async () => { + const row = { + person: 'cdp-member-1', + source: 'github', + role: null, + roleKind: 'contributor', + identityType: 'github-login', + identityValue: 'octo', + endedAt: null, + lastSeenAt: null, + registryUsername: null, + registryDisplayName: null, + akritesMember: 'Google', + cdpOrganizationId: 'org-google', + confidence: 'cdp', + confidenceScore: '1.0', + } + const { qx, calls } = fakeQx({ selectRows: [row] }) + const rows = await findAffiliationsForIdentityKeys(qx, [ + { identityType: 'github-login', identityValue: 'octo' }, + { identityType: 'github-login', identityValue: 'ada' }, + ]) + expect(rows).toEqual([row]) + expect(calls).toHaveLength(1) + expect(calls[0].query).toContain('FROM unnest($(types)::text[], $(values)::text[])') + expect(calls[0].query).toContain('JOIN identity_affiliations a') + expect(calls[0].query).toContain('WHERE a.akrites_member IS NOT NULL') + expect(calls[0].query).toContain( + "COALESCE(a.cdp_member_id::text, a.identity_type || ':' || a.identity_value) AS person", + ) + expect(calls[0].params).toEqual({ + types: ['github-login', 'github-login'], + values: ['octo', 'ada'], + }) + }) +}) diff --git a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts index e592506973..0a198c96f4 100644 --- a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts +++ b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.ts @@ -1,4 +1,5 @@ import { QueryExecutor, formatQuery } from '../queryExecutor' +import type { AffiliatedMemberIdentityRow } from './memberContributors' export type IdentityType = 'github-login' | 'email' export type AffiliationConfidence = 'cdp' | 'email_domain' | 'github_company' @@ -160,6 +161,35 @@ export async function findIdentityKeysWithoutAffiliation( ) } +export async function findAffiliationsForIdentityKeys( + qx: QueryExecutor, + keys: IdentityKey[], +): Promise { + if (keys.length === 0) { + return [] + } + return qx.select( + ` + SELECT COALESCE(a.cdp_member_id::text, a.identity_type || ':' || a.identity_value) AS person, + 'github'::text AS source, NULL::text AS role, 'contributor'::text AS "roleKind", + a.identity_type AS "identityType", a.identity_value AS "identityValue", + NULL::timestamptz AS "endedAt", NULL::timestamptz AS "lastSeenAt", + NULL::text AS "registryUsername", NULL::text AS "registryDisplayName", + a.akrites_member AS "akritesMember", a.cdp_organization_id::text AS "cdpOrganizationId", + a.confidence, a.confidence_score AS "confidenceScore" + FROM unnest($(types)::text[], $(values)::text[]) AS k(identity_type, identity_value) + JOIN identity_affiliations a + ON a.identity_type = k.identity_type AND a.identity_value = k.identity_value + WHERE a.akrites_member IS NOT NULL + ORDER BY person, a.identity_type, a.identity_value + `, + { + types: keys.map((key) => key.identityType), + values: keys.map((key) => key.identityValue), + }, + ) +} + export async function insertIdentityAffiliations( qx: QueryExecutor, rows: StagingAffiliationRow[], From 248f5f64f027dc6203fa4e0531243e8f5b29d4b8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:33:08 +0000 Subject: [PATCH 03/23] refactor: move the member contributors body builder into the data access layer Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- ...tAkritesExternalMemberContributors.test.ts | 3 +- .../getAkritesExternalMemberContributors.ts | 6 +-- .../packages/getMemberContributorsJob.test.ts | 3 +- .../v1/packages/getMemberContributorsJob.ts | 8 ++-- .../v1/packages/memberContributorsJob.ts | 3 +- services/libs/data-access-layer/src/index.ts | 1 + .../data-access-layer/src/osspckgs/index.ts | 1 + .../osspckgs/memberContributorsBody.test.ts | 44 +++++++++++++++++-- .../src/osspckgs/memberContributorsBody.ts | 41 +++++++++++++++-- 9 files changed, 90 insertions(+), 20 deletions(-) rename backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts => services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts (92%) rename backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts => services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts (89%) diff --git a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts index e3c6abb001..7ff262d040 100644 --- a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts +++ b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.test.ts @@ -19,7 +19,8 @@ vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}), })) -vi.mock('@crowd/data-access-layer', () => ({ +vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ + ...(await importOriginal()), findMemberContributorsPackage, findAffiliatedMemberIdentities, })) diff --git a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts index f406196201..76ccc38844 100644 --- a/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts +++ b/backend/src/api/public/v1/packages/getAkritesExternalMemberContributors.ts @@ -7,12 +7,10 @@ import { NotFoundError } from '@crowd/common' import { findAffiliatedMemberIdentities, findMemberContributorsPackage, -} from '@crowd/data-access-layer' - -import { groupAffiliatedMemberIdentities, toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' +} from '@crowd/data-access-layer' + import { purlQuerySchema } from './purl' export async function getAkritesExternalMemberContributors( diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index 372a19668e..782a54dbbd 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -19,7 +19,8 @@ const { findMemberContributorJob, findMemberContributorsPackage, findAffiliatedM })) vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}) })) -vi.mock('@crowd/data-access-layer', () => ({ +vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ + ...(await importOriginal()), findMemberContributorJob, findMemberContributorsPackage, findAffiliatedMemberIdentities, diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts index 0f0704ec87..c54d3eec5a 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts @@ -6,18 +6,16 @@ import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' import { InternalError, NotFoundError } from '@crowd/common' import { + AkritesExternalMemberContributors, MemberContributorJobRow, findAffiliatedMemberIdentities, findMemberContributorJob, findMemberContributorsPackage, + groupAffiliatedMemberIdentities, + toAkritesExternalMemberContributors, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' -import { - AkritesExternalMemberContributors, - groupAffiliatedMemberIdentities, - toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' import { AkritesExternalMemberContributorsJob, toDoneMemberContributorsJob, diff --git a/backend/src/api/public/v1/packages/memberContributorsJob.ts b/backend/src/api/public/v1/packages/memberContributorsJob.ts index 82a36af8b6..21b5d42b2c 100644 --- a/backend/src/api/public/v1/packages/memberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/memberContributorsJob.ts @@ -1,7 +1,6 @@ +import type { AkritesExternalMemberContributors } from '@crowd/data-access-layer' import { TemporalWorkflowId } from '@crowd/types' -import type { AkritesExternalMemberContributors } from './akritesExternalMemberContributors' - export const MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE = 'security-contacts-worker' export const MEMBER_CONTRIBUTORS_JOB_WORKFLOW = 'collectMemberContributors' diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 61a3ce19a8..191766abdf 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -44,4 +44,5 @@ export * from './osspckgs/api' export * from './osspckgs/akritesMembers' export * from './osspckgs/identityAffiliations' export * from './osspckgs/memberContributors' +export * from './osspckgs/memberContributorsBody' export * from './osspckgs/memberContributorJobs' diff --git a/services/libs/data-access-layer/src/osspckgs/index.ts b/services/libs/data-access-layer/src/osspckgs/index.ts index 163917b2bd..f08da9191d 100644 --- a/services/libs/data-access-layer/src/osspckgs/index.ts +++ b/services/libs/data-access-layer/src/osspckgs/index.ts @@ -8,4 +8,5 @@ export * from './stewardships' export * from './api' export * from './identityAffiliations' export * from './memberContributors' +export * from './memberContributorsBody' export * from './memberContributorJobs' diff --git a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts similarity index 92% rename from backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts rename to services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts index 39fa712553..a5e90c651d 100644 --- a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts @@ -3,12 +3,13 @@ import { describe, expect, it } from 'vitest' import type { AffiliatedMemberIdentityRow, MemberContributorsPackageRow, -} from '@crowd/data-access-layer' - + MemberIdentitySource, +} from './memberContributors' import { groupAffiliatedMemberIdentities, toAkritesExternalMemberContributors, -} from './akritesExternalMemberContributors' + toInferredRepoMemberContributorsBody, +} from './memberContributorsBody' const GOOGLE = { akritesMember: 'Google', @@ -431,3 +432,40 @@ describe('toAkritesExternalMemberContributors', () => { expect(result.maintainers.registry).toHaveLength(1) }) }) + +describe('toInferredRepoMemberContributorsBody', () => { + it('builds the sync body with llm provenance, null mapping confidence and empty maintainers', () => { + const entry = { + githubLogin: 'octo', + displayName: null, + emails: [], + registryUsernames: [], + sources: ['github'] as MemberIdentitySource[], + organization: { + id: 'org-google', + name: 'Google', + confidence: 'cdp' as const, + confidenceScore: 1, + }, + } + const body = toInferredRepoMemberContributorsBody( + { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + }, + { contributors: [entry], registry: [], governanceFiles: [] }, + ) + expect(body).toEqual({ + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [entry], + maintainers: { registry: [], governanceFiles: [] }, + }) + }) +}) diff --git a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts similarity index 89% rename from backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts rename to services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts index c722a692e7..60656a0544 100644 --- a/backend/src/api/public/v1/packages/akritesExternalMemberContributors.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.ts @@ -1,12 +1,14 @@ +import { canonicalCdpOrganizationIdByMemberName } from './akritesMembers' +import type { AffiliationConfidence } from './identityAffiliations' import type { AffiliatedMemberIdentityRow, - AffiliationConfidence, MemberContributorsPackageRow, MemberIdentitySource, -} from '@crowd/data-access-layer' -import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' +} from './memberContributors' -import { toNullableNumber } from './mappers' +function toNullableNumber(value: number | string | null): number | null { + return value != null ? Number(value) : null +} const CANONICAL_ORGANIZATION_IDS = canonicalCdpOrganizationIdByMemberName() @@ -276,3 +278,34 @@ export function toAkritesExternalMemberContributors( }, } } + +export interface InferredRepoMemberContributorsBody extends Omit< + AkritesExternalMemberContributors, + 'status' | 'repoMappingConfidence' +> { + repoMappingConfidence: null + repoUrlSource: 'llm_inferred' +} + +export interface InferredRepoPackage { + purl: string + name: string + ecosystem: string + repoUrl: string +} + +export function toInferredRepoMemberContributorsBody( + pkg: InferredRepoPackage, + sections: MemberContributorsSections, +): InferredRepoMemberContributorsBody { + return { + purl: pkg.purl, + name: pkg.name, + ecosystem: pkg.ecosystem, + repoUrl: pkg.repoUrl, + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: sections.contributors, + maintainers: { registry: [], governanceFiles: [] }, + } +} From 36c12ff2fa99d93e9fc44242d7d80ebd940ecd1b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:35:27 +0000 Subject: [PATCH 04/23] feat: route unknown ecosystems to llm repo resolution Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../memberContributorCollectionPaths.test.ts | 12 ++++-------- .../v1/packages/memberContributorCollectionPaths.ts | 6 ++++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index 60a50cf6bf..57ae8419bc 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -28,13 +28,9 @@ describe('collectionPathForPurlType', () => { expect(Object.keys(MEMBER_CONTRIBUTOR_COLLECTION_PATHS)).toHaveLength(8) }) - it('returns null for purl types we do not collect', () => { - expect(collectionPathForPurlType('hackage')).toBeNull() - expect(collectionPathForPurlType('conan')).toBeNull() - expect(collectionPathForPurlType('')).toBeNull() - expect(collectionPathForPurlType('constructor')).toBeNull() - expect(collectionPathForPurlType('go')).toBeNull() - expect(collectionPathForPurlType('packagist')).toBeNull() - expect(collectionPathForPurlType('rubygems')).toBeNull() + it('routes any purl type outside the map to llm repo resolution', () => { + expect(collectionPathForPurlType('hackage')).toBe('llm_repo_resolution') + expect(collectionPathForPurlType('conan')).toBe('llm_repo_resolution') + expect(collectionPathForPurlType('hasOwnProperty')).toBe('llm_repo_resolution') }) }) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts index 6ec1450618..3d5b0d8afa 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts @@ -19,8 +19,10 @@ export function purlType(purl: string): string { return type.toLowerCase() } -export function collectionPathForPurlType(type: string): MemberContributorCollectionPath | null { +export const LLM_REPO_RESOLUTION_PATH: MemberContributorCollectionPath = 'llm_repo_resolution' + +export function collectionPathForPurlType(type: string): MemberContributorCollectionPath { return Object.hasOwn(MEMBER_CONTRIBUTOR_COLLECTION_PATHS, type) ? MEMBER_CONTRIBUTOR_COLLECTION_PATHS[type] - : null + : LLM_REPO_RESOLUTION_PATH } From a27ed2ccdc49549a8f9eb555818286cad5687f80 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:36:42 +0000 Subject: [PATCH 05/23] feat: accept member contributors jobs for unknown ecosystems Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../v1/packages/memberContributorsJob.ts | 3 + .../submitMemberContributorsJob.test.ts | 108 +++++++++++++-- .../packages/submitMemberContributorsJob.ts | 130 ++++++++++++++---- 3 files changed, 207 insertions(+), 34 deletions(-) diff --git a/backend/src/api/public/v1/packages/memberContributorsJob.ts b/backend/src/api/public/v1/packages/memberContributorsJob.ts index 21b5d42b2c..f4eedae58e 100644 --- a/backend/src/api/public/v1/packages/memberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/memberContributorsJob.ts @@ -3,6 +3,9 @@ import { TemporalWorkflowId } from '@crowd/types' export const MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE = 'security-contacts-worker' export const MEMBER_CONTRIBUTORS_JOB_WORKFLOW = 'collectMemberContributors' +export const MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW = + 'collectMemberContributorsFromInferredRepo' +export const RESULT_EXPIRED_REASON = 'Job result expired, resubmit the job' export interface AkritesExternalMemberContributorsJobPending { jobId: string diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index fa33422bec..df9b502534 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -15,6 +15,8 @@ const { findMemberContributorsPackage, findActiveMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, + findLatestMemberContributorJobByPurl, + incrementMemberContributorJobSubmissions, createMemberContributorJob, failMemberContributorJob, } = vi.hoisted(() => ({ @@ -24,6 +26,8 @@ const { findMemberContributorsPackage: vi.fn(), findActiveMemberContributorJobByPurl: vi.fn(), findLatestMemberContributorJobStatusByPurl: vi.fn(), + findLatestMemberContributorJobByPurl: vi.fn(), + incrementMemberContributorJobSubmissions: vi.fn(), createMemberContributorJob: vi.fn(), failMemberContributorJob: vi.fn(), })) @@ -50,6 +54,8 @@ vi.mock('@crowd/data-access-layer', () => ({ findMemberContributorsPackage, findActiveMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, + findLatestMemberContributorJobByPurl, + incrementMemberContributorJobSubmissions, createMemberContributorJob, failMemberContributorJob, })) @@ -102,6 +108,8 @@ beforeEach(() => { findMemberContributorsPackage.mockResolvedValue(packageRow()) findActiveMemberContributorJobByPurl.mockResolvedValue(null) findLatestMemberContributorJobStatusByPurl.mockResolvedValue(null) + findLatestMemberContributorJobByPurl.mockResolvedValue(null) + incrementMemberContributorJobSubmissions.mockResolvedValue(undefined) createMemberContributorJob.mockImplementation(async (_qx, input) => input.id) failMemberContributorJob.mockResolvedValue(undefined) }) @@ -170,16 +178,6 @@ describe('submitMemberContributorsJob', () => { expect(status).toHaveBeenCalledWith(202) }) - it('rejects an ecosystem we do not collect with 400 before any database read', async () => { - const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) - const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) - expect(error).toBeInstanceOf(BadRequestError) - expect((error as BadRequestError).message).toBe('Unsupported ecosystem: hackage') - expect(findMemberContributorsPackage).not.toHaveBeenCalled() - expect(createMemberContributorJob).not.toHaveBeenCalled() - expect(start).not.toHaveBeenCalled() - }) - it('rejects an unknown purl with 404', async () => { findMemberContributorsPackage.mockResolvedValue(null) const { req, res } = mockReqRes({ purl: 'pkg:npm/nope' }) @@ -262,6 +260,7 @@ describe('submitMemberContributorsJob', () => { await submitMemberContributorsJob(req, res) expect(createMemberContributorJob).not.toHaveBeenCalled() + expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'existing-job') expectRedispatched('existing-job') expect(status).toHaveBeenCalledWith(202) expect(json).toHaveBeenCalledWith({ jobId: 'existing-job', status: 'pending' }) @@ -365,3 +364,92 @@ describe('submitMemberContributorsJob', () => { expect(status).not.toHaveBeenCalled() }) }) + +describe('submitMemberContributorsJob, unknown ecosystem', () => { + it('creates an llm repo resolution job without a package lookup and starts the inferred-repo workflow', async () => { + const { req, res, status, json } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect(findMemberContributorsPackage).not.toHaveBeenCalled() + const [, input] = createMemberContributorJob.mock.calls[0] + expect(input).toMatchObject({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + collectionPath: 'llm_repo_resolution', + packageId: null, + repoId: null, + }) + const [workflowType, options] = start.mock.calls[0] + expect(workflowType).toBe('collectMemberContributorsFromInferredRepo') + expect(options).toMatchObject({ + taskQueue: 'security-contacts-worker', + workflowId: `member-contributors-job/${input.id}`, + args: [{ jobId: input.id }], + }) + expect(status).toHaveBeenCalledWith(202) + expect(json).toHaveBeenCalledWith({ jobId: input.id, status: 'pending' }) + }) + + it('returns the active job and counts the submission', async () => { + findActiveMemberContributorJobByPurl.mockResolvedValue({ id: 'active-1' }) + const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'active-1') + expect(createMemberContributorJob).not.toHaveBeenCalled() + expect(status).toHaveBeenCalledWith(202) + expect(start.mock.calls[0][0]).toBe('collectMemberContributorsFromInferredRepo') + }) + + it('refuses with 409 carrying the previous jobId while its result is still readable', async () => { + findLatestMemberContributorJobByPurl.mockResolvedValue({ + id: 'done-1', + status: 'done', + collectionPath: 'llm_repo_resolution', + }) + describeWorkflow.mockResolvedValue({ status: { name: 'COMPLETED' } }) + const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(ConflictError) + expect((error as ConflictError).context).toEqual({ jobId: 'done-1' }) + expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'done-1') + expect(createMemberContributorJob).not.toHaveBeenCalled() + }) + + it('accepts a new job when the previous done result has expired', async () => { + findLatestMemberContributorJobByPurl.mockResolvedValue({ + id: 'done-1', + status: 'done', + collectionPath: 'llm_repo_resolution', + }) + describeWorkflow.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) + const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect(createMemberContributorJob).toHaveBeenCalledTimes(1) + expect(status).toHaveBeenCalledWith(202) + }) + + it.each(['failed', 'unresolvable'] as const)( + 'accepts a new job when the latest job is %s', + async (latest) => { + findLatestMemberContributorJobByPurl.mockResolvedValue({ + id: 'old-1', + status: latest, + collectionPath: 'llm_repo_resolution', + }) + const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + expect(describeWorkflow).not.toHaveBeenCalled() + expect(createMemberContributorJob).toHaveBeenCalledTimes(1) + expect(status).toHaveBeenCalledWith(202) + }, + ) +}) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts index 28120ed5e5..92f3b7bcf1 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts @@ -3,15 +3,17 @@ import type { Request, Response } from 'express' import { getPackagesQx } from '@/db/packagesDb' import { getPackagesTemporalClient } from '@/db/packagesTemporal' import { validateOrThrow } from '@/utils/validation' -import { BadRequestError, ConflictError, NotFoundError, generateUUIDv4 } from '@crowd/common' +import { ConflictError, NotFoundError, generateUUIDv4 } from '@crowd/common' import { MemberContributorCollectionPath, MemberContributorsPackageRow, createMemberContributorJob, failMemberContributorJob, findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, findLatestMemberContributorJobStatusByPurl, findMemberContributorsPackage, + incrementMemberContributorJobSubmissions, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { @@ -19,10 +21,16 @@ import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy, WorkflowIdReusePolicy, + WorkflowNotFoundError, } from '@crowd/temporal' -import { collectionPathForPurlType, purlType } from './memberContributorCollectionPaths' import { + LLM_REPO_RESOLUTION_PATH, + collectionPathForPurlType, + purlType, +} from './memberContributorCollectionPaths' +import { + MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE, MEMBER_CONTRIBUTORS_JOB_WORKFLOW, memberContributorsJobWorkflowId, @@ -37,11 +45,7 @@ function resolveCollectionPath(purl: string): { collectionPath: MemberContributorCollectionPath } { const type = purlType(purl) - const collectionPath = collectionPathForPurlType(type) - if (collectionPath === null) { - throw new BadRequestError(`Unsupported ecosystem: ${type}`) - } - return { purlType: type, collectionPath } + return { purlType: type, collectionPath: collectionPathForPurlType(type) } } async function findLinkedPackage(qx: QueryExecutor, purl: string): Promise { @@ -88,13 +92,10 @@ function jobWorkflowStartOptions(jobId: string) { } } -async function ensureJobWorkflowStarted(jobId: string): Promise { +async function ensureJobWorkflowStarted(jobId: string, workflowType: string): Promise { const packagesTemporal = await getPackagesTemporalClient() try { - await packagesTemporal.workflow.start( - MEMBER_CONTRIBUTORS_JOB_WORKFLOW, - jobWorkflowStartOptions(jobId), - ) + await packagesTemporal.workflow.start(workflowType, jobWorkflowStartOptions(jobId)) } catch (err) { if (!(err instanceof WorkflowExecutionAlreadyStartedError)) { throw err @@ -111,14 +112,15 @@ async function workflowExists(packagesTemporal: Client, jobId: string): Promise< } } -async function startJobWorkflow(qx: QueryExecutor, jobId: string): Promise { +async function startJobWorkflow( + qx: QueryExecutor, + jobId: string, + workflowType: string, +): Promise { let packagesTemporal: Client | undefined try { packagesTemporal = await getPackagesTemporalClient() - await packagesTemporal.workflow.start( - MEMBER_CONTRIBUTORS_JOB_WORKFLOW, - jobWorkflowStartOptions(jobId), - ) + await packagesTemporal.workflow.start(workflowType, jobWorkflowStartOptions(jobId)) } catch (err) { if (err instanceof WorkflowExecutionAlreadyStartedError) { return @@ -132,17 +134,94 @@ async function startJobWorkflow(qx: QueryExecutor, jobId: string): Promise } } +async function workflowResultAvailable(jobId: string): Promise { + const packagesTemporal = await getPackagesTemporalClient() + try { + const description = await packagesTemporal.workflow + .getHandle(memberContributorsJobWorkflowId(jobId)) + .describe() + return description.status.name === 'COMPLETED' + } catch (err) { + if (err instanceof WorkflowNotFoundError) { + return false + } + throw err + } +} + +async function assertInferredRepoResultNotReadable(qx: QueryExecutor, purl: string): Promise { + const latest = await findLatestMemberContributorJobByPurl(qx, purl) + if (!latest || latest.status !== 'done' || latest.collectionPath !== LLM_REPO_RESOLUTION_PATH) { + return + } + if (!(await workflowResultAvailable(latest.id))) { + return + } + await incrementMemberContributorJobSubmissions(qx, latest.id) + throw new ConflictError('Member contributors already collected for this purl', { + jobId: latest.id, + }) +} + +async function respondWithActiveJob( + qx: QueryExecutor, + jobId: string, + workflowType: string, + res: Response, +): Promise { + await incrementMemberContributorJobSubmissions(qx, jobId) + await ensureJobWorkflowStarted(jobId, workflowType) + res.status(202).json(toPendingMemberContributorsJob(jobId)) +} + +async function submitInferredRepoJob( + qx: QueryExecutor, + purl: string, + ecosystem: string, + res: Response, +): Promise { + const active = await findActiveMemberContributorJobByPurl(qx, purl) + if (active) { + await respondWithActiveJob(qx, active.id, MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, res) + return + } + await assertInferredRepoResultNotReadable(qx, purl) + const created = await createMemberContributorJob(qx, { + id: generateUUIDv4(), + purl, + ecosystem, + collectionPath: LLM_REPO_RESOLUTION_PATH, + packageId: null, + repoId: null, + }) + if (created === null) { + await respondWithActiveJob( + qx, + await findActiveJobIdOrThrow(qx, purl), + MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, + res, + ) + return + } + await startJobWorkflow(qx, created, MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW) + res.status(202).json(toPendingMemberContributorsJob(created)) +} + export async function submitMemberContributorsJob(req: Request, res: Response): Promise { const { purl } = validateOrThrow(purlBodySchema, req.body) - const { collectionPath } = resolveCollectionPath(purl) + const { purlType: type, collectionPath } = resolveCollectionPath(purl) const qx = await getPackagesQx() + if (collectionPath === LLM_REPO_RESOLUTION_PATH) { + await submitInferredRepoJob(qx, purl, type, res) + return + } + const pkg = await findLinkedPackage(qx, purl) const active = await findActiveMemberContributorJobByPurl(qx, purl) if (active) { - await ensureJobWorkflowStarted(active.id) - res.status(202).json(toPendingMemberContributorsJob(active.id)) + await respondWithActiveJob(qx, active.id, MEMBER_CONTRIBUTORS_JOB_WORKFLOW, res) return } @@ -157,12 +236,15 @@ export async function submitMemberContributorsJob(req: Request, res: Response): repoId: pkg.repoId, }) if (created === null) { - const activeJobId = await findActiveJobIdOrThrow(qx, purl) - await ensureJobWorkflowStarted(activeJobId) - res.status(202).json(toPendingMemberContributorsJob(activeJobId)) + await respondWithActiveJob( + qx, + await findActiveJobIdOrThrow(qx, purl), + MEMBER_CONTRIBUTORS_JOB_WORKFLOW, + res, + ) return } - await startJobWorkflow(qx, created) + await startJobWorkflow(qx, created, MEMBER_CONTRIBUTORS_JOB_WORKFLOW) res.status(202).json(toPendingMemberContributorsJob(created)) } From 9a9bd7d3debc1138e50ae8c6bc992b080c49f001 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:41:02 +0000 Subject: [PATCH 06/23] feat: serve unresolvable and llm inferred member contributors job results Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../packages/getMemberContributorsJob.test.ts | 106 +++++++++++++++++- .../v1/packages/getMemberContributorsJob.ts | 57 +++++++++- .../v1/packages/memberContributorsJob.ts | 37 +++++- 3 files changed, 186 insertions(+), 14 deletions(-) diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index 782a54dbbd..97bb50a0fd 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -8,16 +8,33 @@ import type { MemberContributorsPackageRow, } from '@crowd/data-access-layer' import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' +import { WorkflowNotFoundError } from '@crowd/temporal' import { getMemberContributorsJob } from './getMemberContributorsJob' -const { findMemberContributorJob, findMemberContributorsPackage, findAffiliatedMemberIdentities } = - vi.hoisted(() => ({ - findMemberContributorJob: vi.fn(), - findMemberContributorsPackage: vi.fn(), - findAffiliatedMemberIdentities: vi.fn(), - })) +const { + getHandle, + result, + findMemberContributorJob, + findMemberContributorsPackage, + findAffiliatedMemberIdentities, +} = vi.hoisted(() => ({ + getHandle: vi.fn(), + result: vi.fn(), + findMemberContributorJob: vi.fn(), + findMemberContributorsPackage: vi.fn(), + findAffiliatedMemberIdentities: vi.fn(), +})) +vi.mock('@/db/packagesTemporal', () => ({ + getPackagesTemporalClient: vi.fn().mockResolvedValue({ workflow: { getHandle } }), +})) +vi.mock('@crowd/temporal', () => { + class MockWorkflowNotFoundError extends Error { + name = 'WorkflowNotFoundError' + } + return { WorkflowNotFoundError: MockWorkflowNotFoundError } +}) vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}) })) vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ ...(await importOriginal()), @@ -86,6 +103,7 @@ function mockReqRes(params: Record) { beforeEach(() => { vi.clearAllMocks() + getHandle.mockReturnValue({ result }) }) describe('getMemberContributorsJob', () => { @@ -168,3 +186,79 @@ describe('getMemberContributorsJob', () => { await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError) }) }) + +describe('getMemberContributorsJob, llm repo resolution path', () => { + it('maps unresolvable with reason and the attempted repo url', async () => { + findMemberContributorJob.mockResolvedValue( + jobRow({ + collectionPath: 'llm_repo_resolution', + status: 'unresolvable', + error: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }), + ) + const { req, res, json } = mockReqRes({ jobId: JOB_ID }) + await getMemberContributorsJob(req, res) + expect(json).toHaveBeenCalledWith({ + jobId: JOB_ID, + status: 'unresolvable', + reason: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }) + expect(getHandle).not.toHaveBeenCalled() + }) + + it('returns the workflow result as the done body', async () => { + const body = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + findMemberContributorJob.mockResolvedValue( + jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), + ) + result.mockResolvedValue(body) + const { req, res, json } = mockReqRes({ jobId: JOB_ID }) + await getMemberContributorsJob(req, res) + expect(getHandle).toHaveBeenCalledWith(`member-contributors-job/${JOB_ID}`) + expect(json).toHaveBeenCalledWith({ ...body, jobId: JOB_ID, status: 'done' }) + expect(findMemberContributorsPackage).not.toHaveBeenCalled() + }) + + it('answers failed with the resubmit reason when the workflow result is gone', async () => { + findMemberContributorJob.mockResolvedValue( + jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), + ) + result.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) + const { req, res, json } = mockReqRes({ jobId: JOB_ID }) + await getMemberContributorsJob(req, res) + expect(json).toHaveBeenCalledWith({ + jobId: JOB_ID, + status: 'failed', + reason: 'Job result expired, resubmit the job', + }) + }) + + it('throws an internal error when a done workflow returned no body', async () => { + findMemberContributorJob.mockResolvedValue( + jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), + ) + result.mockResolvedValue(undefined) + const { req, res } = mockReqRes({ jobId: JOB_ID }) + await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError) + }) + + it('keeps reading github path done jobs from the tables', async () => { + findMemberContributorJob.mockResolvedValue(jobRow({ status: 'done' })) + findMemberContributorsPackage.mockResolvedValue(packageRow()) + findAffiliatedMemberIdentities.mockResolvedValue([identityRow()]) + const { req, res } = mockReqRes({ jobId: JOB_ID }) + await getMemberContributorsJob(req, res) + expect(getHandle).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts index c54d3eec5a..3bd8ced613 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts @@ -2,11 +2,13 @@ import type { Request, Response } from 'express' import { z } from 'zod' import { getPackagesQx } from '@/db/packagesDb' +import { getPackagesTemporalClient } from '@/db/packagesTemporal' import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' import { InternalError, NotFoundError } from '@crowd/common' import { AkritesExternalMemberContributors, + InferredRepoMemberContributorsBody, MemberContributorJobRow, findAffiliatedMemberIdentities, findMemberContributorJob, @@ -15,12 +17,18 @@ import { toAkritesExternalMemberContributors, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { WorkflowNotFoundError } from '@crowd/temporal' import { AkritesExternalMemberContributorsJob, + AkritesExternalMemberContributorsJobDone, + AkritesExternalMemberContributorsJobFailed, + RESULT_EXPIRED_REASON, + memberContributorsJobWorkflowId, toDoneMemberContributorsJob, toFailedMemberContributorsJob, toPendingMemberContributorsJob, + toUnresolvableMemberContributorsJob, } from './memberContributorsJob' const paramsSchema = z.object({ jobId: z.uuid() }) @@ -41,17 +49,62 @@ async function readCollectedBody( ) } +function assertNever(value: never): never { + throw new InternalError(`Unhandled job status ${String(value)}`) +} + +async function readInferredRepoBody( + jobId: string, +): Promise { + const packagesTemporal = await getPackagesTemporalClient() + let body: InferredRepoMemberContributorsBody | undefined + try { + body = await packagesTemporal.workflow + .getHandle<() => Promise>( + memberContributorsJobWorkflowId(jobId), + ) + .result() + } catch (err) { + if (err instanceof WorkflowNotFoundError) { + return null + } + throw err + } + if (!body) { + throw new InternalError(`Done member contributors job ${jobId} has no workflow result`) + } + return body +} + +async function toDoneResponse( + qx: QueryExecutor, + job: MemberContributorJobRow, +): Promise { + if (job.collectionPath !== 'llm_repo_resolution') { + return toDoneMemberContributorsJob(job.id, await readCollectedBody(qx, job.purl)) + } + const body = await readInferredRepoBody(job.id) + return body + ? toDoneMemberContributorsJob(job.id, body) + : toFailedMemberContributorsJob(job.id, RESULT_EXPIRED_REASON) +} + async function toJobResponse( qx: QueryExecutor, job: MemberContributorJobRow, ): Promise { switch (job.status) { case 'done': - return toDoneMemberContributorsJob(job.id, await readCollectedBody(qx, job.purl)) + return toDoneResponse(qx, job) case 'failed': return toFailedMemberContributorsJob(job.id, job.error) - default: + case 'unresolvable': + return toUnresolvableMemberContributorsJob(job.id, job.error, job.repoUrl) + case 'pending': + case 'running': return toPendingMemberContributorsJob(job.id) + default: + return assertNever(job.status) } } diff --git a/backend/src/api/public/v1/packages/memberContributorsJob.ts b/backend/src/api/public/v1/packages/memberContributorsJob.ts index f4eedae58e..00e73ffbd6 100644 --- a/backend/src/api/public/v1/packages/memberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/memberContributorsJob.ts @@ -1,4 +1,7 @@ -import type { AkritesExternalMemberContributors } from '@crowd/data-access-layer' +import type { + AkritesExternalMemberContributors, + InferredRepoMemberContributorsBody, +} from '@crowd/data-access-layer' import { TemporalWorkflowId } from '@crowd/types' export const MEMBER_CONTRIBUTORS_JOB_TASK_QUEUE = 'security-contacts-worker' @@ -18,10 +21,18 @@ export interface AkritesExternalMemberContributorsJobFailed { reason: string } -export type AkritesExternalMemberContributorsJobDone = Omit< - AkritesExternalMemberContributors, - 'status' -> & { +export interface AkritesExternalMemberContributorsJobUnresolvable { + jobId: string + status: 'unresolvable' + reason: string + repoUrl: string | null +} + +export type MemberContributorsDoneBody = + | Omit + | InferredRepoMemberContributorsBody + +export type AkritesExternalMemberContributorsJobDone = MemberContributorsDoneBody & { jobId: string status: 'done' } @@ -29,6 +40,7 @@ export type AkritesExternalMemberContributorsJobDone = Omit< export type AkritesExternalMemberContributorsJob = | AkritesExternalMemberContributorsJobPending | AkritesExternalMemberContributorsJobFailed + | AkritesExternalMemberContributorsJobUnresolvable | AkritesExternalMemberContributorsJobDone export function memberContributorsJobWorkflowId(jobId: string): string { @@ -48,9 +60,22 @@ export function toFailedMemberContributorsJob( return { jobId, status: 'failed', reason: reason ?? 'Job failed' } } +export function toUnresolvableMemberContributorsJob( + jobId: string, + reason: string | null, + repoUrl: string | null, +): AkritesExternalMemberContributorsJobUnresolvable { + return { + jobId, + status: 'unresolvable', + reason: reason ?? 'Repository could not be resolved', + repoUrl, + } +} + export function toDoneMemberContributorsJob( jobId: string, - body: AkritesExternalMemberContributors, + body: MemberContributorsDoneBody, ): AkritesExternalMemberContributorsJobDone { return { ...body, jobId, status: 'done' } } From f348f87b9605dbf508ef492e4de0463dac1ea765 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:42:56 +0000 Subject: [PATCH 07/23] docs: describe inferred-repo member contributors jobs in the akrites openapi Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../public/v1/akrites-external/openapi.yaml | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index c241337ff6..5b2ecdb154 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -957,6 +957,24 @@ components: type: string example: Repository not found on GitHub + MemberContributorsJobUnresolvable: + type: object + required: [jobId, status, reason, repoUrl] + properties: + jobId: + type: string + format: uuid + status: + type: string + enum: [unresolvable] + reason: + type: string + example: LLM confidence 0.4 below threshold 0.7 + repoUrl: + type: string + nullable: true + description: Repository the job attempted before giving up, when there was one. + MemberContributorsJobDone: allOf: - $ref: "#/components/schemas/MemberContributorsBody" @@ -970,17 +988,29 @@ components: type: string enum: [done] description: Job status; the collected/not_collected status of the sync endpoint is not included in this body. + repoUrlSource: + type: string + enum: [llm_inferred] + description: > + Present only for jobs on ecosystems CDP does not track: the + repository was inferred by a language model and verified to + exist on GitHub. repoMappingConfidence is null on such jobs, + maintainers.registry is empty, and the body is available for a + limited time after completion; afterwards the poll answers + failed with a resubmit reason. MemberContributorsJob: oneOf: - $ref: "#/components/schemas/MemberContributorsJobPending" - $ref: "#/components/schemas/MemberContributorsJobFailed" + - $ref: "#/components/schemas/MemberContributorsJobUnresolvable" - $ref: "#/components/schemas/MemberContributorsJobDone" discriminator: propertyName: status mapping: pending: "#/components/schemas/MemberContributorsJobPending" failed: "#/components/schemas/MemberContributorsJobFailed" + unresolvable: "#/components/schemas/MemberContributorsJobUnresolvable" done: "#/components/schemas/MemberContributorsJobDone" MemberContributorEntry: @@ -1281,8 +1311,12 @@ paths: submission also makes sure the job's workflow is running. Packages that already have collected data are refused with 409, unless the purl's most recent job failed, in which case a new job is accepted and re-collects the - repository; ecosystems CDP does not collect - are refused with 400. Rate-limited independently of the read endpoints. + repository. Purls from ecosystems CDP does not collect are accepted and + routed to a workflow that infers the repository with a language model; + those jobs end done with repoUrlSource llm_inferred or unresolvable, + store no contributor data, and a resubmission while the previous result + is still readable is refused with 409 whose error.context.jobId names + the job to poll. Rate-limited independently of the read endpoints. Requires read:akrites-maintainers, or read:maintainer-roles from the old Self Serve client as a fallback. A done job whose repository reports no human contributors on GitHub leaves the package not_collected; @@ -1307,7 +1341,7 @@ paths: schema: $ref: "#/components/schemas/MemberContributorsJobPending" "400": - description: Malformed purl, or an ecosystem CDP does not collect. + description: Malformed purl. content: application/json: schema: @@ -1331,7 +1365,7 @@ paths: schema: $ref: "#/components/schemas/Error" "409": - description: Member contributors are already collected for this purl and its most recent job did not fail; use the sync endpoint. + description: Member contributors already collected for this purl (stored data on tracked ecosystems, or a still-readable result on inferred-repo jobs; error.context.jobId names it); use the sync endpoint or poll that job. content: application/json: schema: @@ -1348,9 +1382,10 @@ paths: operationId: getMemberContributorsJob summary: Poll an on-demand member-contributors collection job description: > - Returns pending until the job finishes, failed with a reason, or done - with the same body the sync endpoint returns for the purl plus jobId - and status. + Returns pending until the job finishes, failed with a reason, + unresolvable with a reason when no repository could be inferred, or + done with the same body the sync endpoint returns for the purl plus + jobId and status. tags: [Contacts] security: - M2MBearer: From 41fdfa02e6f97226ad2a582aaf94792cb15dee58 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:45:46 +0000 Subject: [PATCH 08/23] feat: infer package repositories with claude sonnet 5.5 on bedrock Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 3 + services/apps/packages_worker/package.json | 3 +- services/apps/packages_worker/src/config.ts | 14 ++ .../packages_worker/src/llm/bedrockClient.ts | 45 ++++++ .../job/__tests__/llmRepoInference.test.ts | 120 +++++++++++++++ .../job/llmRepoInference.ts | 139 ++++++++++++++++++ 6 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 services/apps/packages_worker/src/llm/bedrockClient.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a21a17b1cc..3d9441fd53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1672,6 +1672,9 @@ importers: unzipper: specifier: ^0.12.3 version: 0.12.3 + zod: + specifier: 'catalog:' + version: 4.3.6 devDependencies: '@types/js-yaml': specifier: ^4.0.9 diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index 236bee949c..5aeb3f9ac9 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -107,7 +107,8 @@ "tar": "^7.5.20", "tsx": "catalog:", "undici": "^7.29.1", - "unzipper": "^0.12.3" + "unzipper": "^0.12.3", + "zod": "catalog:" }, "devDependencies": { "@types/js-yaml": "^4.0.9", diff --git a/services/apps/packages_worker/src/config.ts b/services/apps/packages_worker/src/config.ts index de5268b4e6..ece7e6d023 100644 --- a/services/apps/packages_worker/src/config.ts +++ b/services/apps/packages_worker/src/config.ts @@ -139,3 +139,17 @@ export function getReportingProtocolConfig() { llmSecretAccessKey: process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY, } } + +export interface BedrockCredentials { + accessKeyId: string + secretAccessKey: string +} + +export function getBedrockCredentials(): BedrockCredentials | null { + const accessKeyId = process.env.CROWD_AWS_BEDROCK_ACCESS_KEY_ID + const secretAccessKey = process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY + if (!accessKeyId || !secretAccessKey) { + return null + } + return { accessKeyId, secretAccessKey } +} diff --git a/services/apps/packages_worker/src/llm/bedrockClient.ts b/services/apps/packages_worker/src/llm/bedrockClient.ts new file mode 100644 index 0000000000..5dcfecdea9 --- /dev/null +++ b/services/apps/packages_worker/src/llm/bedrockClient.ts @@ -0,0 +1,45 @@ +import { BedrockRuntimeClient, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime' + +import type { BedrockCredentials } from '../config' + +export type BedrockInvoke = (modelId: string, body: string) => Promise + +const clients = new Map() + +export function bedrockClientFor( + region: string, + credentials: BedrockCredentials, +): BedrockRuntimeClient { + const key = `${region}:${credentials.accessKeyId}` + let client = clients.get(key) + if (!client) { + client = new BedrockRuntimeClient({ region, credentials }) + clients.set(key, client) + } + return client +} + +export function invokeBedrockModel( + region: string, + credentials: BedrockCredentials, + timeoutMs: number, +): BedrockInvoke { + return async (modelId, body) => { + const controller = new AbortController() + const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs) + try { + const response = await bedrockClientFor(region, credentials).send( + new InvokeModelCommand({ + modelId, + accept: 'application/json', + contentType: 'application/json', + body, + }), + { abortSignal: controller.signal }, + ) + return response.body.transformToString() + } finally { + clearTimeout(timeoutHandle) + } + } +} diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts new file mode 100644 index 0000000000..f94b1b1a9c --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it, vi } from 'vitest' + +import { inferRepositoryCandidates } from '../llmRepoInference' + +type ResponseBlock = { type: 'thinking'; thinking: string } | { type: 'text'; text: string } + +function bedrockResponse( + text: string | null, + usage = { input_tokens: 300, output_tokens: 80 }, +): string { + const content: ResponseBlock[] = [{ type: 'thinking', thinking: '' }] + if (text !== null) content.push({ type: 'text', text }) + return JSON.stringify({ content, usage }) +} + +describe('inferRepositoryCandidates', () => { + it('sends the purl to sonnet 5.5 and parses the ranked candidates from the first text block', async () => { + const invoke = vi.fn().mockResolvedValue( + bedrockResponse( + JSON.stringify({ + candidates: [ + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, + ], + rationale: 'aeson moved to the haskell org', + }), + ), + ) + + const result = await inferRepositoryCandidates('pkg:hackage/aeson', invoke) + + expect(invoke).toHaveBeenCalledTimes(1) + const [modelId, body] = invoke.mock.calls[0] + expect(modelId).toBe('us.anthropic.claude-sonnet-5-5') + const parsed = JSON.parse(body) + expect(parsed.anthropic_version).toBe('bedrock-2023-05-31') + expect(parsed.messages).toEqual([ + { role: 'user', content: [{ type: 'text', text: 'pkg:hackage/aeson' }] }, + ]) + expect(parsed.system).toContain('canonical source repository') + expect(parsed.output_config).toEqual({ effort: 'low' }) + expect(result).toEqual({ + status: 'answered', + candidates: [ + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, + ], + usage: { inputTokens: 300, outputTokens: 80, costUsd: 300 * 0.000002 + 80 * 0.00001 }, + }) + }) + + it('accepts a fenced JSON answer and caps candidates at three', async () => { + const candidates = [1, 2, 3, 4].map((n) => ({ + repositoryUrl: `https://github.com/o/r${n}`, + confidence: 0.9, + })) + const invoke = vi + .fn() + .mockResolvedValue( + bedrockResponse('```json\n' + JSON.stringify({ candidates, rationale: '' }) + '\n```'), + ) + const result = await inferRepositoryCandidates('pkg:conan/boost', invoke) + expect(result.status).toBe('answered') + if (result.status === 'answered') expect(result.candidates).toHaveLength(3) + }) + + it('reports an empty candidate list as answered', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse(JSON.stringify({ candidates: [], rationale: 'unknown' }))) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ + status: 'answered', + candidates: [], + }) + }) + + it('reports unparseable when there is no text block, keeping the usage', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse(null, { input_tokens: 10, output_tokens: 0 })) + const result = await inferRepositoryCandidates('pkg:conan/x', invoke) + expect(result).toMatchObject({ + status: 'unparseable', + reason: 'LLM answer had no text content', + usage: { inputTokens: 10, outputTokens: 0 }, + }) + expect(result.usage.costUsd).toBeCloseTo(10 * 0.000002, 12) + }) + + it('reports unparseable on invalid JSON', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse('I think it is github.com/haskell/aeson')) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ + status: 'unparseable', + reason: 'LLM answer was not valid JSON', + }) + }) + + it('reports unparseable when the JSON does not match the schema', async () => { + const invoke = vi.fn().mockResolvedValue( + bedrockResponse( + JSON.stringify({ + candidates: [{ repositoryUrl: 'https://github.com/a/b', confidence: 1.7 }], + }), + ), + ) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ + status: 'unparseable', + reason: 'LLM answer did not match the expected schema', + }) + }) + + it('propagates transport errors', async () => { + const invoke = vi.fn().mockRejectedValue(new Error('ThrottlingException')) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( + 'ThrottlingException', + ) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts new file mode 100644 index 0000000000..95506e6fa3 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts @@ -0,0 +1,139 @@ +import { z } from 'zod' + +import { parseLlmJson } from '@crowd/common' +import type { LlmUsage } from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' +import { getServiceChildLogger } from '@crowd/logging' +import { LlmModelType, estimateLlmCostUsd } from '@crowd/types' + +import type { BedrockInvoke } from '../../llm/bedrockClient' + +const log = getServiceChildLogger('member-contributors:llm-repo-inference') + +export const REPO_INFERENCE_MODEL_ID = LlmModelType.CLAUDE_SONNET_5_5 +export const MAX_REPO_CANDIDATES = 3 +const MAX_TOKENS = 512 + +const candidateSchema = z.object({ + repositoryUrl: z.string().min(1), + confidence: z.number().min(0).max(1), +}) +const answerSchema = z.object({ + candidates: z.array(candidateSchema), + rationale: z.string().optional(), +}) + +const ANSWER_JSON_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['candidates', 'rationale'], + properties: { + candidates: { + type: 'array', + maxItems: MAX_REPO_CANDIDATES, + items: { + type: 'object', + additionalProperties: false, + required: ['repositoryUrl', 'confidence'], + properties: { + repositoryUrl: { type: 'string' }, + confidence: { type: 'number', minimum: 0, maximum: 1 }, + }, + }, + }, + rationale: { type: 'string' }, + }, +} + +const SYSTEM_PROMPT = `You resolve a software package, given as a package URL (purl, e.g. pkg:hackage/aeson or pkg:conan/boost), to its canonical source repository. +Rules: +- Answer only from what you know. Never invent a URL you have not seen. +- Return up to ${MAX_REPO_CANDIDATES} candidate GitHub repository URLs ranked by likelihood, each with a confidence between 0 and 1 that it is the package's canonical upstream source repository. +- Prefer the upstream project over forks, mirrors, packaging repositories and distribution recipes. +- If you do not know the repository, return an empty candidates list. +- rationale: one short sentence. +Respond ONLY with a single JSON object matching this JSON schema, no prose: +${JSON.stringify(ANSWER_JSON_SCHEMA)}` + +export interface RepoCandidate { + repositoryUrl: string + confidence: number +} + +export type RepoInference = + | { status: 'answered'; candidates: RepoCandidate[]; usage: LlmUsage } + | { status: 'unparseable'; reason: string; usage: LlmUsage } + +interface BedrockMessageResponse { + content?: Array<{ type?: string; text?: string }> + usage?: { input_tokens?: number; output_tokens?: number } +} + +function requestBody(purl: string): string { + return JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: MAX_TOKENS, + output_config: { effort: 'low' }, + system: SYSTEM_PROMPT, + messages: [{ role: 'user', content: [{ type: 'text', text: purl }] }], + }) +} + +function usageOf(response: BedrockMessageResponse): LlmUsage { + const inputTokens = response.usage?.input_tokens ?? 0 + const outputTokens = response.usage?.output_tokens ?? 0 + return { + inputTokens, + outputTokens, + costUsd: estimateLlmCostUsd(REPO_INFERENCE_MODEL_ID, inputTokens, outputTokens), + } +} + +function firstText(response: BedrockMessageResponse): string | null { + const block = response.content?.find( + (entry) => entry.type === 'text' && typeof entry.text === 'string', + ) + return block?.text ?? null +} + +function parseAnswer(text: string): RepoCandidate[] | string { + let raw: unknown + try { + raw = parseLlmJson(text) + } catch { + return 'LLM answer was not valid JSON' + } + const parsed = answerSchema.safeParse(raw) + if (!parsed.success) { + return 'LLM answer did not match the expected schema' + } + return parsed.data.candidates.slice(0, MAX_REPO_CANDIDATES) +} + +export async function inferRepositoryCandidates( + purl: string, + invoke: BedrockInvoke, +): Promise { + const startedAt = Date.now() + const response = JSON.parse( + await invoke(REPO_INFERENCE_MODEL_ID, requestBody(purl)), + ) as BedrockMessageResponse + const usage = usageOf(response) + const text = firstText(response) + const candidates = text === null ? 'LLM answer had no text content' : parseAnswer(text) + log.info( + { + modelId: REPO_INFERENCE_MODEL_ID, + purl, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + costUsd: usage.costUsd, + candidates: typeof candidates === 'string' ? 0 : candidates.length, + durationMs: Date.now() - startedAt, + }, + 'LLM repo inference call', + ) + if (typeof candidates === 'string') { + return { status: 'unparseable', reason: candidates, usage } + } + return { status: 'answered', candidates, usage } +} From c475cedb35da2e123816e96e87b4d19ffbf101aa Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:50:11 +0000 Subject: [PATCH 09/23] feat: resolve an inferred repository against github before collecting Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../job/__tests__/canonicalGithubUrl.test.ts | 34 ++++ .../job/__tests__/resolveRepoViaLlm.test.ts | 160 ++++++++++++++++++ .../job/canonicalGithubUrl.ts | 35 ++++ .../job/resolveRepoViaLlm.ts | 105 ++++++++++++ 4 files changed, 334 insertions(+) create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts new file mode 100644 index 0000000000..264d6d73da --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' + +import { canonicalGithubUrl } from '../canonicalGithubUrl' + +describe('canonicalGithubUrl', () => { + it.each([ + ['https://github.com/haskell/aeson', 'https://github.com/haskell/aeson'], + ['https://github.com/Haskell/Aeson.git', 'https://github.com/haskell/aeson'], + ['http://www.github.com/haskell/aeson/', 'https://github.com/haskell/aeson'], + ['https://github.com/haskell/aeson/tree/main/src', 'https://github.com/haskell/aeson'], + ['github.com/haskell/aeson', 'https://github.com/haskell/aeson'], + ])('canonicalises %s', (raw, url) => { + expect(canonicalGithubUrl(raw)).toEqual({ + kind: 'github', + url, + owner: 'haskell', + name: 'aeson', + }) + }) + + it('reports other hosts', () => { + expect(canonicalGithubUrl('https://gitlab.com/a/b')).toEqual({ + kind: 'other', + host: 'gitlab.com', + }) + }) + + it.each(['https://github.com/haskell', 'not a url', 'https://github.com/'])( + 'rejects %s', + (raw) => { + expect(canonicalGithubUrl(raw)).toEqual({ kind: 'invalid' }) + }, + ) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts new file mode 100644 index 0000000000..4104567c1c --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import * as jobsDal from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { getBedrockCredentials } from '../../../config' +import { githubApiGet } from '../../../security-contacts/githubToken' +import { inferRepositoryCandidates } from '../llmRepoInference' +import { resolveRepoViaLlm } from '../resolveRepoViaLlm' + +const qx = vi.hoisted(() => ({})) + +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('../../../config', () => ({ getBedrockCredentials: vi.fn() })) +vi.mock('../../../llm/bedrockClient', () => ({ invokeBedrockModel: vi.fn(() => vi.fn()) })) +vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) +vi.mock('../llmRepoInference', async (importOriginal) => ({ + ...(await importOriginal()), + inferRepositoryCandidates: vi.fn(), +})) +vi.mock('@crowd/data-access-layer/src/osspckgs/memberContributorJobs', () => ({ + recordMemberContributorJobInferredRepo: vi.fn(), + recordMemberContributorJobLlmUsage: vi.fn(), +})) + +const usage = { inputTokens: 300, outputTokens: 80, costUsd: 0.0014 } +const input = { jobId: 'job-1', purl: 'pkg:hackage/aeson' } +const recordRepo = vi.mocked(jobsDal.recordMemberContributorJobInferredRepo) +const recordUsage = vi.mocked(jobsDal.recordMemberContributorJobLlmUsage) + +function answered(candidates: Array<{ repositoryUrl: string; confidence: number }>) { + vi.mocked(inferRepositoryCandidates).mockResolvedValue({ status: 'answered', candidates, usage }) +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(getBedrockCredentials).mockReturnValue({ accessKeyId: 'k', secretAccessKey: 's' }) + vi.mocked(githubApiGet).mockResolvedValue({ status: 200, text: '{}' }) +}) + +describe('resolveRepoViaLlm', () => { + it('resolves the first candidate that exists on GitHub and records it on the job', async () => { + answered([{ repositoryUrl: 'https://github.com/Haskell/aeson.git', confidence: 0.95 }]) + + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'resolved', + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.95, + }) + expect(githubApiGet).toHaveBeenCalledWith('/repos/haskell/aeson', 15000) + expect(recordRepo).toHaveBeenCalledWith(qx, 'job-1', { + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.95, + ...usage, + }) + expect(recordUsage).not.toHaveBeenCalled() + }) + + it('skips absent candidates and takes the next existing one', async () => { + answered([ + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.8 }, + ]) + vi.mocked(githubApiGet) + .mockResolvedValueOnce({ status: 404, text: null }) + .mockResolvedValueOnce({ status: 200, text: '{}' }) + + await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ + outcome: 'resolved', + repoUrl: 'https://github.com/haskell/aeson', + }) + }) + + it('is unresolvable with the last attempted url when every candidate is absent', async () => { + answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.8 }, + ]) + vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) + + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'Repository not found on GitHub: https://github.com/a/two', + repoUrl: 'https://github.com/a/two', + }) + expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + expect(recordRepo).not.toHaveBeenCalled() + }) + + it('is unresolvable below the confidence threshold without calling GitHub', async () => { + answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }]) + + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }) + expect(githubApiGet).not.toHaveBeenCalled() + }) + + it('ignores lower-ranked candidates below the threshold', async () => { + answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.2 }, + ]) + vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) + + await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ outcome: 'unresolvable' }) + expect(githubApiGet).toHaveBeenCalledTimes(1) + }) + + it('is unresolvable for an unsupported host', async () => { + answered([{ repositoryUrl: 'https://gitlab.com/a/b', confidence: 0.9 }]) + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'Repository host not supported: gitlab.com', + repoUrl: null, + }) + }) + + it('is unresolvable for an empty candidate list', async () => { + answered([]) + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'LLM returned no repository', + repoUrl: null, + }) + }) + + it('is unresolvable for an unparseable answer and still records usage', async () => { + vi.mocked(inferRepositoryCandidates).mockResolvedValue({ + status: 'unparseable', + reason: 'LLM answer was not valid JSON', + usage, + }) + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'LLM answer was not valid JSON', + repoUrl: null, + }) + expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + }) + + it('throws a retryable error when the GitHub existence check fails', async () => { + answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.9 }]) + vi.mocked(githubApiGet).mockRejectedValue(new Error('HTTP 502')) + const error = await resolveRepoViaLlm(input).catch((e: unknown) => e) + expect(error).toBeInstanceOf(Error) + expect(error).not.toMatchObject({ nonRetryable: true }) + }) + + it('fails without retry when Bedrock credentials are missing', async () => { + vi.mocked(getBedrockCredentials).mockReturnValue(null) + const error = await resolveRepoViaLlm(input).catch((e: unknown) => e) + expect(error).toMatchObject({ + message: 'Bedrock credentials are not configured', + nonRetryable: true, + }) + expect(inferRepositoryCandidates).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts b/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts new file mode 100644 index 0000000000..920800e15c --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/canonicalGithubUrl.ts @@ -0,0 +1,35 @@ +const GITHUB_HOSTS = new Set(['github.com', 'www.github.com']) +const SEGMENT = /^[a-z0-9._-]+$/i + +export type GithubUrlParse = + | { kind: 'github'; url: string; owner: string; name: string } + | { kind: 'other'; host: string } + | { kind: 'invalid' } + +function parseUrl(raw: string): URL | null { + const trimmed = raw.trim() + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}` + try { + return new URL(withScheme) + } catch { + return null + } +} + +export function canonicalGithubUrl(raw: string): GithubUrlParse { + const url = parseUrl(raw) + if (!url) return { kind: 'invalid' } + const host = url.hostname.toLowerCase() + if (!GITHUB_HOSTS.has(host)) return { kind: 'other', host } + const [owner, repo] = url.pathname.split('/').filter(Boolean) + const name = repo?.replace(/\.git$/i, '') + if (!owner || !name || !SEGMENT.test(owner) || !SEGMENT.test(name)) return { kind: 'invalid' } + const lowerOwner = owner.toLowerCase() + const lowerName = name.toLowerCase() + return { + kind: 'github', + url: `https://github.com/${lowerOwner}/${lowerName}`, + owner: lowerOwner, + name: lowerName, + } +} diff --git a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts new file mode 100644 index 0000000000..e34232176a --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts @@ -0,0 +1,105 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { + recordMemberContributorJobInferredRepo, + recordMemberContributorJobLlmUsage, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' +import { LLM_MODEL_REGION_MAP } from '@crowd/types' + +import { getBedrockCredentials } from '../../config' +import { getPackagesDb } from '../../db' +import { invokeBedrockModel } from '../../llm/bedrockClient' +import { githubApiGet } from '../../security-contacts/githubToken' +import { canonicalGithubUrl } from './canonicalGithubUrl' +import { + REPO_INFERENCE_MODEL_ID, + RepoCandidate, + inferRepositoryCandidates, +} from './llmRepoInference' + +export const MIN_REPO_CONFIDENCE = 0.7 +const LLM_TIMEOUT_MS = 60_000 +const GITHUB_TIMEOUT_MS = 15_000 + +export interface ResolveRepoViaLlmInput { + jobId: string + purl: string +} + +export type ResolveRepoViaLlmResult = + | { outcome: 'resolved'; repoUrl: string; confidence: number } + | { outcome: 'unresolvable'; reason: string; repoUrl: string | null } + +type Unresolvable = Extract + +function unresolvable(reason: string, repoUrl: string | null = null): Unresolvable { + return { outcome: 'unresolvable', reason, repoUrl } +} + +async function repoExistsOnGithub(owner: string, name: string): Promise { + const result = await githubApiGet(`/repos/${owner}/${name}`, GITHUB_TIMEOUT_MS) + return result.status === 200 +} + +async function pickExistingCandidate( + candidates: RepoCandidate[], +): Promise { + const confident = candidates.filter((candidate) => candidate.confidence >= MIN_REPO_CONFIDENCE) + if (confident.length === 0) { + const top = candidates[0] + return unresolvable(`LLM confidence ${top.confidence} below threshold ${MIN_REPO_CONFIDENCE}`) + } + let last: Unresolvable = unresolvable('LLM returned no repository') + for (const candidate of confident) { + const parsed = canonicalGithubUrl(candidate.repositoryUrl) + if (parsed.kind === 'other') { + last = unresolvable(`Repository host not supported: ${parsed.host}`) + continue + } + if (parsed.kind === 'invalid') { + last = unresolvable(`Repository URL not recognised: ${candidate.repositoryUrl}`) + continue + } + if (await repoExistsOnGithub(parsed.owner, parsed.name)) { + return { outcome: 'resolved', repoUrl: parsed.url, confidence: candidate.confidence } + } + last = unresolvable(`Repository not found on GitHub: ${parsed.url}`, parsed.url) + } + return last +} + +export async function resolveRepoViaLlm( + input: ResolveRepoViaLlmInput, +): Promise { + const credentials = getBedrockCredentials() + if (!credentials) { + throw ApplicationFailure.nonRetryable('Bedrock credentials are not configured') + } + const invoke = invokeBedrockModel( + LLM_MODEL_REGION_MAP[REPO_INFERENCE_MODEL_ID], + credentials, + LLM_TIMEOUT_MS, + ) + const inference = await inferRepositoryCandidates(input.purl, invoke) + const pkgsQx = await getPackagesDb() + + if (inference.status === 'unparseable') { + await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) + return unresolvable(inference.reason) + } + if (inference.candidates.length === 0) { + await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) + return unresolvable('LLM returned no repository') + } + const result = await pickExistingCandidate(inference.candidates) + if (result.outcome === 'resolved') { + await recordMemberContributorJobInferredRepo(pkgsQx, input.jobId, { + repoUrl: result.repoUrl, + confidence: result.confidence, + ...inference.usage, + }) + } else { + await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) + } + return result +} From 43e7a7b5f803383f4611135498dc19d0f8e892b2 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:53:04 +0000 Subject: [PATCH 10/23] feat: fetch github contributors for a repository without storing them Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../github/mapContributors.ts | 16 ++-- .../fetchGithubRepoContributors.test.ts | 93 +++++++++++++++++++ .../job/fetchGithubRepoContributors.ts | 44 +++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts diff --git a/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts b/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts index 7e142dec88..43123e4622 100644 --- a/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts +++ b/services/apps/packages_worker/src/member-contributors/github/mapContributors.ts @@ -19,17 +19,21 @@ function contributorLogin(contributor: GithubContributor): string | null { return login.toLowerCase() } -export function toGithubContributorRows( - contributors: GithubContributor[], - repoId: string, - observedAt: Date, -): RepoContributorRow[] { +export function toGithubContributorLogins(contributors: GithubContributor[]): string[] { const logins = new Set() for (const contributor of contributors) { const login = contributorLogin(contributor) if (login !== null) logins.add(login) } - return [...logins].map((login) => ({ + return [...logins] +} + +export function toGithubContributorRows( + contributors: GithubContributor[], + repoId: string, + observedAt: Date, +): RepoContributorRow[] { + return toGithubContributorLogins(contributors).map((login) => ({ repoId, source: GITHUB_SOURCE, role: GIT_ACTIVITY_ROLE, diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts new file mode 100644 index 0000000000..d27ce36e62 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts @@ -0,0 +1,93 @@ +import { heartbeat } from '@temporalio/activity' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { githubApiGet } from '../../../security-contacts/githubToken' +import { fetchGithubRepoContributors } from '../fetchGithubRepoContributors' + +vi.mock('@temporalio/activity', async (importOriginal) => ({ + ...(await importOriginal()), + heartbeat: vi.fn(), +})) +vi.mock('../../../config', () => ({ getGithubAppConfig: vi.fn(() => ({ appId: '1' })) })) +vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) + +function contributorsJson(entries: Array<{ login: string; type?: string }>): string { + return JSON.stringify(entries.map((e) => ({ type: 'User', ...e }))) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('fetchGithubRepoContributors', () => { + it('returns the lowercased human logins without writing anything', async () => { + vi.mocked(githubApiGet).mockResolvedValue({ + status: 200, + text: contributorsJson([ + { login: 'Octo' }, + { login: 'dependabot[bot]', type: 'Bot' }, + { login: 'ada' }, + ]), + }) + await expect( + fetchGithubRepoContributors({ repoUrl: 'https://github.com/haskell/aeson' }), + ).resolves.toEqual({ + status: 'fetched', + logins: ['octo', 'ada'], + }) + expect(githubApiGet).toHaveBeenCalledWith( + expect.stringContaining('/repos/haskell/aeson/contributors'), + 15000, + { extraOkStatuses: [204] }, + ) + }) + + it('returns empty on 204 and when every contributor is a bot', async () => { + vi.mocked(githubApiGet).mockResolvedValue({ status: 204, text: null }) + await expect( + fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }), + ).resolves.toEqual({ status: 'empty', logins: [] }) + vi.mocked(githubApiGet).mockResolvedValue({ + status: 200, + text: contributorsJson([{ login: 'x[bot]', type: 'Bot' }]), + }) + await expect( + fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }), + ).resolves.toEqual({ status: 'empty', logins: [] }) + }) + + it('fails without retry when the repo is missing', async () => { + vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) + const error = await fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }).catch( + (e: unknown) => e, + ) + expect(error).toMatchObject({ message: 'Repository not found on GitHub', nonRetryable: true }) + }) + + it('throws a retryable error when the fetch errors', async () => { + vi.mocked(githubApiGet).mockRejectedValue(new Error('socket hang up')) + const error = await fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }).catch( + (e: unknown) => e, + ) + expect((error as Error).message).toContain('GitHub contributors fetch failed') + expect(error).not.toMatchObject({ nonRetryable: true }) + }) + + it('heartbeats while the fetch is in flight', async () => { + vi.useFakeTimers() + vi.mocked(githubApiGet).mockImplementation( + () => + new Promise((resolve) => setTimeout(() => resolve({ status: 204, text: null }), 45_000)), + ) + const pending = fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }) + await vi.advanceTimersByTimeAsync(30_000) + expect(heartbeat).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(15_000) + await expect(pending).resolves.toEqual({ status: 'empty', logins: [] }) + expect(vi.getTimerCount()).toBe(0) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts b/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts new file mode 100644 index 0000000000..4ffbc91e6b --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/fetchGithubRepoContributors.ts @@ -0,0 +1,44 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { getGithubAppConfig } from '../../config' +import { githubApiGet } from '../../security-contacts/githubToken' +import { RepoContributorsFetch, fetchRepoContributors } from '../github/fetchRepoContributors' +import { toGithubContributorLogins } from '../github/mapContributors' +import { HEARTBEAT_INTERVAL_MS, heartbeatSafely } from '../github/syncGithubContributorsBatch' + +const FETCH_TIMEOUT_MS = 15_000 + +export interface FetchGithubRepoContributorsInput { + repoUrl: string +} + +export interface GithubRepoContributorsFetch { + status: 'fetched' | 'empty' + logins: string[] +} + +async function fetchWithHeartbeat(repoUrl: string): Promise { + const heartbeatTimer = setInterval(() => heartbeatSafely(), HEARTBEAT_INTERVAL_MS) + try { + return await fetchRepoContributors(repoUrl, (path) => + githubApiGet(path, FETCH_TIMEOUT_MS, { extraOkStatuses: [204] }), + ) + } finally { + clearInterval(heartbeatTimer) + } +} + +export async function fetchGithubRepoContributors( + input: FetchGithubRepoContributorsInput, +): Promise { + getGithubAppConfig() + const fetched = await fetchWithHeartbeat(input.repoUrl) + if (fetched.status === 'missing') { + throw ApplicationFailure.nonRetryable('Repository not found on GitHub') + } + if (fetched.status === 'error') { + throw new Error(`GitHub contributors fetch failed for ${input.repoUrl}`) + } + const logins = toGithubContributorLogins(fetched.contributors) + return { status: logins.length === 0 ? 'empty' : 'fetched', logins } +} From 804a4f6d1c5ce0c583d80215b6b6a66967c154de Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:55:30 +0000 Subject: [PATCH 11/23] feat: build the inferred-repo member contributors body from identity affiliations Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../__tests__/buildInferredRepoBody.test.ts | 88 +++++++++++++++++++ .../job/__tests__/startInferredRepo.test.ts | 36 ++++++++ .../job/buildInferredRepoBody.ts | 48 ++++++++++ .../src/member-contributors/job/finish.ts | 17 ++++ .../job/startInferredRepo.ts | 22 +++++ 5 files changed, 211 insertions(+) create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts create mode 100644 services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts new file mode 100644 index 0000000000..36bef6127e --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import * as affiliationsDal from '@crowd/data-access-layer/src/osspckgs/identityAffiliations' + +import { buildInferredRepoMemberContributors, purlPackageName } from '../buildInferredRepoBody' + +const qx = vi.hoisted(() => ({})) +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('@crowd/data-access-layer/src/osspckgs/identityAffiliations', async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@crowd/data-access-layer/src/osspckgs/identityAffiliations') + >()), + findAffiliationsForIdentityKeys: vi.fn(), +})) + +const find = vi.mocked(affiliationsDal.findAffiliationsForIdentityKeys) + +beforeEach(() => vi.clearAllMocks()) + +describe('purlPackageName', () => { + it.each([ + ['pkg:hackage/aeson', 'aeson'], + ['pkg:conan/boost', 'boost'], + ['pkg:hackage/aeson@2.2.3.0?arch=x#sub', 'aeson'], + ['pkg:npm/%40babel/core', '@babel/core'], + ['pkg:huggingface/distilbert-base-uncased', 'distilbert-base-uncased'], + ])('%s → %s', (purl, name) => { + expect(purlPackageName(purl)).toBe(name) + }) +}) + +describe('buildInferredRepoMemberContributors', () => { + const input = { + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + logins: ['octo', 'nobody'], + } + + it('builds the body from affiliated logins with llm provenance', async () => { + find.mockResolvedValue([ + { + person: 'cdp-1', + source: 'github', + role: null, + roleKind: 'contributor', + identityType: 'github-login', + identityValue: 'octo', + endedAt: null, + lastSeenAt: null, + registryUsername: null, + registryDisplayName: null, + akritesMember: 'Google', + cdpOrganizationId: 'org-google', + confidence: 'cdp', + confidenceScore: '1.0', + }, + ]) + + const body = await buildInferredRepoMemberContributors(input) + + expect(find).toHaveBeenCalledWith(qx, [ + { identityType: 'github-login', identityValue: 'octo' }, + { identityType: 'github-login', identityValue: 'nobody' }, + ]) + expect(body).toMatchObject({ + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + maintainers: { registry: [], governanceFiles: [] }, + }) + expect(body.contributors).toHaveLength(1) + expect(body.contributors[0]).toMatchObject({ + githubLogin: 'octo', + sources: ['github'], + organization: { name: 'Google', confidence: 'cdp', confidenceScore: 1 }, + }) + }) + + it('returns an empty contributors list without a query when there are no logins', async () => { + const body = await buildInferredRepoMemberContributors({ ...input, logins: [] }) + expect(find).not.toHaveBeenCalled() + expect(body.contributors).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts new file mode 100644 index 0000000000..466faa8cac --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts @@ -0,0 +1,36 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import * as jobsDal from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { startInferredRepoMemberContributorsJob } from '../startInferredRepo' + +const qx = vi.hoisted(() => ({})) +vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) +vi.mock('@crowd/data-access-layer/src/osspckgs/memberContributorJobs', () => ({ + markMemberContributorJobRunning: vi.fn(), + findMemberContributorJobPurl: vi.fn(), +})) + +beforeEach(() => vi.clearAllMocks()) + +describe('startInferredRepoMemberContributorsJob', () => { + it('marks running then returns purl and ecosystem', async () => { + vi.mocked(jobsDal.findMemberContributorJobPurl).mockResolvedValue({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + await expect(startInferredRepoMemberContributorsJob({ jobId: 'job-1' })).resolves.toEqual({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + expect(jobsDal.markMemberContributorJobRunning).toHaveBeenCalledWith(qx, 'job-1') + }) + + it('fails without retry when the job row is missing', async () => { + vi.mocked(jobsDal.findMemberContributorJobPurl).mockResolvedValue(null) + const error = await startInferredRepoMemberContributorsJob({ jobId: 'job-1' }).catch( + (e: unknown) => e, + ) + expect(error).toMatchObject({ message: 'Job not found', nonRetryable: true }) + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts b/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts new file mode 100644 index 0000000000..e76c0f1cb9 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/buildInferredRepoBody.ts @@ -0,0 +1,48 @@ +import { + IdentityKey, + findAffiliationsForIdentityKeys, +} from '@crowd/data-access-layer/src/osspckgs/identityAffiliations' +import { + InferredRepoMemberContributorsBody, + groupAffiliatedMemberIdentities, + toInferredRepoMemberContributorsBody, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorsBody' + +import { getPackagesDb } from '../../db' + +export interface BuildInferredRepoBodyInput { + purl: string + ecosystem: string + repoUrl: string + logins: string[] +} + +export function purlPackageName(purl: string): string { + const afterScheme = purl.slice('pkg:'.length) + const afterType = afterScheme.slice(afterScheme.indexOf('/') + 1) + const withoutQualifiers = afterType.split(/[?#]/)[0] + const withoutVersion = withoutQualifiers.replace(/@[^/@]+$/, '') + return decodeURIComponent(withoutVersion) +} + +function toLoginKeys(logins: string[]): IdentityKey[] { + return logins.map((identityValue) => ({ identityType: 'github-login', identityValue })) +} + +export async function buildInferredRepoMemberContributors( + input: BuildInferredRepoBodyInput, +): Promise { + const rows = + input.logins.length === 0 + ? [] + : await findAffiliationsForIdentityKeys(await getPackagesDb(), toLoginKeys(input.logins)) + return toInferredRepoMemberContributorsBody( + { + purl: input.purl, + name: purlPackageName(input.purl), + ecosystem: input.ecosystem, + repoUrl: input.repoUrl, + }, + groupAffiliatedMemberIdentities(rows), + ) +} diff --git a/services/apps/packages_worker/src/member-contributors/job/finish.ts b/services/apps/packages_worker/src/member-contributors/job/finish.ts index 1563832313..a7c62d27bd 100644 --- a/services/apps/packages_worker/src/member-contributors/job/finish.ts +++ b/services/apps/packages_worker/src/member-contributors/job/finish.ts @@ -1,6 +1,7 @@ import { completeMemberContributorJob, failMemberContributorJob, + markMemberContributorJobUnresolvable, } from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' import { getPackagesDb } from '../../db' @@ -21,3 +22,19 @@ export async function failMemberContributorsJob( ): Promise { await failMemberContributorJob(await getPackagesDb(), input.jobId, input.reason) } + +export interface MarkUnresolvableInput extends MemberContributorsJobRef { + reason: string + repoUrl: string | null +} + +export async function markMemberContributorsJobUnresolvable( + input: MarkUnresolvableInput, +): Promise { + await markMemberContributorJobUnresolvable( + await getPackagesDb(), + input.jobId, + input.reason, + input.repoUrl, + ) +} diff --git a/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts b/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts new file mode 100644 index 0000000000..95e9e67083 --- /dev/null +++ b/services/apps/packages_worker/src/member-contributors/job/startInferredRepo.ts @@ -0,0 +1,22 @@ +import { ApplicationFailure } from '@temporalio/activity' + +import { + MemberContributorJobPurl, + findMemberContributorJobPurl, + markMemberContributorJobRunning, +} from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' + +import { getPackagesDb } from '../../db' +import { MemberContributorsJobRef } from './start' + +export async function startInferredRepoMemberContributorsJob( + input: MemberContributorsJobRef, +): Promise { + const pkgsQx = await getPackagesDb() + await markMemberContributorJobRunning(pkgsQx, input.jobId) + const job = await findMemberContributorJobPurl(pkgsQx, input.jobId) + if (!job) { + throw ApplicationFailure.nonRetryable('Job not found') + } + return job +} From 863d798cdea159f28a46ab04a9866f966aef133c Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 11:58:14 +0000 Subject: [PATCH 12/23] feat: collect member contributors from an llm inferred repository Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../apps/packages_worker/src/activities.ts | 5 + .../__tests__/workflows.test.ts | 105 ++++++++++++++++++ .../src/member-contributors/activities.ts | 5 + .../src/member-contributors/workflows.ts | 66 +++++++++++ .../packages_worker/src/workflows/index.ts | 1 + 5 files changed, 182 insertions(+) diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts index 0a90ccf540..9e23e1cbe1 100644 --- a/services/apps/packages_worker/src/activities.ts +++ b/services/apps/packages_worker/src/activities.ts @@ -70,13 +70,18 @@ export { blastRadiusReport, } from './blast-radius/activities' export { + buildInferredRepoMemberContributors, collectGithubRepoContributors, completeMemberContributorsJob, failMemberContributorsJob, + fetchGithubRepoContributors, + markMemberContributorsJobUnresolvable, mergeIdentityAffiliations, prepareIdentityAffiliationsRun, resolveIdentityAffiliationsChunk, resolveNewIdentityAffiliations, + resolveRepoViaLlm, + startInferredRepoMemberContributorsJob, startMemberContributorsJob, syncGitActivityContributors, syncGithubContributorsBatch, diff --git a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts index c234641413..9588a0a431 100644 --- a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts +++ b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts @@ -6,6 +6,7 @@ import type { } from '../git-activity/syncGitActivityContributors' import { collectMemberContributors, + collectMemberContributorsFromInferredRepo, resolveIdentityAffiliations, syncRepoContributorsFromGitActivity, } from '../workflows' @@ -36,6 +37,11 @@ const h = vi.hoisted(() => { resolveNewIdentityAffiliations: vi.fn(), completeMemberContributorsJob: vi.fn(), failMemberContributorsJob: vi.fn(), + startInferredRepoMemberContributorsJob: vi.fn(), + resolveRepoViaLlm: vi.fn(), + fetchGithubRepoContributors: vi.fn(), + buildInferredRepoMemberContributors: vi.fn(), + markMemberContributorsJobUnresolvable: vi.fn(), } const proxyCalls: Array<{ options: Record; activities: string[] }> = [] return { @@ -276,3 +282,102 @@ describe('activity proxy options', () => { }) }) }) + +describe('collectMemberContributorsFromInferredRepo', () => { + const body = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + + beforeEach(() => { + h.acts.startInferredRepoMemberContributorsJob.mockResolvedValue({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + }) + h.acts.resolveRepoViaLlm.mockResolvedValue({ + outcome: 'resolved', + repoUrl: 'https://github.com/haskell/aeson', + confidence: 0.95, + }) + h.acts.fetchGithubRepoContributors.mockResolvedValue({ status: 'fetched', logins: ['octo'] }) + h.acts.resolveNewIdentityAffiliations.mockResolvedValue({ + identities: 1, + inserted: 1, + counts: {}, + }) + h.acts.buildInferredRepoMemberContributors.mockResolvedValue(body) + }) + + it('runs start, resolve, fetch, affiliations, build, complete and returns the body', async () => { + await expect(collectMemberContributorsFromInferredRepo({ jobId: 'job-1' })).resolves.toEqual( + body, + ) + expect(h.acts.resolveRepoViaLlm).toHaveBeenCalledWith({ + jobId: 'job-1', + purl: 'pkg:hackage/aeson', + }) + expect(h.acts.fetchGithubRepoContributors).toHaveBeenCalledWith({ + repoUrl: 'https://github.com/haskell/aeson', + }) + expect(h.acts.resolveNewIdentityAffiliations).toHaveBeenCalledWith({ logins: ['octo'] }) + expect(h.acts.buildInferredRepoMemberContributors).toHaveBeenCalledWith({ + purl: 'pkg:hackage/aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + logins: ['octo'], + }) + expect(h.acts.completeMemberContributorsJob).toHaveBeenCalledWith({ jobId: 'job-1' }) + expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() + expect(h.acts.markMemberContributorsJobUnresolvable).not.toHaveBeenCalled() + }) + + it('marks unresolvable and returns nothing without fetching', async () => { + h.acts.resolveRepoViaLlm.mockResolvedValue({ + outcome: 'unresolvable', + reason: 'LLM returned no repository', + repoUrl: null, + }) + await expect( + collectMemberContributorsFromInferredRepo({ jobId: 'job-1' }), + ).resolves.toBeUndefined() + expect(h.acts.markMemberContributorsJobUnresolvable).toHaveBeenCalledWith({ + jobId: 'job-1', + reason: 'LLM returned no repository', + repoUrl: null, + }) + expect(h.acts.fetchGithubRepoContributors).not.toHaveBeenCalled() + expect(h.acts.completeMemberContributorsJob).not.toHaveBeenCalled() + expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() + }) + + it('fails the job with the activity cause and rethrows', async () => { + const cause = new Error('Bedrock credentials are not configured') + h.acts.resolveRepoViaLlm.mockRejectedValue(new h.ActivityFailure('Activity failed', cause)) + await expect(collectMemberContributorsFromInferredRepo({ jobId: 'job-1' })).rejects.toThrow( + 'Activity failed', + ) + expect(h.acts.failMemberContributorsJob).toHaveBeenCalledWith({ + jobId: 'job-1', + reason: 'Bedrock credentials are not configured', + }) + expect(h.nonCancellable).toHaveBeenCalled() + }) + + it('proxies resolveRepoViaLlm with at most two attempts and the unresolvable finisher with unbounded retry', () => { + const resolveProxy = h.proxyCalls.find((c) => c.activities.includes('resolveRepoViaLlm')) + expect(resolveProxy?.options).toMatchObject({ + startToCloseTimeout: '3 minutes', + retry: { maximumAttempts: 2 }, + }) + const finishProxy = h.proxyCalls.find((c) => + c.activities.includes('markMemberContributorsJobUnresolvable'), + ) + expect(finishProxy?.options.retry).not.toHaveProperty('maximumAttempts') + }) +}) diff --git a/services/apps/packages_worker/src/member-contributors/activities.ts b/services/apps/packages_worker/src/member-contributors/activities.ts index 80dfb86b9d..0033778a05 100644 --- a/services/apps/packages_worker/src/member-contributors/activities.ts +++ b/services/apps/packages_worker/src/member-contributors/activities.ts @@ -7,3 +7,8 @@ export { startMemberContributorsJob } from './job/start' export { collectGithubRepoContributors } from './job/collectGithubRepoContributors' export { resolveNewIdentityAffiliations } from './job/resolveNewIdentityAffiliations' export { completeMemberContributorsJob, failMemberContributorsJob } from './job/finish' +export { startInferredRepoMemberContributorsJob } from './job/startInferredRepo' +export { resolveRepoViaLlm } from './job/resolveRepoViaLlm' +export { fetchGithubRepoContributors } from './job/fetchGithubRepoContributors' +export { buildInferredRepoMemberContributors } from './job/buildInferredRepoBody' +export { markMemberContributorsJobUnresolvable } from './job/finish' diff --git a/services/apps/packages_worker/src/member-contributors/workflows.ts b/services/apps/packages_worker/src/member-contributors/workflows.ts index c3ed23bbc2..1d0c77ecb2 100644 --- a/services/apps/packages_worker/src/member-contributors/workflows.ts +++ b/services/apps/packages_worker/src/member-contributors/workflows.ts @@ -9,6 +9,8 @@ import { workflowInfo, } from '@temporalio/workflow' +import type { InferredRepoMemberContributorsBody } from '@crowd/data-access-layer/src/osspckgs/memberContributorsBody' + import type * as memberContributorActivities from './activities' import { addChunkCounts, emptyChunkCounts } from './affiliations/counts' import type { IdentityAffiliationsRunCounts, IdentityAffiliationsState } from './affiliations/state' @@ -91,6 +93,38 @@ const { completeMemberContributorsJob, failMemberContributorsJob } = proxyActivi retry: UNBOUNDED_RETRY, }) +const { startInferredRepoMemberContributorsJob } = proxyActivities< + typeof memberContributorActivities +>({ + startToCloseTimeout: '5 minutes', + retry: RETRY, +}) + +const { resolveRepoViaLlm } = proxyActivities({ + startToCloseTimeout: '3 minutes', + retry: { ...RETRY, maximumAttempts: 2 }, +}) + +const { fetchGithubRepoContributors } = proxyActivities({ + startToCloseTimeout: '1 hour', + heartbeatTimeout: '5 minutes', + retry: RETRY, +}) + +const { buildInferredRepoMemberContributors } = proxyActivities( + { + startToCloseTimeout: '5 minutes', + retry: RETRY, + }, +) + +const { markMemberContributorsJobUnresolvable } = proxyActivities< + typeof memberContributorActivities +>({ + startToCloseTimeout: '1 minute', + retry: UNBOUNDED_RETRY, +}) + const GITHUB_BATCH_SIZE = 500 const GITHUB_BATCHES_PER_RUN = 20 @@ -202,3 +236,35 @@ export async function collectMemberContributors(input: MemberContributorsJobRef) throw err } } + +export async function collectMemberContributorsFromInferredRepo( + input: MemberContributorsJobRef, +): Promise { + try { + const { purl, ecosystem } = await startInferredRepoMemberContributorsJob(input) + const resolved = await resolveRepoViaLlm({ jobId: input.jobId, purl }) + if (resolved.outcome === 'unresolvable') { + await markMemberContributorsJobUnresolvable({ + jobId: input.jobId, + reason: resolved.reason, + repoUrl: resolved.repoUrl, + }) + return undefined + } + const collected = await fetchGithubRepoContributors({ repoUrl: resolved.repoUrl }) + await resolveNewIdentityAffiliations({ logins: collected.logins }) + const body = await buildInferredRepoMemberContributors({ + purl, + ecosystem, + repoUrl: resolved.repoUrl, + logins: collected.logins, + }) + await completeMemberContributorsJob(input) + return body + } catch (err) { + await CancellationScope.nonCancellable(() => + failMemberContributorsJob({ jobId: input.jobId, reason: failureReason(err) }), + ) + throw err + } +} diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts index 231641da55..38ff6d16e4 100644 --- a/services/apps/packages_worker/src/workflows/index.ts +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -41,6 +41,7 @@ export { } from '../security-contacts/workflows' export { collectMemberContributors, + collectMemberContributorsFromInferredRepo, resolveIdentityAffiliations, syncGovernanceFileContributors, syncRepoContributorsFromGitActivity, From a07a5960ff7fc84893e5cd91899752e0093cc2d8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:00:29 +0000 Subject: [PATCH 13/23] style: format member contributor jobs data access layer Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../src/osspckgs/memberContributorJobs.test.ts | 6 +++++- .../src/osspckgs/memberContributorJobs.ts | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts index 6162771e31..a0bfc57579 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts @@ -167,7 +167,11 @@ describe('findMemberContributorJobPurl', () => { describe('findLatestMemberContributorJobByPurl', () => { it('returns id, status and collection path of the newest job', async () => { - const { qx, calls } = fakeQx({ id: 'job-1', status: 'done', collectionPath: 'llm_repo_resolution' }) + const { qx, calls } = fakeQx({ + id: 'job-1', + status: 'done', + collectionPath: 'llm_repo_resolution', + }) await expect(findLatestMemberContributorJobByPurl(qx, 'pkg:hackage/aeson')).resolves.toEqual({ id: 'job-1', status: 'done', diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts index 4d8a26e88d..f723814643 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts @@ -179,9 +179,12 @@ export async function findMemberContributorJobPurl( qx: QueryExecutor, id: string, ): Promise { - return qx.selectOneOrNone(`SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)`, { - id, - }) + return qx.selectOneOrNone( + `SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)`, + { + id, + }, + ) } export async function findLatestMemberContributorJobByPurl( From 44467b465c1f1f69e8dde6c831457613b9674a9f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:09:21 +0000 Subject: [PATCH 14/23] fix: reject malformed purls before llm repo resolution Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../memberContributorCollectionPaths.test.ts | 15 +++++++++++++++ .../packages/memberContributorCollectionPaths.ts | 6 ++++++ .../packages/submitMemberContributorsJob.test.ts | 14 ++++++++++++++ .../v1/packages/submitMemberContributorsJob.ts | 6 +++++- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index 57ae8419bc..097dbbce93 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest' import { MEMBER_CONTRIBUTOR_COLLECTION_PATHS, collectionPathForPurlType, + isWellFormedPurl, purlType, } from './memberContributorCollectionPaths' @@ -34,3 +35,17 @@ describe('collectionPathForPurlType', () => { expect(collectionPathForPurlType('hasOwnProperty')).toBe('llm_repo_resolution') }) }) + +describe('isWellFormedPurl', () => { + it('accepts a purl with a type and a name segment', () => { + expect(isWellFormedPurl('pkg:hackage/aeson')).toBe(true) + expect(isWellFormedPurl('pkg:conan/openssl.org/openssl')).toBe(true) + }) + + it('rejects a purl without a type or a name segment', () => { + expect(isWellFormedPurl('pkg:')).toBe(false) + expect(isWellFormedPurl('pkg:hackage')).toBe(false) + expect(isWellFormedPurl('pkg:hackage/')).toBe(false) + expect(isWellFormedPurl('pkg:/x')).toBe(false) + }) +}) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts index 3d5b0d8afa..a157e1936a 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts @@ -19,6 +19,12 @@ export function purlType(purl: string): string { return type.toLowerCase() } +const WELL_FORMED_PURL = /^pkg:[a-z][a-z0-9.+-]*\/[^/?#]+/ + +export function isWellFormedPurl(purl: string): boolean { + return WELL_FORMED_PURL.test(purl) +} + export const LLM_REPO_RESOLUTION_PATH: MemberContributorCollectionPath = 'llm_repo_resolution' export function collectionPathForPurlType(type: string): MemberContributorCollectionPath { diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index df9b502534..f3c3d1d5bf 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -391,6 +391,20 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { expect(json).toHaveBeenCalledWith({ jobId: input.id, status: 'pending' }) }) + it.each(['pkg:', 'pkg:hackage', 'pkg:/x'])( + 'rejects the malformed purl %s with 400 before any database call', + async (purl) => { + const { req, res } = mockReqRes({ purl }) + const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) + expect(error).toBeInstanceOf(BadRequestError) + expect((error as BadRequestError).message).toBe('Malformed purl') + expect(findActiveMemberContributorJobByPurl).not.toHaveBeenCalled() + expect(findLatestMemberContributorJobByPurl).not.toHaveBeenCalled() + expect(createMemberContributorJob).not.toHaveBeenCalled() + expect(start).not.toHaveBeenCalled() + }, + ) + it('returns the active job and counts the submission', async () => { findActiveMemberContributorJobByPurl.mockResolvedValue({ id: 'active-1' }) const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts index 92f3b7bcf1..e5101136c5 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts @@ -3,7 +3,7 @@ import type { Request, Response } from 'express' import { getPackagesQx } from '@/db/packagesDb' import { getPackagesTemporalClient } from '@/db/packagesTemporal' import { validateOrThrow } from '@/utils/validation' -import { ConflictError, NotFoundError, generateUUIDv4 } from '@crowd/common' +import { BadRequestError, ConflictError, NotFoundError, generateUUIDv4 } from '@crowd/common' import { MemberContributorCollectionPath, MemberContributorsPackageRow, @@ -27,6 +27,7 @@ import { import { LLM_REPO_RESOLUTION_PATH, collectionPathForPurlType, + isWellFormedPurl, purlType, } from './memberContributorCollectionPaths' import { @@ -180,6 +181,9 @@ async function submitInferredRepoJob( ecosystem: string, res: Response, ): Promise { + if (!isWellFormedPurl(purl)) { + throw new BadRequestError('Malformed purl') + } const active = await findActiveMemberContributorJobByPurl(qx, purl) if (active) { await respondWithActiveJob(qx, active.id, MEMBER_CONTRIBUTORS_INFERRED_REPO_WORKFLOW, res) From d64a6b2b9f1f8f8fb7cf2a3eeae6ad0c1ce638bf Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:10:28 +0000 Subject: [PATCH 15/23] fix: surface truncated and refused llm answers instead of marking them unresolvable Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../job/__tests__/llmRepoInference.test.ts | 74 ++++++++++++++++++- .../job/llmRepoInference.ts | 31 ++++++-- 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts index f94b1b1a9c..9cc13e888b 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts @@ -1,18 +1,27 @@ -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { inferRepositoryCandidates } from '../llmRepoInference' +const { log } = vi.hoisted(() => ({ log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })) + +vi.mock('@crowd/logging', () => ({ getServiceChildLogger: () => log })) + type ResponseBlock = { type: 'thinking'; thinking: string } | { type: 'text'; text: string } function bedrockResponse( text: string | null, - usage = { input_tokens: 300, output_tokens: 80 }, + usage: unknown = { input_tokens: 300, output_tokens: 80 }, + stopReason = 'end_turn', ): string { const content: ResponseBlock[] = [{ type: 'thinking', thinking: '' }] if (text !== null) content.push({ type: 'text', text }) - return JSON.stringify({ content, usage }) + return JSON.stringify({ content, usage, stop_reason: stopReason }) } +beforeEach(() => { + vi.clearAllMocks() +}) + describe('inferRepositoryCandidates', () => { it('sends the purl to sonnet 5.5 and parses the ranked candidates from the first text block', async () => { const invoke = vi.fn().mockResolvedValue( @@ -33,6 +42,14 @@ describe('inferRepositoryCandidates', () => { const [modelId, body] = invoke.mock.calls[0] expect(modelId).toBe('us.anthropic.claude-sonnet-5-5') const parsed = JSON.parse(body) + expect(Object.keys(parsed).sort()).toEqual([ + 'anthropic_version', + 'max_tokens', + 'messages', + 'output_config', + 'system', + ]) + expect(parsed.max_tokens).toBe(2048) expect(parsed.anthropic_version).toBe('bedrock-2023-05-31') expect(parsed.messages).toEqual([ { role: 'user', content: [{ type: 'text', text: 'pkg:hackage/aeson' }] }, @@ -111,6 +128,57 @@ describe('inferRepositoryCandidates', () => { }) }) + it('logs the stop reason of the call', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse(JSON.stringify({ candidates: [], rationale: 'unknown' }))) + await inferRepositoryCandidates('pkg:conan/x', invoke) + expect(log.info).toHaveBeenCalledWith( + expect.objectContaining({ stopReason: 'end_turn' }), + 'LLM repo inference call', + ) + }) + + it('rejects a truncated answer so the activity retries instead of marking it unresolvable', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse('{"candidates": [', undefined, 'max_tokens')) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( + 'LLM answer truncated at max_tokens', + ) + expect(log.info).toHaveBeenCalledWith( + expect.objectContaining({ stopReason: 'max_tokens', inputTokens: 300, outputTokens: 80 }), + 'LLM repo inference call', + ) + }) + + it('reports a refusal as unparseable, keeping the usage', async () => { + const invoke = vi + .fn() + .mockResolvedValue(bedrockResponse(null, { input_tokens: 20, output_tokens: 5 }, 'refusal')) + const result = await inferRepositoryCandidates('pkg:conan/x', invoke) + expect(result).toMatchObject({ + status: 'unparseable', + reason: 'LLM refused to answer', + usage: { inputTokens: 20, outputTokens: 5 }, + }) + }) + + it.each([ + ['missing', undefined], + ['non-numeric', { input_tokens: '300', output_tokens: 80 }], + ['partial', { input_tokens: 300 }], + ])('rejects a response whose usage envelope is %s', async (_label, usage) => { + const invoke = vi + .fn() + .mockResolvedValue( + JSON.stringify({ content: [{ type: 'text', text: '{}' }], usage, stop_reason: 'end_turn' }), + ) + await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( + 'Bedrock response had no token usage', + ) + }) + it('propagates transport errors', async () => { const invoke = vi.fn().mockRejectedValue(new Error('ThrottlingException')) await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( diff --git a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts index 95506e6fa3..f04a713e4d 100644 --- a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts +++ b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts @@ -11,7 +11,8 @@ const log = getServiceChildLogger('member-contributors:llm-repo-inference') export const REPO_INFERENCE_MODEL_ID = LlmModelType.CLAUDE_SONNET_5_5 export const MAX_REPO_CANDIDATES = 3 -const MAX_TOKENS = 512 +const MAX_TOKENS = 2048 +const TRUNCATED_ANSWER = 'LLM answer truncated at max_tokens' const candidateSchema = z.object({ repositoryUrl: z.string().min(1), @@ -65,7 +66,8 @@ export type RepoInference = interface BedrockMessageResponse { content?: Array<{ type?: string; text?: string }> - usage?: { input_tokens?: number; output_tokens?: number } + usage?: { input_tokens?: unknown; output_tokens?: unknown } + stop_reason?: string } function requestBody(purl: string): string { @@ -79,8 +81,11 @@ function requestBody(purl: string): string { } function usageOf(response: BedrockMessageResponse): LlmUsage { - const inputTokens = response.usage?.input_tokens ?? 0 - const outputTokens = response.usage?.output_tokens ?? 0 + const inputTokens = response.usage?.input_tokens + const outputTokens = response.usage?.output_tokens + if (typeof inputTokens !== 'number' || typeof outputTokens !== 'number') { + throw new Error('Bedrock response had no token usage') + } return { inputTokens, outputTokens, @@ -95,6 +100,17 @@ function firstText(response: BedrockMessageResponse): string | null { return block?.text ?? null } +function candidatesOf(response: BedrockMessageResponse): RepoCandidate[] | string { + if (response.stop_reason === 'max_tokens') { + return TRUNCATED_ANSWER + } + if (response.stop_reason === 'refusal') { + return 'LLM refused to answer' + } + const text = firstText(response) + return text === null ? 'LLM answer had no text content' : parseAnswer(text) +} + function parseAnswer(text: string): RepoCandidate[] | string { let raw: unknown try { @@ -118,12 +134,12 @@ export async function inferRepositoryCandidates( await invoke(REPO_INFERENCE_MODEL_ID, requestBody(purl)), ) as BedrockMessageResponse const usage = usageOf(response) - const text = firstText(response) - const candidates = text === null ? 'LLM answer had no text content' : parseAnswer(text) + const candidates = candidatesOf(response) log.info( { modelId: REPO_INFERENCE_MODEL_ID, purl, + stopReason: response.stop_reason, inputTokens: usage.inputTokens, outputTokens: usage.outputTokens, costUsd: usage.costUsd, @@ -132,6 +148,9 @@ export async function inferRepositoryCandidates( }, 'LLM repo inference call', ) + if (candidates === TRUNCATED_ANSWER) { + throw new Error(TRUNCATED_ANSWER) + } if (typeof candidates === 'string') { return { status: 'unparseable', reason: candidates, usage } } From f86c44ed36de1de574e1e5c7943f9cf734e1861b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:12:00 +0000 Subject: [PATCH 16/23] fix: record llm usage per call and the attempted repository on the job row Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../job/__tests__/resolveRepoViaLlm.test.ts | 63 ++++++++++++++++--- .../job/resolveRepoViaLlm.ts | 20 +++--- .../osspckgs/memberContributorJobs.test.ts | 23 ++++--- .../src/osspckgs/memberContributorJobs.ts | 7 ++- 4 files changed, 85 insertions(+), 28 deletions(-) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts index 4104567c1c..dd43be0b0c 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import * as jobsDal from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' -import { getBedrockCredentials } from '../../../config' +import { getBedrockCredentials, getGithubAppConfig } from '../../../config' import { githubApiGet } from '../../../security-contacts/githubToken' import { inferRepositoryCandidates } from '../llmRepoInference' import { resolveRepoViaLlm } from '../resolveRepoViaLlm' @@ -10,7 +10,10 @@ import { resolveRepoViaLlm } from '../resolveRepoViaLlm' const qx = vi.hoisted(() => ({})) vi.mock('../../../db', () => ({ getPackagesDb: vi.fn().mockResolvedValue(qx) })) -vi.mock('../../../config', () => ({ getBedrockCredentials: vi.fn() })) +vi.mock('../../../config', () => ({ + getBedrockCredentials: vi.fn(), + getGithubAppConfig: vi.fn(), +})) vi.mock('../../../llm/bedrockClient', () => ({ invokeBedrockModel: vi.fn(() => vi.fn()) })) vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) vi.mock('../llmRepoInference', async (importOriginal) => ({ @@ -27,6 +30,15 @@ const input = { jobId: 'job-1', purl: 'pkg:hackage/aeson' } const recordRepo = vi.mocked(jobsDal.recordMemberContributorJobInferredRepo) const recordUsage = vi.mocked(jobsDal.recordMemberContributorJobLlmUsage) +function expectUsageRecordedOnceBeforeGithub() { + expect(recordUsage).toHaveBeenCalledTimes(1) + expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + const githubCalls = vi.mocked(githubApiGet).mock.invocationCallOrder + if (githubCalls.length > 0) { + expect(recordUsage.mock.invocationCallOrder[0]).toBeLessThan(githubCalls[0]) + } +} + function answered(candidates: Array<{ repositoryUrl: string; confidence: number }>) { vi.mocked(inferRepositoryCandidates).mockResolvedValue({ status: 'answered', candidates, usage }) } @@ -34,6 +46,9 @@ function answered(candidates: Array<{ repositoryUrl: string; confidence: number beforeEach(() => { vi.clearAllMocks() vi.mocked(getBedrockCredentials).mockReturnValue({ accessKeyId: 'k', secretAccessKey: 's' }) + vi.mocked(getGithubAppConfig).mockReturnValue( + {} as unknown as ReturnType, + ) vi.mocked(githubApiGet).mockResolvedValue({ status: 200, text: '{}' }) }) @@ -50,9 +65,8 @@ describe('resolveRepoViaLlm', () => { expect(recordRepo).toHaveBeenCalledWith(qx, 'job-1', { repoUrl: 'https://github.com/haskell/aeson', confidence: 0.95, - ...usage, }) - expect(recordUsage).not.toHaveBeenCalled() + expectUsageRecordedOnceBeforeGithub() }) it('skips absent candidates and takes the next existing one', async () => { @@ -82,7 +96,7 @@ describe('resolveRepoViaLlm', () => { reason: 'Repository not found on GitHub: https://github.com/a/two', repoUrl: 'https://github.com/a/two', }) - expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + expectUsageRecordedOnceBeforeGithub() expect(recordRepo).not.toHaveBeenCalled() }) @@ -95,6 +109,18 @@ describe('resolveRepoViaLlm', () => { repoUrl: null, }) expect(githubApiGet).not.toHaveBeenCalled() + expectUsageRecordedOnceBeforeGithub() + }) + + it('quotes the highest candidate confidence when all are below the threshold', async () => { + answered([ + { repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }, + { repositoryUrl: 'https://github.com/a/c', confidence: 0.6 }, + ]) + + await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ + reason: 'LLM confidence 0.6 below threshold 0.7', + }) }) it('ignores lower-ranked candidates below the threshold', async () => { @@ -113,7 +139,17 @@ describe('resolveRepoViaLlm', () => { await expect(resolveRepoViaLlm(input)).resolves.toEqual({ outcome: 'unresolvable', reason: 'Repository host not supported: gitlab.com', - repoUrl: null, + repoUrl: 'https://gitlab.com/a/b', + }) + expectUsageRecordedOnceBeforeGithub() + }) + + it('is unresolvable with the attempted url when it is not recognised', async () => { + answered([{ repositoryUrl: 'not a url', confidence: 0.9 }]) + await expect(resolveRepoViaLlm(input)).resolves.toEqual({ + outcome: 'unresolvable', + reason: 'Repository URL not recognised: not a url', + repoUrl: 'not a url', }) }) @@ -124,6 +160,7 @@ describe('resolveRepoViaLlm', () => { reason: 'LLM returned no repository', repoUrl: null, }) + expectUsageRecordedOnceBeforeGithub() }) it('is unresolvable for an unparseable answer and still records usage', async () => { @@ -137,7 +174,7 @@ describe('resolveRepoViaLlm', () => { reason: 'LLM answer was not valid JSON', repoUrl: null, }) - expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + expectUsageRecordedOnceBeforeGithub() }) it('throws a retryable error when the GitHub existence check fails', async () => { @@ -145,7 +182,19 @@ describe('resolveRepoViaLlm', () => { vi.mocked(githubApiGet).mockRejectedValue(new Error('HTTP 502')) const error = await resolveRepoViaLlm(input).catch((e: unknown) => e) expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toBe('HTTP 502') expect(error).not.toMatchObject({ nonRetryable: true }) + expectUsageRecordedOnceBeforeGithub() + expect(recordRepo).not.toHaveBeenCalled() + }) + + it('fails before calling the LLM when the GitHub App config is missing', async () => { + vi.mocked(getGithubAppConfig).mockImplementation(() => { + throw new Error('Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY') + }) + await expect(resolveRepoViaLlm(input)).rejects.toThrow('CROWD_GITHUB_PRIVATE_KEY') + expect(inferRepositoryCandidates).not.toHaveBeenCalled() + expect(recordUsage).not.toHaveBeenCalled() }) it('fails without retry when Bedrock credentials are missing', async () => { diff --git a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts index e34232176a..a3e04b4438 100644 --- a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts +++ b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts @@ -6,7 +6,7 @@ import { } from '@crowd/data-access-layer/src/osspckgs/memberContributorJobs' import { LLM_MODEL_REGION_MAP } from '@crowd/types' -import { getBedrockCredentials } from '../../config' +import { getBedrockCredentials, getGithubAppConfig } from '../../config' import { getPackagesDb } from '../../db' import { invokeBedrockModel } from '../../llm/bedrockClient' import { githubApiGet } from '../../security-contacts/githubToken' @@ -46,18 +46,21 @@ async function pickExistingCandidate( ): Promise { const confident = candidates.filter((candidate) => candidate.confidence >= MIN_REPO_CONFIDENCE) if (confident.length === 0) { - const top = candidates[0] - return unresolvable(`LLM confidence ${top.confidence} below threshold ${MIN_REPO_CONFIDENCE}`) + const highest = Math.max(...candidates.map((candidate) => candidate.confidence)) + return unresolvable(`LLM confidence ${highest} below threshold ${MIN_REPO_CONFIDENCE}`) } let last: Unresolvable = unresolvable('LLM returned no repository') for (const candidate of confident) { const parsed = canonicalGithubUrl(candidate.repositoryUrl) if (parsed.kind === 'other') { - last = unresolvable(`Repository host not supported: ${parsed.host}`) + last = unresolvable(`Repository host not supported: ${parsed.host}`, candidate.repositoryUrl) continue } if (parsed.kind === 'invalid') { - last = unresolvable(`Repository URL not recognised: ${candidate.repositoryUrl}`) + last = unresolvable( + `Repository URL not recognised: ${candidate.repositoryUrl}`, + candidate.repositoryUrl, + ) continue } if (await repoExistsOnGithub(parsed.owner, parsed.name)) { @@ -75,6 +78,7 @@ export async function resolveRepoViaLlm( if (!credentials) { throw ApplicationFailure.nonRetryable('Bedrock credentials are not configured') } + getGithubAppConfig() const invoke = invokeBedrockModel( LLM_MODEL_REGION_MAP[REPO_INFERENCE_MODEL_ID], credentials, @@ -82,13 +86,12 @@ export async function resolveRepoViaLlm( ) const inference = await inferRepositoryCandidates(input.purl, invoke) const pkgsQx = await getPackagesDb() + await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) if (inference.status === 'unparseable') { - await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) return unresolvable(inference.reason) } if (inference.candidates.length === 0) { - await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) return unresolvable('LLM returned no repository') } const result = await pickExistingCandidate(inference.candidates) @@ -96,10 +99,7 @@ export async function resolveRepoViaLlm( await recordMemberContributorJobInferredRepo(pkgsQx, input.jobId, { repoUrl: result.repoUrl, confidence: result.confidence, - ...inference.usage, }) - } else { - await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) } return result } diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts index a0bfc57579..48a47ad48a 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts @@ -183,31 +183,29 @@ describe('findLatestMemberContributorJobByPurl', () => { }) describe('recordMemberContributorJobInferredRepo', () => { - it('writes repo url, source, confidence and usage on a running job', async () => { + it('writes only repo url, source and confidence on a running job', async () => { const { qx, calls } = fakeQx() await recordMemberContributorJobInferredRepo(qx, 'job-1', { repoUrl: 'https://github.com/haskell/aeson', confidence: 0.92, - inputTokens: 300, - outputTokens: 80, - costUsd: 0.0014, }) expect(calls[0].method).toBe('result') expect(calls[0].query).toContain("repo_url_source = 'llm_inferred'") + expect(calls[0].query).toContain('llm_confidence = $(confidence)') + expect(calls[0].query).not.toContain('llm_input_tokens') + expect(calls[0].query).not.toContain('llm_output_tokens') + expect(calls[0].query).not.toContain('llm_cost_usd') expect(calls[0].query).toContain("WHERE id = $(id) AND status = 'running'") expect(calls[0].params).toEqual({ id: 'job-1', repoUrl: 'https://github.com/haskell/aeson', confidence: 0.92, - inputTokens: 300, - outputTokens: 80, - costUsd: 0.0014, }) }) }) describe('recordMemberContributorJobLlmUsage', () => { - it('writes only the usage columns on a running job', async () => { + it('adds the usage to the running totals of a running job', async () => { const { qx, calls } = fakeQx() await recordMemberContributorJobLlmUsage(qx, 'job-1', { inputTokens: 300, @@ -215,6 +213,15 @@ describe('recordMemberContributorJobLlmUsage', () => { costUsd: null, }) expect(calls[0].query).not.toContain('repo_url') + expect(calls[0].query).toContain( + 'llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens)', + ) + expect(calls[0].query).toContain( + 'llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens)', + ) + expect(calls[0].query).toContain( + 'llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END', + ) expect(calls[0].query).toContain("status = 'running'") expect(calls[0].params).toEqual({ id: 'job-1', diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts index f723814643..bb914bfce9 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.ts @@ -43,7 +43,7 @@ export interface LlmUsage { costUsd: number | null } -export interface InferredRepoRecord extends LlmUsage { +export interface InferredRepoRecord { repoUrl: string confidence: number } @@ -212,7 +212,6 @@ export async function recordMemberContributorJobInferredRepo( ` UPDATE member_contributor_jobs SET repo_url = $(repoUrl), repo_url_source = 'llm_inferred', llm_confidence = $(confidence), - llm_input_tokens = $(inputTokens), llm_output_tokens = $(outputTokens), llm_cost_usd = $(costUsd), updated_at = NOW() WHERE id = $(id) AND status = 'running' `, @@ -228,7 +227,9 @@ export async function recordMemberContributorJobLlmUsage( await qx.result( ` UPDATE member_contributor_jobs - SET llm_input_tokens = $(inputTokens), llm_output_tokens = $(outputTokens), llm_cost_usd = $(costUsd), + SET llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens), + llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens), + llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END, updated_at = NOW() WHERE id = $(id) AND status = 'running' `, From 2d7c4b74a954e7343aaed9b578ada4e19b407a1f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:12:33 +0000 Subject: [PATCH 17/23] docs: document the conflict jobId and merge duplicate activity exports Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../public/v1/akrites-external/openapi.yaml | 23 ++++++++++++++++++- .../src/member-contributors/activities.ts | 7 ++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index 5b2ecdb154..7a7aec93ee 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -117,6 +117,27 @@ components: message: type: string + ConflictError: + type: object + required: [error] + properties: + error: + type: object + required: [code, message] + properties: + code: + type: string + example: CONFLICT + message: + type: string + context: + type: object + properties: + jobId: + type: string + format: uuid + description: The job whose result is still readable; poll it. + Ecosystem: type: string example: npm @@ -1369,7 +1390,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Error" + $ref: "#/components/schemas/ConflictError" "429": description: Too many requests — rate-limited independently of the other endpoints. content: diff --git a/services/apps/packages_worker/src/member-contributors/activities.ts b/services/apps/packages_worker/src/member-contributors/activities.ts index 0033778a05..fa0dd8a6ce 100644 --- a/services/apps/packages_worker/src/member-contributors/activities.ts +++ b/services/apps/packages_worker/src/member-contributors/activities.ts @@ -6,9 +6,12 @@ export { syncGovernanceContributors } from './governance/syncGovernanceContribut export { startMemberContributorsJob } from './job/start' export { collectGithubRepoContributors } from './job/collectGithubRepoContributors' export { resolveNewIdentityAffiliations } from './job/resolveNewIdentityAffiliations' -export { completeMemberContributorsJob, failMemberContributorsJob } from './job/finish' +export { + completeMemberContributorsJob, + failMemberContributorsJob, + markMemberContributorsJobUnresolvable, +} from './job/finish' export { startInferredRepoMemberContributorsJob } from './job/startInferredRepo' export { resolveRepoViaLlm } from './job/resolveRepoViaLlm' export { fetchGithubRepoContributors } from './job/fetchGithubRepoContributors' export { buildInferredRepoMemberContributors } from './job/buildInferredRepoBody' -export { markMemberContributorsJobUnresolvable } from './job/finish' From 92b231a7fe5615c1428f946c2bfc9822778a03e1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:14:31 +0000 Subject: [PATCH 18/23] fix: accept uppercase purl types on llm repo resolution Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../memberContributorCollectionPaths.test.ts | 1 + .../v1/packages/memberContributorCollectionPaths.ts | 2 +- .../v1/packages/submitMemberContributorsJob.test.ts | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index 097dbbce93..8d8e1984d9 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -40,6 +40,7 @@ describe('isWellFormedPurl', () => { it('accepts a purl with a type and a name segment', () => { expect(isWellFormedPurl('pkg:hackage/aeson')).toBe(true) expect(isWellFormedPurl('pkg:conan/openssl.org/openssl')).toBe(true) + expect(isWellFormedPurl('pkg:Hackage/aeson')).toBe(true) }) it('rejects a purl without a type or a name segment', () => { diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts index a157e1936a..179e8edfdb 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts @@ -19,7 +19,7 @@ export function purlType(purl: string): string { return type.toLowerCase() } -const WELL_FORMED_PURL = /^pkg:[a-z][a-z0-9.+-]*\/[^/?#]+/ +const WELL_FORMED_PURL = /^pkg:[a-z][a-z0-9.+-]*\/[^/?#]+/i export function isWellFormedPurl(purl: string): boolean { return WELL_FORMED_PURL.test(purl) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index f3c3d1d5bf..9aa6f0d11b 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -405,6 +405,19 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { }, ) + it('accepts an uppercase purl type and records the lowercased ecosystem', async () => { + const { req, res, status } = mockReqRes({ purl: 'pkg:Hackage/aeson' }) + + await submitMemberContributorsJob(req, res) + + const [, input] = createMemberContributorJob.mock.calls[0] + expect(input).toMatchObject({ + ecosystem: 'hackage', + collectionPath: 'llm_repo_resolution', + }) + expect(status).toHaveBeenCalledWith(202) + }) + it('returns the active job and counts the submission', async () => { findActiveMemberContributorJobByPurl.mockResolvedValue({ id: 'active-1' }) const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) From e2e6f78b1ebd6be6015a8ce71da9d052df5c2239 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:32:30 +0000 Subject: [PATCH 19/23] test: consolidate member contributors llm repo resolution tests Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../packages/getMemberContributorsJob.test.ts | 135 ++++---- .../memberContributorCollectionPaths.test.ts | 32 +- .../submitMemberContributorsJob.test.ts | 154 ++++----- .../__tests__/workflows.test.ts | 118 ++++--- .../__tests__/buildInferredRepoBody.test.ts | 60 ++-- .../job/__tests__/canonicalGithubUrl.test.ts | 44 +-- .../fetchGithubRepoContributors.test.ts | 78 ++--- .../job/__tests__/llmRepoInference.test.ts | 232 ++++++------- .../job/__tests__/resolveRepoViaLlm.test.ts | 318 ++++++++++-------- .../job/__tests__/startInferredRepo.test.ts | 19 +- .../src/osspckgs/identityAffiliations.test.ts | 20 +- .../osspckgs/memberContributorJobs.test.ts | 161 ++++----- .../osspckgs/memberContributorsBody.test.ts | 10 +- 13 files changed, 723 insertions(+), 658 deletions(-) diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index 97bb50a0fd..e3300fead2 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -188,77 +188,90 @@ describe('getMemberContributorsJob', () => { }) describe('getMemberContributorsJob, llm repo resolution path', () => { - it('maps unresolvable with reason and the attempted repo url', async () => { - findMemberContributorJob.mockResolvedValue( - jobRow({ - collectionPath: 'llm_repo_resolution', + const llmJob = (overrides: Partial) => + jobRow({ collectionPath: 'llm_repo_resolution', ...overrides }) + const workflowId = `member-contributors-job/${JOB_ID}` + const inferredBody = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } + + it.each([ + { + name: 'reports unresolvable with the reason and attempted repo url, without the workflow', + arrange: () => + findMemberContributorJob.mockResolvedValue( + llmJob({ + status: 'unresolvable', + error: 'LLM confidence 0.4 below threshold 0.7', + repoUrl: null, + }), + ), + response: { + jobId: JOB_ID, status: 'unresolvable', - error: 'LLM confidence 0.4 below threshold 0.7', + reason: 'LLM confidence 0.4 below threshold 0.7', repoUrl: null, + }, + workflowIds: [], + packageReads: [], + }, + { + name: 'returns the workflow result as the done body', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockResolvedValue(inferredBody) + }, + response: { ...inferredBody, jobId: JOB_ID, status: 'done' }, + workflowIds: [[workflowId]], + packageReads: [], + }, + { + name: 'reports failed with the resubmit reason when the workflow result is gone', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) + }, + response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' }, + workflowIds: [[workflowId]], + packageReads: [], + }, + { + name: 'keeps reading github path done jobs from the tables, not the workflow', + arrange: () => { + findMemberContributorJob.mockResolvedValue(jobRow({ status: 'done' })) + findMemberContributorsPackage.mockResolvedValue(packageRow()) + findAffiliatedMemberIdentities.mockResolvedValue([identityRow()]) + }, + response: expect.objectContaining({ + jobId: JOB_ID, + status: 'done', + purl: 'pkg:npm/request-oauth', }), - ) - const { req, res, json } = mockReqRes({ jobId: JOB_ID }) - await getMemberContributorsJob(req, res) - expect(json).toHaveBeenCalledWith({ - jobId: JOB_ID, - status: 'unresolvable', - reason: 'LLM confidence 0.4 below threshold 0.7', - repoUrl: null, - }) - expect(getHandle).not.toHaveBeenCalled() - }) - - it('returns the workflow result as the done body', async () => { - const body = { - purl: 'pkg:hackage/aeson', - name: 'aeson', - ecosystem: 'hackage', - repoUrl: 'https://github.com/haskell/aeson', - repoUrlSource: 'llm_inferred', - repoMappingConfidence: null, - contributors: [], - maintainers: { registry: [], governanceFiles: [] }, - } - findMemberContributorJob.mockResolvedValue( - jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), - ) - result.mockResolvedValue(body) - const { req, res, json } = mockReqRes({ jobId: JOB_ID }) - await getMemberContributorsJob(req, res) - expect(getHandle).toHaveBeenCalledWith(`member-contributors-job/${JOB_ID}`) - expect(json).toHaveBeenCalledWith({ ...body, jobId: JOB_ID, status: 'done' }) - expect(findMemberContributorsPackage).not.toHaveBeenCalled() - }) - - it('answers failed with the resubmit reason when the workflow result is gone', async () => { - findMemberContributorJob.mockResolvedValue( - jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), - ) - result.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) + workflowIds: [], + packageReads: [[{}, 'pkg:npm/request-oauth']], + }, + ])('$name', async ({ arrange, response, workflowIds, packageReads }) => { + arrange() const { req, res, json } = mockReqRes({ jobId: JOB_ID }) await getMemberContributorsJob(req, res) - expect(json).toHaveBeenCalledWith({ - jobId: JOB_ID, - status: 'failed', - reason: 'Job result expired, resubmit the job', - }) + expect({ + responses: json.mock.calls, + workflowIds: getHandle.mock.calls, + packageReads: findMemberContributorsPackage.mock.calls, + }).toEqual({ responses: [[response]], workflowIds, packageReads }) }) it('throws an internal error when a done workflow returned no body', async () => { - findMemberContributorJob.mockResolvedValue( - jobRow({ collectionPath: 'llm_repo_resolution', status: 'done' }), - ) + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) result.mockResolvedValue(undefined) const { req, res } = mockReqRes({ jobId: JOB_ID }) await expect(getMemberContributorsJob(req, res)).rejects.toBeInstanceOf(InternalError) }) - - it('keeps reading github path done jobs from the tables', async () => { - findMemberContributorJob.mockResolvedValue(jobRow({ status: 'done' })) - findMemberContributorsPackage.mockResolvedValue(packageRow()) - findAffiliatedMemberIdentities.mockResolvedValue([identityRow()]) - const { req, res } = mockReqRes({ jobId: JOB_ID }) - await getMemberContributorsJob(req, res) - expect(getHandle).not.toHaveBeenCalled() - }) }) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index 8d8e1984d9..c91e1abdb5 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -29,24 +29,24 @@ describe('collectionPathForPurlType', () => { expect(Object.keys(MEMBER_CONTRIBUTOR_COLLECTION_PATHS)).toHaveLength(8) }) - it('routes any purl type outside the map to llm repo resolution', () => { - expect(collectionPathForPurlType('hackage')).toBe('llm_repo_resolution') - expect(collectionPathForPurlType('conan')).toBe('llm_repo_resolution') - expect(collectionPathForPurlType('hasOwnProperty')).toBe('llm_repo_resolution') - }) + it.each(['hackage', 'conan', 'hasOwnProperty'])( + 'routes the %s purl type outside the map to llm repo resolution', + (type) => { + expect(collectionPathForPurlType(type)).toBe('llm_repo_resolution') + }, + ) }) describe('isWellFormedPurl', () => { - it('accepts a purl with a type and a name segment', () => { - expect(isWellFormedPurl('pkg:hackage/aeson')).toBe(true) - expect(isWellFormedPurl('pkg:conan/openssl.org/openssl')).toBe(true) - expect(isWellFormedPurl('pkg:Hackage/aeson')).toBe(true) - }) - - it('rejects a purl without a type or a name segment', () => { - expect(isWellFormedPurl('pkg:')).toBe(false) - expect(isWellFormedPurl('pkg:hackage')).toBe(false) - expect(isWellFormedPurl('pkg:hackage/')).toBe(false) - expect(isWellFormedPurl('pkg:/x')).toBe(false) + it.each([ + ['pkg:hackage/aeson', true], + ['pkg:conan/openssl.org/openssl', true], + ['pkg:Hackage/aeson', true], + ['pkg:', false], + ['pkg:hackage', false], + ['pkg:hackage/', false], + ['pkg:/x', false], + ])('requires a type and a name segment: %s is %s', (purl, wellFormed) => { + expect(isWellFormedPurl(purl)).toBe(wellFormed) }) }) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index 9aa6f0d11b..b67304bdf1 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -366,30 +366,57 @@ describe('submitMemberContributorsJob', () => { }) describe('submitMemberContributorsJob, unknown ecosystem', () => { - it('creates an llm repo resolution job without a package lookup and starts the inferred-repo workflow', async () => { - const { req, res, status, json } = mockReqRes({ purl: 'pkg:hackage/aeson' }) - - await submitMemberContributorsJob(req, res) + const INFERRED_WORKFLOW = 'collectMemberContributorsFromInferredRepo' + + function inferredStart(jobId: string) { + return [ + INFERRED_WORKFLOW, + { + taskQueue: 'security-contacts-worker', + workflowId: `member-contributors-job/${jobId}`, + workflowIdConflictPolicy: 'USE_EXISTING', + workflowIdReusePolicy: 'REJECT_DUPLICATE', + retry: { maximumAttempts: 1 }, + args: [{ jobId }], + }, + ] + } - expect(findMemberContributorsPackage).not.toHaveBeenCalled() - const [, input] = createMemberContributorJob.mock.calls[0] - expect(input).toMatchObject({ - purl: 'pkg:hackage/aeson', - ecosystem: 'hackage', + function latestJob(id: string, latestStatus: 'done' | 'failed' | 'unresolvable') { + findLatestMemberContributorJobByPurl.mockResolvedValue({ + id, + status: latestStatus, collectionPath: 'llm_repo_resolution', - packageId: null, - repoId: null, - }) - const [workflowType, options] = start.mock.calls[0] - expect(workflowType).toBe('collectMemberContributorsFromInferredRepo') - expect(options).toMatchObject({ - taskQueue: 'security-contacts-worker', - workflowId: `member-contributors-job/${input.id}`, - args: [{ jobId: input.id }], }) - expect(status).toHaveBeenCalledWith(202) - expect(json).toHaveBeenCalledWith({ jobId: input.id, status: 'pending' }) - }) + } + + it.each(['pkg:hackage/aeson', 'pkg:Hackage/aeson'])( + 'creates an llm repo resolution job for %s without a package lookup and starts the inferred-repo workflow', + async (purl) => { + const { req, res, status, json } = mockReqRes({ purl }) + + await submitMemberContributorsJob(req, res) + + const [, input] = createMemberContributorJob.mock.calls[0] + expect(createMemberContributorJob.mock.calls).toEqual([ + [ + {}, + { + id: expect.stringMatching(UUID), + purl, + ecosystem: 'hackage', + collectionPath: 'llm_repo_resolution', + packageId: null, + repoId: null, + }, + ], + ]) + expect(start.mock.calls).toEqual([inferredStart(input.id)]) + expect(findMemberContributorsPackage).not.toHaveBeenCalled() + expect(status).toHaveBeenCalledWith(202) + expect(json).toHaveBeenCalledWith({ jobId: input.id, status: 'pending' }) + }, + ) it.each(['pkg:', 'pkg:hackage', 'pkg:/x'])( 'rejects the malformed purl %s with 400 before any database call', @@ -398,44 +425,32 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) expect(error).toBeInstanceOf(BadRequestError) expect((error as BadRequestError).message).toBe('Malformed purl') - expect(findActiveMemberContributorJobByPurl).not.toHaveBeenCalled() - expect(findLatestMemberContributorJobByPurl).not.toHaveBeenCalled() - expect(createMemberContributorJob).not.toHaveBeenCalled() - expect(start).not.toHaveBeenCalled() + expect( + [ + findActiveMemberContributorJobByPurl, + findLatestMemberContributorJobByPurl, + createMemberContributorJob, + start, + ].map((fn) => fn.mock.calls.length), + ).toEqual([0, 0, 0, 0]) }, ) - it('accepts an uppercase purl type and records the lowercased ecosystem', async () => { - const { req, res, status } = mockReqRes({ purl: 'pkg:Hackage/aeson' }) - - await submitMemberContributorsJob(req, res) - - const [, input] = createMemberContributorJob.mock.calls[0] - expect(input).toMatchObject({ - ecosystem: 'hackage', - collectionPath: 'llm_repo_resolution', - }) - expect(status).toHaveBeenCalledWith(202) - }) - - it('returns the active job and counts the submission', async () => { + it('returns the active job, counts the submission and re-dispatches the inferred-repo workflow', async () => { findActiveMemberContributorJobByPurl.mockResolvedValue({ id: 'active-1' }) - const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + const { req, res, status, json } = mockReqRes({ purl: 'pkg:hackage/aeson' }) await submitMemberContributorsJob(req, res) - expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'active-1') + expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'active-1']]) + expect(start.mock.calls).toEqual([inferredStart('active-1')]) expect(createMemberContributorJob).not.toHaveBeenCalled() expect(status).toHaveBeenCalledWith(202) - expect(start.mock.calls[0][0]).toBe('collectMemberContributorsFromInferredRepo') + expect(json).toHaveBeenCalledWith({ jobId: 'active-1', status: 'pending' }) }) - it('refuses with 409 carrying the previous jobId while its result is still readable', async () => { - findLatestMemberContributorJobByPurl.mockResolvedValue({ - id: 'done-1', - status: 'done', - collectionPath: 'llm_repo_resolution', - }) + it('refuses with 409 carrying the previous jobId while its workflow result is still readable', async () => { + latestJob('done-1', 'done') describeWorkflow.mockResolvedValue({ status: { name: 'COMPLETED' } }) const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) @@ -443,40 +458,29 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { expect(error).toBeInstanceOf(ConflictError) expect((error as ConflictError).context).toEqual({ jobId: 'done-1' }) - expect(incrementMemberContributorJobSubmissions).toHaveBeenCalledWith({}, 'done-1') + expect(getHandle.mock.calls).toEqual([['member-contributors-job/done-1']]) + expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'done-1']]) expect(createMemberContributorJob).not.toHaveBeenCalled() }) - it('accepts a new job when the previous done result has expired', async () => { - findLatestMemberContributorJobByPurl.mockResolvedValue({ - id: 'done-1', - status: 'done', - collectionPath: 'llm_repo_resolution', - }) - describeWorkflow.mockRejectedValue(new WorkflowNotFoundError('gone', 'id', undefined)) - const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) - - await submitMemberContributorsJob(req, res) - - expect(createMemberContributorJob).toHaveBeenCalledTimes(1) - expect(status).toHaveBeenCalledWith(202) - }) - - it.each(['failed', 'unresolvable'] as const)( - 'accepts a new job when the latest job is %s', - async (latest) => { - findLatestMemberContributorJobByPurl.mockResolvedValue({ - id: 'old-1', - status: latest, - collectionPath: 'llm_repo_resolution', - }) + it.each([ + { latestStatus: 'done' as const, describedWorkflows: [['member-contributors-job/old-1']] }, + { latestStatus: 'failed' as const, describedWorkflows: [] }, + { latestStatus: 'unresolvable' as const, describedWorkflows: [] }, + ])( + 'accepts a new job when the latest job is $latestStatus and its result is not readable', + async ({ latestStatus, describedWorkflows }) => { + latestJob('old-1', latestStatus) const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) await submitMemberContributorsJob(req, res) - expect(describeWorkflow).not.toHaveBeenCalled() - expect(createMemberContributorJob).toHaveBeenCalledTimes(1) - expect(status).toHaveBeenCalledWith(202) + expect({ + describedWorkflows: getHandle.mock.calls, + created: createMemberContributorJob.mock.calls.length, + submissionsCounted: incrementMemberContributorJobSubmissions.mock.calls, + statuses: status.mock.calls, + }).toEqual({ describedWorkflows, created: 1, submissionsCounted: [], statuses: [[202]] }) }, ) }) diff --git a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts index 9588a0a431..77a64d5c9a 100644 --- a/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts +++ b/services/apps/packages_worker/src/member-contributors/__tests__/workflows.test.ts @@ -281,14 +281,40 @@ describe('activity proxy options', () => { retry: { initialInterval: '30 seconds', backoffCoefficient: 2, maximumAttempts: 3 }, }) }) + + it('bounds the llm resolution to two attempts and retries the inferred repo finisher forever', () => { + const retry = { initialInterval: '30 seconds', backoffCoefficient: 2, maximumAttempts: 3 } + expect({ + start: optionsFor('startInferredRepoMemberContributorsJob'), + resolve: optionsFor('resolveRepoViaLlm'), + fetch: optionsFor('fetchGithubRepoContributors'), + build: optionsFor('buildInferredRepoMemberContributors'), + unresolvable: optionsFor('markMemberContributorsJobUnresolvable'), + }).toEqual({ + start: { startToCloseTimeout: '5 minutes', retry }, + resolve: { startToCloseTimeout: '3 minutes', retry: { ...retry, maximumAttempts: 2 } }, + fetch: { startToCloseTimeout: '1 hour', heartbeatTimeout: '5 minutes', retry }, + build: { startToCloseTimeout: '5 minutes', retry }, + unresolvable: { + startToCloseTimeout: '1 minute', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumInterval: '5 minutes', + }, + }, + }) + }) }) describe('collectMemberContributorsFromInferredRepo', () => { + const job = { jobId: 'job-1' } + const repoUrl = 'https://github.com/haskell/aeson' const body = { purl: 'pkg:hackage/aeson', name: 'aeson', ecosystem: 'hackage', - repoUrl: 'https://github.com/haskell/aeson', + repoUrl, repoUrlSource: 'llm_inferred', repoMappingConfidence: null, contributors: [], @@ -300,11 +326,7 @@ describe('collectMemberContributorsFromInferredRepo', () => { purl: 'pkg:hackage/aeson', ecosystem: 'hackage', }) - h.acts.resolveRepoViaLlm.mockResolvedValue({ - outcome: 'resolved', - repoUrl: 'https://github.com/haskell/aeson', - confidence: 0.95, - }) + h.acts.resolveRepoViaLlm.mockResolvedValue({ outcome: 'resolved', repoUrl, confidence: 0.95 }) h.acts.fetchGithubRepoContributors.mockResolvedValue({ status: 'fetched', logins: ['octo'] }) h.acts.resolveNewIdentityAffiliations.mockResolvedValue({ identities: 1, @@ -314,70 +336,58 @@ describe('collectMemberContributorsFromInferredRepo', () => { h.acts.buildInferredRepoMemberContributors.mockResolvedValue(body) }) - it('runs start, resolve, fetch, affiliations, build, complete and returns the body', async () => { - await expect(collectMemberContributorsFromInferredRepo({ jobId: 'job-1' })).resolves.toEqual( - body, - ) - expect(h.acts.resolveRepoViaLlm).toHaveBeenCalledWith({ - jobId: 'job-1', - purl: 'pkg:hackage/aeson', - }) - expect(h.acts.fetchGithubRepoContributors).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/haskell/aeson', - }) - expect(h.acts.resolveNewIdentityAffiliations).toHaveBeenCalledWith({ logins: ['octo'] }) - expect(h.acts.buildInferredRepoMemberContributors).toHaveBeenCalledWith({ - purl: 'pkg:hackage/aeson', - ecosystem: 'hackage', - repoUrl: 'https://github.com/haskell/aeson', - logins: ['octo'], - }) - expect(h.acts.completeMemberContributorsJob).toHaveBeenCalledWith({ jobId: 'job-1' }) + it('resolves the repo, collects and affiliates its contributors, then completes with the body', async () => { + await expect(collectMemberContributorsFromInferredRepo(job)).resolves.toEqual(body) + + const sequence = [ + h.acts.startInferredRepoMemberContributorsJob, + h.acts.resolveRepoViaLlm, + h.acts.fetchGithubRepoContributors, + h.acts.resolveNewIdentityAffiliations, + h.acts.buildInferredRepoMemberContributors, + h.acts.completeMemberContributorsJob, + ] + expect(sequence.map((fn) => fn.mock.calls)).toEqual([ + [[job]], + [[{ jobId: 'job-1', purl: 'pkg:hackage/aeson' }]], + [[{ repoUrl }]], + [[{ logins: ['octo'] }]], + [[{ purl: 'pkg:hackage/aeson', ecosystem: 'hackage', repoUrl, logins: ['octo'] }]], + [[job]], + ]) + const order = sequence.map((fn) => fn.mock.invocationCallOrder[0]) + expect(order).toEqual([...order].sort((a, b) => a - b)) expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() expect(h.acts.markMemberContributorsJobUnresolvable).not.toHaveBeenCalled() }) - it('marks unresolvable and returns nothing without fetching', async () => { + it('marks the job unresolvable with the reason and attempted url, returning nothing', async () => { h.acts.resolveRepoViaLlm.mockResolvedValue({ outcome: 'unresolvable', reason: 'LLM returned no repository', repoUrl: null, }) - await expect( - collectMemberContributorsFromInferredRepo({ jobId: 'job-1' }), - ).resolves.toBeUndefined() - expect(h.acts.markMemberContributorsJobUnresolvable).toHaveBeenCalledWith({ - jobId: 'job-1', - reason: 'LLM returned no repository', - repoUrl: null, - }) + + await expect(collectMemberContributorsFromInferredRepo(job)).resolves.toBeUndefined() + + expect(h.acts.markMemberContributorsJobUnresolvable.mock.calls).toEqual([ + [{ jobId: 'job-1', reason: 'LLM returned no repository', repoUrl: null }], + ]) expect(h.acts.fetchGithubRepoContributors).not.toHaveBeenCalled() expect(h.acts.completeMemberContributorsJob).not.toHaveBeenCalled() expect(h.acts.failMemberContributorsJob).not.toHaveBeenCalled() }) - it('fails the job with the activity cause and rethrows', async () => { + it('fails the job non-cancellably with the activity cause and rethrows', async () => { const cause = new Error('Bedrock credentials are not configured') h.acts.resolveRepoViaLlm.mockRejectedValue(new h.ActivityFailure('Activity failed', cause)) - await expect(collectMemberContributorsFromInferredRepo({ jobId: 'job-1' })).rejects.toThrow( - 'Activity failed', - ) - expect(h.acts.failMemberContributorsJob).toHaveBeenCalledWith({ - jobId: 'job-1', - reason: 'Bedrock credentials are not configured', - }) - expect(h.nonCancellable).toHaveBeenCalled() - }) - it('proxies resolveRepoViaLlm with at most two attempts and the unresolvable finisher with unbounded retry', () => { - const resolveProxy = h.proxyCalls.find((c) => c.activities.includes('resolveRepoViaLlm')) - expect(resolveProxy?.options).toMatchObject({ - startToCloseTimeout: '3 minutes', - retry: { maximumAttempts: 2 }, - }) - const finishProxy = h.proxyCalls.find((c) => - c.activities.includes('markMemberContributorsJobUnresolvable'), - ) - expect(finishProxy?.options.retry).not.toHaveProperty('maximumAttempts') + await expect(collectMemberContributorsFromInferredRepo(job)).rejects.toThrow('Activity failed') + + expect(h.acts.failMemberContributorsJob.mock.calls).toEqual([ + [{ jobId: 'job-1', reason: 'Bedrock credentials are not configured' }], + ]) + expect(h.nonCancellable).toHaveBeenCalledTimes(1) + expect(h.acts.completeMemberContributorsJob).not.toHaveBeenCalled() }) }) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts index 36bef6127e..a8dc3fe616 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/buildInferredRepoBody.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' import * as affiliationsDal from '@crowd/data-access-layer/src/osspckgs/identityAffiliations' import { buildInferredRepoMemberContributors, purlPackageName } from '../buildInferredRepoBody' @@ -36,6 +37,16 @@ describe('buildInferredRepoMemberContributors', () => { repoUrl: 'https://github.com/haskell/aeson', logins: ['octo', 'nobody'], } + const emptyBody = { + purl: 'pkg:hackage/aeson', + name: 'aeson', + ecosystem: 'hackage', + repoUrl: 'https://github.com/haskell/aeson', + repoUrlSource: 'llm_inferred', + repoMappingConfidence: null, + contributors: [], + maintainers: { registry: [], governanceFiles: [] }, + } it('builds the body from affiliated logins with llm provenance', async () => { find.mockResolvedValue([ @@ -59,30 +70,39 @@ describe('buildInferredRepoMemberContributors', () => { const body = await buildInferredRepoMemberContributors(input) - expect(find).toHaveBeenCalledWith(qx, [ - { identityType: 'github-login', identityValue: 'octo' }, - { identityType: 'github-login', identityValue: 'nobody' }, + expect(find.mock.calls).toEqual([ + [ + qx, + [ + { identityType: 'github-login', identityValue: 'octo' }, + { identityType: 'github-login', identityValue: 'nobody' }, + ], + ], ]) - expect(body).toMatchObject({ - purl: 'pkg:hackage/aeson', - name: 'aeson', - ecosystem: 'hackage', - repoUrl: 'https://github.com/haskell/aeson', - repoUrlSource: 'llm_inferred', - repoMappingConfidence: null, - maintainers: { registry: [], governanceFiles: [] }, - }) - expect(body.contributors).toHaveLength(1) - expect(body.contributors[0]).toMatchObject({ - githubLogin: 'octo', - sources: ['github'], - organization: { name: 'Google', confidence: 'cdp', confidenceScore: 1 }, + expect(body).toEqual({ + ...emptyBody, + contributors: [ + { + githubLogin: 'octo', + displayName: null, + emails: [], + registryUsernames: [], + sources: ['github'], + organization: { + id: canonicalCdpOrganizationIdByMemberName().get('Google'), + name: 'Google', + confidence: 'cdp', + confidenceScore: 1, + }, + }, + ], }) }) - it('returns an empty contributors list without a query when there are no logins', async () => { - const body = await buildInferredRepoMemberContributors({ ...input, logins: [] }) + it('returns an empty body without querying affiliations when there are no logins', async () => { + await expect(buildInferredRepoMemberContributors({ ...input, logins: [] })).resolves.toEqual( + emptyBody, + ) expect(find).not.toHaveBeenCalled() - expect(body.contributors).toEqual([]) }) }) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts index 264d6d73da..9e3d65e000 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/canonicalGithubUrl.test.ts @@ -2,33 +2,25 @@ import { describe, expect, it } from 'vitest' import { canonicalGithubUrl } from '../canonicalGithubUrl' +const aeson = { + kind: 'github', + url: 'https://github.com/haskell/aeson', + owner: 'haskell', + name: 'aeson', +} + describe('canonicalGithubUrl', () => { it.each([ - ['https://github.com/haskell/aeson', 'https://github.com/haskell/aeson'], - ['https://github.com/Haskell/Aeson.git', 'https://github.com/haskell/aeson'], - ['http://www.github.com/haskell/aeson/', 'https://github.com/haskell/aeson'], - ['https://github.com/haskell/aeson/tree/main/src', 'https://github.com/haskell/aeson'], - ['github.com/haskell/aeson', 'https://github.com/haskell/aeson'], - ])('canonicalises %s', (raw, url) => { - expect(canonicalGithubUrl(raw)).toEqual({ - kind: 'github', - url, - owner: 'haskell', - name: 'aeson', - }) - }) - - it('reports other hosts', () => { - expect(canonicalGithubUrl('https://gitlab.com/a/b')).toEqual({ - kind: 'other', - host: 'gitlab.com', - }) + ['https://github.com/haskell/aeson', aeson], + ['https://github.com/Haskell/Aeson.git', aeson], + ['http://www.github.com/haskell/aeson/', aeson], + ['https://github.com/haskell/aeson/tree/main/src', aeson], + ['github.com/haskell/aeson', aeson], + ['https://gitlab.com/a/b', { kind: 'other', host: 'gitlab.com' }], + ['https://github.com/haskell', { kind: 'invalid' }], + ['not a url', { kind: 'invalid' }], + ['https://github.com/', { kind: 'invalid' }], + ])('classifies %s', (raw, expected) => { + expect(canonicalGithubUrl(raw)).toEqual(expected) }) - - it.each(['https://github.com/haskell', 'not a url', 'https://github.com/'])( - 'rejects %s', - (raw) => { - expect(canonicalGithubUrl(raw)).toEqual({ kind: 'invalid' }) - }, - ) }) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts index d27ce36e62..a98ab72fb4 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/fetchGithubRepoContributors.test.ts @@ -11,6 +11,8 @@ vi.mock('@temporalio/activity', async (importOriginal) => ({ vi.mock('../../../config', () => ({ getGithubAppConfig: vi.fn(() => ({ appId: '1' })) })) vi.mock('../../../security-contacts/githubToken', () => ({ githubApiGet: vi.fn() })) +const repo = { repoUrl: 'https://github.com/a/b' } + function contributorsJson(entries: Array<{ login: string; type?: string }>): string { return JSON.stringify(entries.map((e) => ({ type: 'User', ...e }))) } @@ -24,7 +26,7 @@ afterEach(() => { }) describe('fetchGithubRepoContributors', () => { - it('returns the lowercased human logins without writing anything', async () => { + it('returns the lowercased human logins of the first contributors page', async () => { vi.mocked(githubApiGet).mockResolvedValue({ status: 200, text: contributorsJson([ @@ -35,55 +37,55 @@ describe('fetchGithubRepoContributors', () => { }) await expect( fetchGithubRepoContributors({ repoUrl: 'https://github.com/haskell/aeson' }), - ).resolves.toEqual({ - status: 'fetched', - logins: ['octo', 'ada'], - }) - expect(githubApiGet).toHaveBeenCalledWith( - expect.stringContaining('/repos/haskell/aeson/contributors'), - 15000, - { extraOkStatuses: [204] }, - ) + ).resolves.toEqual({ status: 'fetched', logins: ['octo', 'ada'] }) + expect(vi.mocked(githubApiGet).mock.calls).toEqual([ + ['/repos/haskell/aeson/contributors?per_page=100&page=1', 15000, { extraOkStatuses: [204] }], + ]) }) - it('returns empty on 204 and when every contributor is a bot', async () => { - vi.mocked(githubApiGet).mockResolvedValue({ status: 204, text: null }) - await expect( - fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }), - ).resolves.toEqual({ status: 'empty', logins: [] }) - vi.mocked(githubApiGet).mockResolvedValue({ - status: 200, - text: contributorsJson([{ login: 'x[bot]', type: 'Bot' }]), + it.each([ + ['an empty repository', { status: 204, text: null }], + [ + 'only bot contributors', + { status: 200, text: contributorsJson([{ login: 'x[bot]', type: 'Bot' }]) }, + ], + ])('returns empty for %s', async (_label, response) => { + vi.mocked(githubApiGet).mockResolvedValue(response) + await expect(fetchGithubRepoContributors(repo)).resolves.toEqual({ + status: 'empty', + logins: [], }) - await expect( - fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }), - ).resolves.toEqual({ status: 'empty', logins: [] }) }) - it('fails without retry when the repo is missing', async () => { - vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) - const error = await fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }).catch( - (e: unknown) => e, - ) - expect(error).toMatchObject({ message: 'Repository not found on GitHub', nonRetryable: true }) - }) - - it('throws a retryable error when the fetch errors', async () => { - vi.mocked(githubApiGet).mockRejectedValue(new Error('socket hang up')) - const error = await fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }).catch( - (e: unknown) => e, - ) - expect((error as Error).message).toContain('GitHub contributors fetch failed') - expect(error).not.toMatchObject({ nonRetryable: true }) + it.each([ + [ + 'without retry when the repo is missing', + () => vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }), + { message: 'Repository not found on GitHub', nonRetryable: true }, + ], + [ + 'retryably when the fetch errors', + () => vi.mocked(githubApiGet).mockRejectedValue(new Error('socket hang up')), + { + message: 'GitHub contributors fetch failed for https://github.com/a/b', + nonRetryable: false, + }, + ], + ])('fails %s', async (_label, arrange, expected) => { + arrange() + const error = (await fetchGithubRepoContributors(repo).catch((e: unknown) => e)) as Error & { + nonRetryable?: boolean + } + expect({ message: error.message, nonRetryable: error.nonRetryable === true }).toEqual(expected) }) - it('heartbeats while the fetch is in flight', async () => { + it('heartbeats while the fetch is in flight and stops once it settles', async () => { vi.useFakeTimers() vi.mocked(githubApiGet).mockImplementation( () => new Promise((resolve) => setTimeout(() => resolve({ status: 204, text: null }), 45_000)), ) - const pending = fetchGithubRepoContributors({ repoUrl: 'https://github.com/a/b' }) + const pending = fetchGithubRepoContributors(repo) await vi.advanceTimersByTimeAsync(30_000) expect(heartbeat).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(15_000) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts index 9cc13e888b..cc21e7de93 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts @@ -8,14 +8,32 @@ vi.mock('@crowd/logging', () => ({ getServiceChildLogger: () => log })) type ResponseBlock = { type: 'thinking'; thinking: string } | { type: 'text'; text: string } -function bedrockResponse( - text: string | null, - usage: unknown = { input_tokens: 300, output_tokens: 80 }, - stopReason = 'end_turn', -): string { +const usage = { inputTokens: 300, outputTokens: 80, costUsd: 300 * 0.000002 + 80 * 0.00001 } +const twoCandidates = [ + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, +] +const fourCandidates = [1, 2, 3, 4].map((n) => ({ + repositoryUrl: `https://github.com/o/r${n}`, + confidence: 0.9, +})) + +function bedrockResponse(text: string | null, stopReason = 'end_turn'): string { const content: ResponseBlock[] = [{ type: 'thinking', thinking: '' }] if (text !== null) content.push({ type: 'text', text }) - return JSON.stringify({ content, usage, stop_reason: stopReason }) + return JSON.stringify({ + content, + usage: { input_tokens: 300, output_tokens: 80 }, + stop_reason: stopReason, + }) +} + +function answer(candidates: unknown[], rationale = ''): string { + return JSON.stringify({ candidates, rationale }) +} + +function invokeReturning(response: string) { + return vi.fn().mockResolvedValue(response) } beforeEach(() => { @@ -23,17 +41,9 @@ beforeEach(() => { }) describe('inferRepositoryCandidates', () => { - it('sends the purl to sonnet 5.5 and parses the ranked candidates from the first text block', async () => { - const invoke = vi.fn().mockResolvedValue( - bedrockResponse( - JSON.stringify({ - candidates: [ - { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, - { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, - ], - rationale: 'aeson moved to the haskell org', - }), - ), + it('sends the purl to sonnet 5.5, parses the first text block and logs the call', async () => { + const invoke = invokeReturning( + bedrockResponse(answer(twoCandidates, 'aeson moved to the haskell org')), ) const result = await inferRepositoryCandidates('pkg:hackage/aeson', invoke) @@ -41,108 +51,70 @@ describe('inferRepositoryCandidates', () => { expect(invoke).toHaveBeenCalledTimes(1) const [modelId, body] = invoke.mock.calls[0] expect(modelId).toBe('us.anthropic.claude-sonnet-5-5') - const parsed = JSON.parse(body) - expect(Object.keys(parsed).sort()).toEqual([ - 'anthropic_version', - 'max_tokens', - 'messages', - 'output_config', - 'system', - ]) - expect(parsed.max_tokens).toBe(2048) - expect(parsed.anthropic_version).toBe('bedrock-2023-05-31') - expect(parsed.messages).toEqual([ - { role: 'user', content: [{ type: 'text', text: 'pkg:hackage/aeson' }] }, - ]) - expect(parsed.system).toContain('canonical source repository') - expect(parsed.output_config).toEqual({ effort: 'low' }) - expect(result).toEqual({ - status: 'answered', - candidates: [ - { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.95 }, - { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.4 }, - ], - usage: { inputTokens: 300, outputTokens: 80, costUsd: 300 * 0.000002 + 80 * 0.00001 }, - }) - }) - - it('accepts a fenced JSON answer and caps candidates at three', async () => { - const candidates = [1, 2, 3, 4].map((n) => ({ - repositoryUrl: `https://github.com/o/r${n}`, - confidence: 0.9, - })) - const invoke = vi - .fn() - .mockResolvedValue( - bedrockResponse('```json\n' + JSON.stringify({ candidates, rationale: '' }) + '\n```'), - ) - const result = await inferRepositoryCandidates('pkg:conan/boost', invoke) - expect(result.status).toBe('answered') - if (result.status === 'answered') expect(result.candidates).toHaveLength(3) - }) - - it('reports an empty candidate list as answered', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse(JSON.stringify({ candidates: [], rationale: 'unknown' }))) - await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ - status: 'answered', - candidates: [], - }) - }) - - it('reports unparseable when there is no text block, keeping the usage', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse(null, { input_tokens: 10, output_tokens: 0 })) - const result = await inferRepositoryCandidates('pkg:conan/x', invoke) - expect(result).toMatchObject({ - status: 'unparseable', - reason: 'LLM answer had no text content', - usage: { inputTokens: 10, outputTokens: 0 }, - }) - expect(result.usage.costUsd).toBeCloseTo(10 * 0.000002, 12) - }) - - it('reports unparseable on invalid JSON', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse('I think it is github.com/haskell/aeson')) - await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ - status: 'unparseable', - reason: 'LLM answer was not valid JSON', - }) - }) - - it('reports unparseable when the JSON does not match the schema', async () => { - const invoke = vi.fn().mockResolvedValue( - bedrockResponse( - JSON.stringify({ - candidates: [{ repositoryUrl: 'https://github.com/a/b', confidence: 1.7 }], - }), - ), - ) - await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).resolves.toMatchObject({ - status: 'unparseable', - reason: 'LLM answer did not match the expected schema', + expect(JSON.parse(body)).toEqual({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 2048, + output_config: { effort: 'low' }, + system: expect.stringContaining('canonical source repository'), + messages: [{ role: 'user', content: [{ type: 'text', text: 'pkg:hackage/aeson' }] }], }) + expect(result).toEqual({ status: 'answered', candidates: twoCandidates, usage }) + expect(log.info.mock.calls).toEqual([ + [ + { + modelId: 'us.anthropic.claude-sonnet-5-5', + purl: 'pkg:hackage/aeson', + stopReason: 'end_turn', + inputTokens: 300, + outputTokens: 80, + costUsd: usage.costUsd, + candidates: 2, + durationMs: expect.any(Number), + }, + 'LLM repo inference call', + ], + ]) }) - it('logs the stop reason of the call', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse(JSON.stringify({ candidates: [], rationale: 'unknown' }))) - await inferRepositoryCandidates('pkg:conan/x', invoke) - expect(log.info).toHaveBeenCalledWith( - expect.objectContaining({ stopReason: 'end_turn' }), - 'LLM repo inference call', - ) + it.each([ + [ + 'a fenced answer, capped at three candidates', + bedrockResponse('```json\n' + answer(fourCandidates) + '\n```'), + { status: 'answered', candidates: fourCandidates.slice(0, 3), usage }, + ], + [ + 'an empty candidate list', + bedrockResponse(answer([], 'unknown')), + { status: 'answered', candidates: [], usage }, + ], + [ + 'a response without a text block', + bedrockResponse(null), + { status: 'unparseable', reason: 'LLM answer had no text content', usage }, + ], + [ + 'an answer that is not JSON', + bedrockResponse('I think it is github.com/haskell/aeson'), + { status: 'unparseable', reason: 'LLM answer was not valid JSON', usage }, + ], + [ + 'an answer outside the schema', + bedrockResponse(answer([{ repositoryUrl: 'https://github.com/a/b', confidence: 1.7 }])), + { status: 'unparseable', reason: 'LLM answer did not match the expected schema', usage }, + ], + [ + 'a refusal', + bedrockResponse(null, 'refusal'), + { status: 'unparseable', reason: 'LLM refused to answer', usage }, + ], + ])('returns the result with usage for %s', async (_label, response, expected) => { + await expect( + inferRepositoryCandidates('pkg:conan/x', invokeReturning(response)), + ).resolves.toEqual(expected) }) - it('rejects a truncated answer so the activity retries instead of marking it unresolvable', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse('{"candidates": [', undefined, 'max_tokens')) + it('rejects a truncated answer after logging its usage so the activity retries', async () => { + const invoke = invokeReturning(bedrockResponse('{"candidates": [', 'max_tokens')) await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( 'LLM answer truncated at max_tokens', ) @@ -152,28 +124,18 @@ describe('inferRepositoryCandidates', () => { ) }) - it('reports a refusal as unparseable, keeping the usage', async () => { - const invoke = vi - .fn() - .mockResolvedValue(bedrockResponse(null, { input_tokens: 20, output_tokens: 5 }, 'refusal')) - const result = await inferRepositoryCandidates('pkg:conan/x', invoke) - expect(result).toMatchObject({ - status: 'unparseable', - reason: 'LLM refused to answer', - usage: { inputTokens: 20, outputTokens: 5 }, - }) - }) - it.each([ - ['missing', undefined], - ['non-numeric', { input_tokens: '300', output_tokens: 80 }], - ['partial', { input_tokens: 300 }], - ])('rejects a response whose usage envelope is %s', async (_label, usage) => { - const invoke = vi - .fn() - .mockResolvedValue( - JSON.stringify({ content: [{ type: 'text', text: '{}' }], usage, stop_reason: 'end_turn' }), - ) + ['usage is missing', undefined], + ['usage is non-numeric', { input_tokens: '300', output_tokens: 80 }], + ['usage is partial', { input_tokens: 300 }], + ])('rejects a response whose %s', async (_label, responseUsage) => { + const invoke = invokeReturning( + JSON.stringify({ + content: [{ type: 'text', text: '{}' }], + usage: responseUsage, + stop_reason: 'end_turn', + }), + ) await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( 'Bedrock response had no token usage', ) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts index dd43be0b0c..c765759af7 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts @@ -29,18 +29,40 @@ const usage = { inputTokens: 300, outputTokens: 80, costUsd: 0.0014 } const input = { jobId: 'job-1', purl: 'pkg:hackage/aeson' } const recordRepo = vi.mocked(jobsDal.recordMemberContributorJobInferredRepo) const recordUsage = vi.mocked(jobsDal.recordMemberContributorJobLlmUsage) +const found = { status: 200, text: '{}' } +const absent = { status: 404, text: null } + +type Inference = Awaited> +type GithubResult = Awaited> + +function answered(candidates: Array<{ repositoryUrl: string; confidence: number }>): Inference { + return { status: 'answered', candidates, usage } +} + +function githubCheck(path: string) { + return [path, 15000] +} function expectUsageRecordedOnceBeforeGithub() { - expect(recordUsage).toHaveBeenCalledTimes(1) - expect(recordUsage).toHaveBeenCalledWith(qx, 'job-1', usage) + expect(recordUsage.mock.calls).toEqual([[qx, 'job-1', usage]]) const githubCalls = vi.mocked(githubApiGet).mock.invocationCallOrder if (githubCalls.length > 0) { expect(recordUsage.mock.invocationCallOrder[0]).toBeLessThan(githubCalls[0]) } } -function answered(candidates: Array<{ repositoryUrl: string; confidence: number }>) { - vi.mocked(inferRepositoryCandidates).mockResolvedValue({ status: 'answered', candidates, usage }) +async function resolveWith(inference: Inference, githubResults: GithubResult[]) { + vi.mocked(inferRepositoryCandidates).mockResolvedValue(inference) + for (const githubResult of githubResults) { + vi.mocked(githubApiGet).mockResolvedValueOnce(githubResult) + } + const result = await resolveRepoViaLlm(input) + expectUsageRecordedOnceBeforeGithub() + return { + result, + githubCalls: vi.mocked(githubApiGet).mock.calls, + recordedRepos: recordRepo.mock.calls, + } } beforeEach(() => { @@ -49,161 +71,185 @@ beforeEach(() => { vi.mocked(getGithubAppConfig).mockReturnValue( {} as unknown as ReturnType, ) - vi.mocked(githubApiGet).mockResolvedValue({ status: 200, text: '{}' }) + vi.mocked(githubApiGet).mockResolvedValue(found) }) describe('resolveRepoViaLlm', () => { - it('resolves the first candidate that exists on GitHub and records it on the job', async () => { - answered([{ repositoryUrl: 'https://github.com/Haskell/aeson.git', confidence: 0.95 }]) - - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'resolved', - repoUrl: 'https://github.com/haskell/aeson', - confidence: 0.95, - }) - expect(githubApiGet).toHaveBeenCalledWith('/repos/haskell/aeson', 15000) - expect(recordRepo).toHaveBeenCalledWith(qx, 'job-1', { + it.each([ + { + name: 'canonicalises the first existing candidate', + inference: answered([ + { repositoryUrl: 'https://github.com/Haskell/aeson.git', confidence: 0.95 }, + ]), + github: [found], + checked: [githubCheck('/repos/haskell/aeson')], repoUrl: 'https://github.com/haskell/aeson', confidence: 0.95, - }) - expectUsageRecordedOnceBeforeGithub() - }) - - it('skips absent candidates and takes the next existing one', async () => { - answered([ - { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.9 }, - { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.8 }, - ]) - vi.mocked(githubApiGet) - .mockResolvedValueOnce({ status: 404, text: null }) - .mockResolvedValueOnce({ status: 200, text: '{}' }) - - await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ - outcome: 'resolved', + }, + { + name: 'skips an absent candidate for the next existing one', + inference: answered([ + { repositoryUrl: 'https://github.com/bos/aeson', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/haskell/aeson', confidence: 0.8 }, + ]), + github: [absent, found], + checked: [githubCheck('/repos/bos/aeson'), githubCheck('/repos/haskell/aeson')], repoUrl: 'https://github.com/haskell/aeson', - }) - }) - - it('is unresolvable with the last attempted url when every candidate is absent', async () => { - answered([ - { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, - { repositoryUrl: 'https://github.com/a/two', confidence: 0.8 }, - ]) - vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) + confidence: 0.8, + }, + ])( + '$name and records it on the job', + async ({ inference, github, checked, repoUrl, confidence }) => { + await expect(resolveWith(inference, github)).resolves.toEqual({ + result: { outcome: 'resolved', repoUrl, confidence }, + githubCalls: checked, + recordedRepos: [[qx, 'job-1', { repoUrl, confidence }]], + }) + }, + ) - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + it.each([ + { + name: 'every candidate is absent, keeping the last attempted url', + inference: answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.8 }, + ]), + github: [absent, absent], + checked: [githubCheck('/repos/a/one'), githubCheck('/repos/a/two')], reason: 'Repository not found on GitHub: https://github.com/a/two', repoUrl: 'https://github.com/a/two', - }) - expectUsageRecordedOnceBeforeGithub() - expect(recordRepo).not.toHaveBeenCalled() - }) - - it('is unresolvable below the confidence threshold without calling GitHub', async () => { - answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }]) - - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + }, + { + name: 'the only candidate is below the confidence threshold', + inference: answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }]), + github: [], + checked: [], reason: 'LLM confidence 0.4 below threshold 0.7', repoUrl: null, - }) - expect(githubApiGet).not.toHaveBeenCalled() - expectUsageRecordedOnceBeforeGithub() - }) - - it('quotes the highest candidate confidence when all are below the threshold', async () => { - answered([ - { repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }, - { repositoryUrl: 'https://github.com/a/c', confidence: 0.6 }, - ]) - - await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ + }, + { + name: 'all candidates are below the threshold, quoting the highest confidence', + inference: answered([ + { repositoryUrl: 'https://github.com/a/b', confidence: 0.4 }, + { repositoryUrl: 'https://github.com/a/c', confidence: 0.6 }, + ]), + github: [], + checked: [], reason: 'LLM confidence 0.6 below threshold 0.7', - }) - }) - - it('ignores lower-ranked candidates below the threshold', async () => { - answered([ - { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, - { repositoryUrl: 'https://github.com/a/two', confidence: 0.2 }, - ]) - vi.mocked(githubApiGet).mockResolvedValue({ status: 404, text: null }) - - await expect(resolveRepoViaLlm(input)).resolves.toMatchObject({ outcome: 'unresolvable' }) - expect(githubApiGet).toHaveBeenCalledTimes(1) - }) - - it('is unresolvable for an unsupported host', async () => { - answered([{ repositoryUrl: 'https://gitlab.com/a/b', confidence: 0.9 }]) - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + repoUrl: null, + }, + { + name: 'the confident candidate is absent and lower-ranked ones are below the threshold', + inference: answered([ + { repositoryUrl: 'https://github.com/a/one', confidence: 0.9 }, + { repositoryUrl: 'https://github.com/a/two', confidence: 0.2 }, + ]), + github: [absent], + checked: [githubCheck('/repos/a/one')], + reason: 'Repository not found on GitHub: https://github.com/a/one', + repoUrl: 'https://github.com/a/one', + }, + { + name: 'the candidate host is not GitHub', + inference: answered([{ repositoryUrl: 'https://gitlab.com/a/b', confidence: 0.9 }]), + github: [], + checked: [], reason: 'Repository host not supported: gitlab.com', repoUrl: 'https://gitlab.com/a/b', - }) - expectUsageRecordedOnceBeforeGithub() - }) - - it('is unresolvable with the attempted url when it is not recognised', async () => { - answered([{ repositoryUrl: 'not a url', confidence: 0.9 }]) - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + }, + { + name: 'the candidate url is not recognised', + inference: answered([{ repositoryUrl: 'not a url', confidence: 0.9 }]), + github: [], + checked: [], reason: 'Repository URL not recognised: not a url', repoUrl: 'not a url', - }) - }) - - it('is unresolvable for an empty candidate list', async () => { - answered([]) - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + }, + { + name: 'the LLM returns no candidate', + inference: answered([]), + github: [], + checked: [], reason: 'LLM returned no repository', repoUrl: null, - }) - expectUsageRecordedOnceBeforeGithub() - }) - - it('is unresolvable for an unparseable answer and still records usage', async () => { - vi.mocked(inferRepositoryCandidates).mockResolvedValue({ - status: 'unparseable', - reason: 'LLM answer was not valid JSON', - usage, - }) - await expect(resolveRepoViaLlm(input)).resolves.toEqual({ - outcome: 'unresolvable', + }, + { + name: 'the LLM answer is unparseable', + inference: { status: 'unparseable', reason: 'LLM answer was not valid JSON', usage }, + github: [], + checked: [], reason: 'LLM answer was not valid JSON', repoUrl: null, + }, + ] satisfies Array<{ + name: string + inference: Inference + github: GithubResult[] + checked: Array> + reason: string + repoUrl: string | null + }>)('is unresolvable when $name', async ({ inference, github, checked, reason, repoUrl }) => { + await expect(resolveWith(inference, github)).resolves.toEqual({ + result: { outcome: 'unresolvable', reason, repoUrl }, + githubCalls: checked, + recordedRepos: [], }) - expectUsageRecordedOnceBeforeGithub() }) - it('throws a retryable error when the GitHub existence check fails', async () => { - answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.9 }]) - vi.mocked(githubApiGet).mockRejectedValue(new Error('HTTP 502')) - const error = await resolveRepoViaLlm(input).catch((e: unknown) => e) - expect(error).toBeInstanceOf(Error) - expect((error as Error).message).toBe('HTTP 502') - expect(error).not.toMatchObject({ nonRetryable: true }) - expectUsageRecordedOnceBeforeGithub() + it.each([ + { + name: 'retryably when the GitHub existence check fails, after recording usage', + arrange: () => { + vi.mocked(inferRepositoryCandidates).mockResolvedValue( + answered([{ repositoryUrl: 'https://github.com/a/b', confidence: 0.9 }]), + ) + vi.mocked(githubApiGet).mockRejectedValue(new Error('HTTP 502')) + }, + expected: { + message: 'HTTP 502', + nonRetryable: false, + llmCalls: 1, + usageCalls: [[qx, 'job-1', usage]], + }, + }, + { + name: 'before calling the LLM when the GitHub App config is missing', + arrange: () => { + vi.mocked(getGithubAppConfig).mockImplementation(() => { + throw new Error('Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY') + }) + }, + expected: { + message: 'Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY', + nonRetryable: false, + llmCalls: 0, + usageCalls: [], + }, + }, + { + name: 'without retry and before calling the LLM when Bedrock credentials are missing', + arrange: () => { + vi.mocked(getBedrockCredentials).mockReturnValue(null) + }, + expected: { + message: 'Bedrock credentials are not configured', + nonRetryable: true, + llmCalls: 0, + usageCalls: [], + }, + }, + ])('fails $name', async ({ arrange, expected }) => { + arrange() + const error = (await resolveRepoViaLlm(input).catch((e: unknown) => e)) as Error & { + nonRetryable?: boolean + } + expect({ + message: error.message, + nonRetryable: error.nonRetryable === true, + llmCalls: vi.mocked(inferRepositoryCandidates).mock.calls.length, + usageCalls: recordUsage.mock.calls, + }).toEqual(expected) expect(recordRepo).not.toHaveBeenCalled() }) - - it('fails before calling the LLM when the GitHub App config is missing', async () => { - vi.mocked(getGithubAppConfig).mockImplementation(() => { - throw new Error('Missing required environment variable: CROWD_GITHUB_PRIVATE_KEY') - }) - await expect(resolveRepoViaLlm(input)).rejects.toThrow('CROWD_GITHUB_PRIVATE_KEY') - expect(inferRepositoryCandidates).not.toHaveBeenCalled() - expect(recordUsage).not.toHaveBeenCalled() - }) - - it('fails without retry when Bedrock credentials are missing', async () => { - vi.mocked(getBedrockCredentials).mockReturnValue(null) - const error = await resolveRepoViaLlm(input).catch((e: unknown) => e) - expect(error).toMatchObject({ - message: 'Bedrock credentials are not configured', - nonRetryable: true, - }) - expect(inferRepositoryCandidates).not.toHaveBeenCalled() - }) }) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts index 466faa8cac..eddcedb5e1 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/startInferredRepo.test.ts @@ -11,23 +11,28 @@ vi.mock('@crowd/data-access-layer/src/osspckgs/memberContributorJobs', () => ({ findMemberContributorJobPurl: vi.fn(), })) +const markRunning = vi.mocked(jobsDal.markMemberContributorJobRunning) +const findPurl = vi.mocked(jobsDal.findMemberContributorJobPurl) + beforeEach(() => vi.clearAllMocks()) describe('startInferredRepoMemberContributorsJob', () => { - it('marks running then returns purl and ecosystem', async () => { - vi.mocked(jobsDal.findMemberContributorJobPurl).mockResolvedValue({ - purl: 'pkg:hackage/aeson', - ecosystem: 'hackage', - }) + it('marks the job running before reading its purl and ecosystem', async () => { + findPurl.mockResolvedValue({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage' }) + await expect(startInferredRepoMemberContributorsJob({ jobId: 'job-1' })).resolves.toEqual({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage', }) - expect(jobsDal.markMemberContributorJobRunning).toHaveBeenCalledWith(qx, 'job-1') + expect(markRunning.mock.calls).toEqual([[qx, 'job-1']]) + expect(findPurl.mock.calls).toEqual([[qx, 'job-1']]) + expect(markRunning.mock.invocationCallOrder[0]).toBeLessThan( + findPurl.mock.invocationCallOrder[0], + ) }) it('fails without retry when the job row is missing', async () => { - vi.mocked(jobsDal.findMemberContributorJobPurl).mockResolvedValue(null) + findPurl.mockResolvedValue(null) const error = await startInferredRepoMemberContributorsJob({ jobId: 'job-1' }).catch( (e: unknown) => e, ) diff --git a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts index 81892a7c46..6cbb32d4e3 100644 --- a/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/identityAffiliations.test.ts @@ -317,13 +317,13 @@ describe('mergeIdentityAffiliationsStaging delete guard', () => { }) describe('findAffiliationsForIdentityKeys', () => { - it('returns nothing without a query for no keys', async () => { + it('returns no rows without querying for no keys', async () => { const { qx, calls } = fakeQx() await expect(findAffiliationsForIdentityKeys(qx, [])).resolves.toEqual([]) expect(calls).toHaveLength(0) }) - it('reads affiliated rows for the given keys in one query shaped like the sync rows', async () => { + it('reads only member-affiliated rows for the keys in one query, keyed by person', async () => { const row = { person: 'cdp-member-1', source: 'github', @@ -346,16 +346,16 @@ describe('findAffiliationsForIdentityKeys', () => { { identityType: 'github-login', identityValue: 'ada' }, ]) expect(rows).toEqual([row]) - expect(calls).toHaveLength(1) - expect(calls[0].query).toContain('FROM unnest($(types)::text[], $(values)::text[])') - expect(calls[0].query).toContain('JOIN identity_affiliations a') - expect(calls[0].query).toContain('WHERE a.akrites_member IS NOT NULL') + expect(calls).toEqual([ + { + method: 'select', + query: expect.stringContaining('WHERE a.akrites_member IS NOT NULL'), + params: { types: ['github-login', 'github-login'], values: ['octo', 'ada'] }, + inTx: false, + }, + ]) expect(calls[0].query).toContain( "COALESCE(a.cdp_member_id::text, a.identity_type || ':' || a.identity_value) AS person", ) - expect(calls[0].params).toEqual({ - types: ['github-login', 'github-login'], - values: ['octo', 'ada'], - }) }) }) diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts index 48a47ad48a..3b4fe152c6 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorJobs.test.ts @@ -45,6 +45,10 @@ function fakeQx(selectOneOrNoneRow: object | null = null) { return { qx: executor as unknown as QueryExecutor, calls } } +function singleLine(query: string): string { + return query.replace(/\s+/g, ' ') +} + const input = { id: '11111111-1111-4111-8111-111111111111', purl: 'pkg:npm/request-oauth', @@ -153,115 +157,116 @@ describe('status transitions', () => { }) }) -describe('findMemberContributorJobPurl', () => { - it('reads purl and ecosystem by id', async () => { +describe('inferred repo job reads', () => { + it('reads the purl and ecosystem of a job by id', async () => { const { qx, calls } = fakeQx({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage' }) await expect(findMemberContributorJobPurl(qx, 'job-1')).resolves.toEqual({ purl: 'pkg:hackage/aeson', ecosystem: 'hackage', }) - expect(calls[0].query).toContain('SELECT purl, ecosystem') - expect(calls[0].params).toEqual({ id: 'job-1' }) + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: 'SELECT purl, ecosystem FROM member_contributor_jobs WHERE id = $(id)', + params: { id: 'job-1' }, + }, + ]) }) -}) -describe('findLatestMemberContributorJobByPurl', () => { - it('returns id, status and collection path of the newest job', async () => { - const { qx, calls } = fakeQx({ - id: 'job-1', - status: 'done', - collectionPath: 'llm_repo_resolution', - }) - await expect(findLatestMemberContributorJobByPurl(qx, 'pkg:hackage/aeson')).resolves.toEqual({ - id: 'job-1', - status: 'done', - collectionPath: 'llm_repo_resolution', - }) - expect(calls[0].query).toContain('ORDER BY created_at DESC, id DESC') - expect(calls[0].query).toContain('LIMIT 1') + it('reads the newest job of a purl with its status and collection path', async () => { + const latest = { id: 'job-1', status: 'done', collectionPath: 'llm_repo_resolution' } + const { qx, calls } = fakeQx(latest) + await expect(findLatestMemberContributorJobByPurl(qx, 'pkg:hackage/aeson')).resolves.toEqual( + latest, + ) + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: expect.stringContaining('WHERE purl = $(purl)'), + params: { purl: 'pkg:hackage/aeson' }, + }, + ]) + expect(singleLine(calls[0].query)).toContain('ORDER BY created_at DESC, id DESC LIMIT 1') + }) + + it('selects the collection path and inferred repo columns of a job', async () => { + const { qx, calls } = fakeQx(null) + await findMemberContributorJob(qx, 'job-1') + expect(calls).toEqual([ + { + method: 'selectOneOrNone', + query: expect.stringContaining('repo_url AS "repoUrl", repo_url_source AS "repoUrlSource"'), + params: { id: 'job-1' }, + }, + ]) + expect(calls[0].query).toContain('collection_path AS "collectionPath"') }) }) -describe('recordMemberContributorJobInferredRepo', () => { - it('writes only repo url, source and confidence on a running job', async () => { +describe('inferred repo job writes', () => { + it('records the inferred repo on a running job without touching llm usage', async () => { const { qx, calls } = fakeQx() await recordMemberContributorJobInferredRepo(qx, 'job-1', { repoUrl: 'https://github.com/haskell/aeson', confidence: 0.92, }) - expect(calls[0].method).toBe('result') - expect(calls[0].query).toContain("repo_url_source = 'llm_inferred'") - expect(calls[0].query).toContain('llm_confidence = $(confidence)') - expect(calls[0].query).not.toContain('llm_input_tokens') - expect(calls[0].query).not.toContain('llm_output_tokens') - expect(calls[0].query).not.toContain('llm_cost_usd') - expect(calls[0].query).toContain("WHERE id = $(id) AND status = 'running'") - expect(calls[0].params).toEqual({ - id: 'job-1', - repoUrl: 'https://github.com/haskell/aeson', - confidence: 0.92, - }) + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status = 'running'"), + params: { id: 'job-1', repoUrl: 'https://github.com/haskell/aeson', confidence: 0.92 }, + }, + ]) + expect(calls[0].query).toContain( + "SET repo_url = $(repoUrl), repo_url_source = 'llm_inferred', llm_confidence = $(confidence),", + ) + expect(calls[0].query).not.toMatch(/llm_(input_tokens|output_tokens|cost_usd)/) }) -}) -describe('recordMemberContributorJobLlmUsage', () => { - it('adds the usage to the running totals of a running job', async () => { + it('accumulates llm usage on a running job, keeping the cost when it is unknown', async () => { const { qx, calls } = fakeQx() await recordMemberContributorJobLlmUsage(qx, 'job-1', { inputTokens: 300, outputTokens: 20, costUsd: null, }) - expect(calls[0].query).not.toContain('repo_url') - expect(calls[0].query).toContain( - 'llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens)', + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status = 'running'"), + params: { id: 'job-1', inputTokens: 300, outputTokens: 20, costUsd: null }, + }, + ]) + expect(singleLine(calls[0].query)).toContain( + 'SET llm_input_tokens = COALESCE(llm_input_tokens, 0) + $(inputTokens), llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens), llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END,', ) - expect(calls[0].query).toContain( - 'llm_output_tokens = COALESCE(llm_output_tokens, 0) + $(outputTokens)', - ) - expect(calls[0].query).toContain( - 'llm_cost_usd = CASE WHEN $(costUsd) IS NULL THEN llm_cost_usd ELSE COALESCE(llm_cost_usd, 0) + $(costUsd) END', - ) - expect(calls[0].query).toContain("status = 'running'") - expect(calls[0].params).toEqual({ - id: 'job-1', - inputTokens: 300, - outputTokens: 20, - costUsd: null, - }) + expect(calls[0].query).not.toContain('repo_url') }) -}) -describe('markMemberContributorJobUnresolvable', () => { - it('sets unresolvable with reason and repo url unless the job is done', async () => { + it('marks a job unresolvable unless it is done, keeping the known repo url', async () => { const { qx, calls } = fakeQx() await markMemberContributorJobUnresolvable(qx, 'job-1', 'LLM returned no repository', null) - expect(calls[0].query).toContain("status = 'unresolvable'") - expect(calls[0].query).toContain('repo_url = COALESCE($(repoUrl), repo_url)') - expect(calls[0].query).toContain("WHERE id = $(id) AND status <> 'done'") - expect(calls[0].params).toEqual({ - id: 'job-1', - reason: 'LLM returned no repository', - repoUrl: null, - }) + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining("WHERE id = $(id) AND status <> 'done'"), + params: { id: 'job-1', reason: 'LLM returned no repository', repoUrl: null }, + }, + ]) + expect(calls[0].query).toContain( + "SET status = 'unresolvable', error = $(reason), repo_url = COALESCE($(repoUrl), repo_url),", + ) }) -}) -describe('incrementMemberContributorJobSubmissions', () => { - it('increments submission_count', async () => { + it('increments the submission count of a job', async () => { const { qx, calls } = fakeQx() await incrementMemberContributorJobSubmissions(qx, 'job-1') - expect(calls[0].query).toContain('submission_count = submission_count + 1') - expect(calls[0].params).toEqual({ id: 'job-1' }) - }) -}) - -describe('findMemberContributorJob columns', () => { - it('selects the collection path and inferred repo columns', async () => { - const { qx, calls } = fakeQx(null) - await findMemberContributorJob(qx, 'job-1') - expect(calls[0].query).toContain('collection_path AS "collectionPath"') - expect(calls[0].query).toContain('repo_url AS "repoUrl"') - expect(calls[0].query).toContain('repo_url_source AS "repoUrlSource"') + expect(calls).toEqual([ + { + method: 'result', + query: expect.stringContaining('SET submission_count = submission_count + 1'), + params: { id: 'job-1' }, + }, + ]) }) }) diff --git a/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts index a5e90c651d..ca58f6b18c 100644 --- a/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts +++ b/services/libs/data-access-layer/src/osspckgs/memberContributorsBody.test.ts @@ -434,7 +434,7 @@ describe('toAkritesExternalMemberContributors', () => { }) describe('toInferredRepoMemberContributorsBody', () => { - it('builds the sync body with llm provenance, null mapping confidence and empty maintainers', () => { + it('builds the sync body with llm provenance, null mapping confidence and no maintainers', () => { const entry = { githubLogin: 'octo', displayName: null, @@ -455,7 +455,13 @@ describe('toInferredRepoMemberContributorsBody', () => { ecosystem: 'hackage', repoUrl: 'https://github.com/haskell/aeson', }, - { contributors: [entry], registry: [], governanceFiles: [] }, + { + contributors: [entry], + registry: [ + { ...entry, roles: [{ role: 'maintainer', roleKind: 'maintainer', current: true }] }, + ], + governanceFiles: [], + }, ) expect(body).toEqual({ purl: 'pkg:hackage/aeson', From 1c0d1a4d958b86c49abff8407d4f761fc8a77cec Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:46:34 +0000 Subject: [PATCH 20/23] fix: reject purls with malformed percent encoding before llm repo resolution Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../packages/memberContributorCollectionPaths.test.ts | 5 ++++- .../v1/packages/memberContributorCollectionPaths.ts | 11 ++++++++++- .../v1/packages/submitMemberContributorsJob.test.ts | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts index c91e1abdb5..194599eb9a 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.test.ts @@ -42,11 +42,14 @@ describe('isWellFormedPurl', () => { ['pkg:hackage/aeson', true], ['pkg:conan/openssl.org/openssl', true], ['pkg:Hackage/aeson', true], + ['pkg:npm/%40babel/core', true], ['pkg:', false], ['pkg:hackage', false], ['pkg:hackage/', false], ['pkg:/x', false], - ])('requires a type and a name segment: %s is %s', (purl, wellFormed) => { + ['pkg:hackage/aeson%ZZ', false], + ['pkg:hackage/aeson%FF', false], + ])('requires a type, a name segment and valid percent encoding: %s is %s', (purl, wellFormed) => { expect(isWellFormedPurl(purl)).toBe(wellFormed) }) }) diff --git a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts index 179e8edfdb..0ec826d758 100644 --- a/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts +++ b/backend/src/api/public/v1/packages/memberContributorCollectionPaths.ts @@ -21,8 +21,17 @@ export function purlType(purl: string): string { const WELL_FORMED_PURL = /^pkg:[a-z][a-z0-9.+-]*\/[^/?#]+/i +function isDecodable(value: string): boolean { + try { + decodeURIComponent(value) + return true + } catch { + return false + } +} + export function isWellFormedPurl(purl: string): boolean { - return WELL_FORMED_PURL.test(purl) + return WELL_FORMED_PURL.test(purl) && isDecodable(purl.slice('pkg:'.length)) } export const LLM_REPO_RESOLUTION_PATH: MemberContributorCollectionPath = 'llm_repo_resolution' diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index b67304bdf1..1cd23bb60d 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -418,7 +418,7 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { }, ) - it.each(['pkg:', 'pkg:hackage', 'pkg:/x'])( + it.each(['pkg:', 'pkg:hackage', 'pkg:/x', 'pkg:hackage/aeson%ZZ'])( 'rejects the malformed purl %s with 400 before any database call', async (purl) => { const { req, res } = mockReqRes({ purl }) From b015757c5bca61cf750af72559237c30d623043e Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:47:01 +0000 Subject: [PATCH 21/23] fix: treat a done job whose workflow is still closing as a readable result Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../submitMemberContributorsJob.test.ts | 47 ++++++++++++------- .../packages/submitMemberContributorsJob.ts | 8 ++-- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts index 1cd23bb60d..b893143258 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.test.ts @@ -449,28 +449,43 @@ describe('submitMemberContributorsJob, unknown ecosystem', () => { expect(json).toHaveBeenCalledWith({ jobId: 'active-1', status: 'pending' }) }) - it('refuses with 409 carrying the previous jobId while its workflow result is still readable', async () => { - latestJob('done-1', 'done') - describeWorkflow.mockResolvedValue({ status: { name: 'COMPLETED' } }) - const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) + it.each(['COMPLETED', 'RUNNING'])( + 'refuses with 409 carrying the previous done jobId while its workflow is %s', + async (workflowStatus) => { + latestJob('done-1', 'done') + describeWorkflow.mockResolvedValue({ status: { name: workflowStatus } }) + const { req, res } = mockReqRes({ purl: 'pkg:hackage/aeson' }) - const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) + const error = await submitMemberContributorsJob(req, res).catch((e: unknown) => e) - expect(error).toBeInstanceOf(ConflictError) - expect((error as ConflictError).context).toEqual({ jobId: 'done-1' }) - expect(getHandle.mock.calls).toEqual([['member-contributors-job/done-1']]) - expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'done-1']]) - expect(createMemberContributorJob).not.toHaveBeenCalled() - }) + expect(error).toBeInstanceOf(ConflictError) + expect((error as ConflictError).context).toEqual({ jobId: 'done-1' }) + expect(getHandle.mock.calls).toEqual([['member-contributors-job/done-1']]) + expect(incrementMemberContributorJobSubmissions.mock.calls).toEqual([[{}, 'done-1']]) + expect(createMemberContributorJob).not.toHaveBeenCalled() + }, + ) it.each([ - { latestStatus: 'done' as const, describedWorkflows: [['member-contributors-job/old-1']] }, - { latestStatus: 'failed' as const, describedWorkflows: [] }, - { latestStatus: 'unresolvable' as const, describedWorkflows: [] }, + { + latestStatus: 'done' as const, + workflowStatus: 'not found', + describedWorkflows: [['member-contributors-job/old-1']], + }, + { + latestStatus: 'done' as const, + workflowStatus: 'FAILED', + describedWorkflows: [['member-contributors-job/old-1']], + }, + { latestStatus: 'failed' as const, workflowStatus: 'not found', describedWorkflows: [] }, + { latestStatus: 'unresolvable' as const, workflowStatus: 'not found', describedWorkflows: [] }, ])( - 'accepts a new job when the latest job is $latestStatus and its result is not readable', - async ({ latestStatus, describedWorkflows }) => { + 'accepts a new job when the latest job is $latestStatus with its workflow $workflowStatus', + async ({ latestStatus, workflowStatus, describedWorkflows }) => { latestJob('old-1', latestStatus) + if (workflowStatus !== 'not found') { + describeWorkflow.mockResolvedValue({ status: { name: workflowStatus } }) + } const { req, res, status } = mockReqRes({ purl: 'pkg:hackage/aeson' }) await submitMemberContributorsJob(req, res) diff --git a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts index e5101136c5..d937b630c5 100644 --- a/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/submitMemberContributorsJob.ts @@ -135,13 +135,15 @@ async function startJobWorkflow( } } -async function workflowResultAvailable(jobId: string): Promise { +const READABLE_DONE_JOB_WORKFLOW_STATUSES = new Set(['COMPLETED', 'RUNNING']) + +async function doneJobWorkflowResultReadable(jobId: string): Promise { const packagesTemporal = await getPackagesTemporalClient() try { const description = await packagesTemporal.workflow .getHandle(memberContributorsJobWorkflowId(jobId)) .describe() - return description.status.name === 'COMPLETED' + return READABLE_DONE_JOB_WORKFLOW_STATUSES.has(description.status.name) } catch (err) { if (err instanceof WorkflowNotFoundError) { return false @@ -155,7 +157,7 @@ async function assertInferredRepoResultNotReadable(qx: QueryExecutor, purl: stri if (!latest || latest.status !== 'done' || latest.collectionPath !== LLM_REPO_RESOLUTION_PATH) { return } - if (!(await workflowResultAvailable(latest.id))) { + if (!(await doneJobWorkflowResultReadable(latest.id))) { return } await incrementMemberContributorJobSubmissions(qx, latest.id) From c8c660fdefaa68964537e9018fd36bc7562905a8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 12:48:10 +0000 Subject: [PATCH 22/23] fix: persist the usage of truncated llm attempts before retrying Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../job/__tests__/llmRepoInference.test.ts | 28 +++++++++++-------- .../job/__tests__/resolveRepoViaLlm.test.ts | 17 +++++++++++ .../job/llmRepoInference.ts | 5 ++-- .../job/resolveRepoViaLlm.ts | 4 +++ 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts index cc21e7de93..39fac38405 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/llmRepoInference.test.ts @@ -107,21 +107,25 @@ describe('inferRepositoryCandidates', () => { bedrockResponse(null, 'refusal'), { status: 'unparseable', reason: 'LLM refused to answer', usage }, ], - ])('returns the result with usage for %s', async (_label, response, expected) => { + [ + 'an answer truncated at max_tokens', + bedrockResponse('{"candidates": [', 'max_tokens'), + { status: 'truncated', usage }, + ], + ])('returns and logs the result with usage for %s', async (_label, response, expected) => { await expect( inferRepositoryCandidates('pkg:conan/x', invokeReturning(response)), ).resolves.toEqual(expected) - }) - - it('rejects a truncated answer after logging its usage so the activity retries', async () => { - const invoke = invokeReturning(bedrockResponse('{"candidates": [', 'max_tokens')) - await expect(inferRepositoryCandidates('pkg:conan/x', invoke)).rejects.toThrow( - 'LLM answer truncated at max_tokens', - ) - expect(log.info).toHaveBeenCalledWith( - expect.objectContaining({ stopReason: 'max_tokens', inputTokens: 300, outputTokens: 80 }), - 'LLM repo inference call', - ) + expect(log.info.mock.calls).toEqual([ + [ + expect.objectContaining({ + stopReason: JSON.parse(response).stop_reason, + inputTokens: 300, + outputTokens: 80, + }), + 'LLM repo inference call', + ], + ]) }) it.each([ diff --git a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts index c765759af7..1d54133889 100644 --- a/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts +++ b/services/apps/packages_worker/src/member-contributors/job/__tests__/resolveRepoViaLlm.test.ts @@ -211,6 +211,20 @@ describe('resolveRepoViaLlm', () => { nonRetryable: false, llmCalls: 1, usageCalls: [[qx, 'job-1', usage]], + githubCalls: 1, + }, + }, + { + name: 'retryably when the LLM answer is truncated, after recording usage', + arrange: () => { + vi.mocked(inferRepositoryCandidates).mockResolvedValue({ status: 'truncated', usage }) + }, + expected: { + message: 'LLM answer truncated at max_tokens', + nonRetryable: false, + llmCalls: 1, + usageCalls: [[qx, 'job-1', usage]], + githubCalls: 0, }, }, { @@ -225,6 +239,7 @@ describe('resolveRepoViaLlm', () => { nonRetryable: false, llmCalls: 0, usageCalls: [], + githubCalls: 0, }, }, { @@ -237,6 +252,7 @@ describe('resolveRepoViaLlm', () => { nonRetryable: true, llmCalls: 0, usageCalls: [], + githubCalls: 0, }, }, ])('fails $name', async ({ arrange, expected }) => { @@ -249,6 +265,7 @@ describe('resolveRepoViaLlm', () => { nonRetryable: error.nonRetryable === true, llmCalls: vi.mocked(inferRepositoryCandidates).mock.calls.length, usageCalls: recordUsage.mock.calls, + githubCalls: vi.mocked(githubApiGet).mock.calls.length, }).toEqual(expected) expect(recordRepo).not.toHaveBeenCalled() }) diff --git a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts index f04a713e4d..ebe4ba635c 100644 --- a/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts +++ b/services/apps/packages_worker/src/member-contributors/job/llmRepoInference.ts @@ -12,7 +12,7 @@ const log = getServiceChildLogger('member-contributors:llm-repo-inference') export const REPO_INFERENCE_MODEL_ID = LlmModelType.CLAUDE_SONNET_5_5 export const MAX_REPO_CANDIDATES = 3 const MAX_TOKENS = 2048 -const TRUNCATED_ANSWER = 'LLM answer truncated at max_tokens' +export const TRUNCATED_ANSWER = 'LLM answer truncated at max_tokens' const candidateSchema = z.object({ repositoryUrl: z.string().min(1), @@ -63,6 +63,7 @@ export interface RepoCandidate { export type RepoInference = | { status: 'answered'; candidates: RepoCandidate[]; usage: LlmUsage } | { status: 'unparseable'; reason: string; usage: LlmUsage } + | { status: 'truncated'; usage: LlmUsage } interface BedrockMessageResponse { content?: Array<{ type?: string; text?: string }> @@ -149,7 +150,7 @@ export async function inferRepositoryCandidates( 'LLM repo inference call', ) if (candidates === TRUNCATED_ANSWER) { - throw new Error(TRUNCATED_ANSWER) + return { status: 'truncated', usage } } if (typeof candidates === 'string') { return { status: 'unparseable', reason: candidates, usage } diff --git a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts index a3e04b4438..e3405e58b2 100644 --- a/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts +++ b/services/apps/packages_worker/src/member-contributors/job/resolveRepoViaLlm.ts @@ -14,6 +14,7 @@ import { canonicalGithubUrl } from './canonicalGithubUrl' import { REPO_INFERENCE_MODEL_ID, RepoCandidate, + TRUNCATED_ANSWER, inferRepositoryCandidates, } from './llmRepoInference' @@ -88,6 +89,9 @@ export async function resolveRepoViaLlm( const pkgsQx = await getPackagesDb() await recordMemberContributorJobLlmUsage(pkgsQx, input.jobId, inference.usage) + if (inference.status === 'truncated') { + throw new Error(TRUNCATED_ANSWER) + } if (inference.status === 'unparseable') { return unresolvable(inference.reason) } From 5264d1fc6d23a27391176251b0148d6dcc3f1953 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 16 Oct 2026 13:18:26 +0000 Subject: [PATCH 23/23] fix: answer failed instead of 500 when a done job's workflow ended abnormally Refs linuxfoundation/lfx-cdp-insights-ops#502 Signed-off-by: Mouad BANI --- .../packages/getMemberContributorsJob.test.ts | 22 +++++++++++++++++-- .../v1/packages/getMemberContributorsJob.ts | 4 ++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts index e3300fead2..732aaf7722 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.test.ts @@ -8,7 +8,7 @@ import type { MemberContributorsPackageRow, } from '@crowd/data-access-layer' import { canonicalCdpOrganizationIdByMemberName } from '@crowd/data-access-layer/src/osspckgs/akritesMembers' -import { WorkflowNotFoundError } from '@crowd/temporal' +import { WorkflowFailedError, WorkflowNotFoundError } from '@crowd/temporal' import { getMemberContributorsJob } from './getMemberContributorsJob' @@ -33,7 +33,13 @@ vi.mock('@crowd/temporal', () => { class MockWorkflowNotFoundError extends Error { name = 'WorkflowNotFoundError' } - return { WorkflowNotFoundError: MockWorkflowNotFoundError } + class MockWorkflowFailedError extends Error { + name = 'WorkflowFailedError' + } + return { + WorkflowNotFoundError: MockWorkflowNotFoundError, + WorkflowFailedError: MockWorkflowFailedError, + } }) vi.mock('@/db/packagesDb', () => ({ getPackagesQx: vi.fn().mockResolvedValue({}) })) vi.mock('@crowd/data-access-layer', async (importOriginal) => ({ @@ -242,6 +248,18 @@ describe('getMemberContributorsJob, llm repo resolution path', () => { workflowIds: [[workflowId]], packageReads: [], }, + { + name: 'reports failed with the resubmit reason when the workflow ended abnormally', + arrange: () => { + findMemberContributorJob.mockResolvedValue(llmJob({ status: 'done' })) + result.mockRejectedValue( + new WorkflowFailedError('terminated', undefined, 'NON_RETRYABLE_FAILURE'), + ) + }, + response: { jobId: JOB_ID, status: 'failed', reason: 'Job result expired, resubmit the job' }, + workflowIds: [[workflowId]], + packageReads: [], + }, { name: 'keeps reading github path done jobs from the tables, not the workflow', arrange: () => { diff --git a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts index 3bd8ced613..87f9681751 100644 --- a/backend/src/api/public/v1/packages/getMemberContributorsJob.ts +++ b/backend/src/api/public/v1/packages/getMemberContributorsJob.ts @@ -17,7 +17,7 @@ import { toAkritesExternalMemberContributors, } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' -import { WorkflowNotFoundError } from '@crowd/temporal' +import { WorkflowFailedError, WorkflowNotFoundError } from '@crowd/temporal' import { AkritesExternalMemberContributorsJob, @@ -65,7 +65,7 @@ async function readInferredRepoBody( ) .result() } catch (err) { - if (err instanceof WorkflowNotFoundError) { + if (err instanceof WorkflowNotFoundError || err instanceof WorkflowFailedError) { return null } throw err