From ad947f82aea5c2ac60927c5e4c6adfbdc8019dfe Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 19:59:39 -0700 Subject: [PATCH] chore(tests): remove quarantined integration suites The seven suites excluded from the integration run since 2026-09-25 failed against a freshly provisioned database (fixtures predate connector and credential changes) and never ran in CI. Delete them, the quarantine list in vitest.config.ts, their explicit-any baseline entries, and getCredentialGroupOAuthContext (plus its @sim/testing mock), whose only caller was the GitHub member suite. The testing rule now says to fix or delete a failing suite rather than exclude it. --- .claude/rules/sim-testing.md | 4 +- .cursor/rules/sim-testing.mdc | 4 +- apps/sim/lib/credential-groups/enrollments.ts | 14 - ...rganization-personal-tokens.integration.ts | 371 ----- .../confluence-enrollment.integration.ts | 332 ---- .../excluded-member-documents.integration.ts | 250 --- .../github-member.integration.ts | 1367 ----------------- .../gmail-member.integration.ts | 646 -------- .../google-calendar-member.integration.ts | 650 -------- .../jira-member.integration.ts | 655 -------- apps/sim/vitest.config.ts | 20 +- .../credential-groups-enrollments.mock.ts | 2 - scripts/check-explicit-any.baseline.json | 6 - 13 files changed, 5 insertions(+), 4316 deletions(-) delete mode 100644 apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/confluence-enrollment.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/github-member.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/jira-member.integration.ts diff --git a/.claude/rules/sim-testing.md b/.claude/rules/sim-testing.md index 5e84f540ab9..96553945475 100644 --- a/.claude/rules/sim-testing.md +++ b/.claude/rules/sim-testing.md @@ -48,8 +48,8 @@ contracts, and demonstrated regressions. those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in - `vitest.shared.ts`, which CI uploads. Never add a passing suite to the quarantine list in - `apps/sim/vitest.config.ts`. + `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in + `apps/sim/vitest.config.ts`: fix it or delete it. - `bun run test:integration` starts disposable Postgres and Redis containers, provisions the schema, and runs both workspaces; pass filenames to narrow the `apps/sim` run. diff --git a/.cursor/rules/sim-testing.mdc b/.cursor/rules/sim-testing.mdc index 6ef5cf6e8b5..d71779a2cf0 100644 --- a/.cursor/rules/sim-testing.mdc +++ b/.cursor/rules/sim-testing.mdc @@ -46,8 +46,8 @@ contracts, and demonstrated regressions. those checks. Isolate with a unique schema or generated IDs, and clean up in `afterAll`. - Integration files run one at a time against one shared database. A new `*.integration.ts` is picked up by CI with no workflow change, and the run writes the JSON report configured in - `vitest.shared.ts`, which CI uploads. Never add a passing suite to the quarantine list in - `apps/sim/vitest.config.ts`. + `vitest.shared.ts`, which CI uploads. Never exclude a failing suite from the run in + `apps/sim/vitest.config.ts`: fix it or delete it. - `bun run test:integration` starts disposable Postgres and Redis containers, provisions the schema, and runs both workspaces; pass filenames to narrow the `apps/sim` run. diff --git a/apps/sim/lib/credential-groups/enrollments.ts b/apps/sim/lib/credential-groups/enrollments.ts index 2d1299075aa..5618cdce568 100644 --- a/apps/sim/lib/credential-groups/enrollments.ts +++ b/apps/sim/lib/credential-groups/enrollments.ts @@ -1348,20 +1348,6 @@ async function completeResolvedCredentialGroupEnrollment( }) } -/** Resolves the private, server-only context bound to a public enrollment link and option. */ -export async function getCredentialGroupOAuthContext( - token: string, - optionId: string -): Promise { - const row = await resolvePublicEnrollmentRowByIdentity({ - invitationTokenHash: hashInvitationToken(token), - }) - if (!row) return null - const option = row.options.find((candidate) => candidate.id === optionId) - if (!option || option.status !== 'active') return null - return credentialGroupOAuthContextFromRow(row, option) -} - export async function getAuthorizedCredentialGroupOAuthContext( identity: PublicCredentialGroupEnrollmentIdentity, optionId: string diff --git a/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts b/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts deleted file mode 100644 index 1287fbbd393..00000000000 --- a/apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts +++ /dev/null @@ -1,371 +0,0 @@ -/** Real storage, encryption, migration, and authorization; no external GitLab calls. */ -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - member, - organization, - permissions, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { sha256Hex } from '@sim/security/hash' -import { generateId } from '@sim/utils/id' -import { and, eq, inArray } from 'drizzle-orm' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { buildOrganizationAccountAccessPolicy } from '@/lib/credential-groups/application/workspace-access-policy' -import { updateWorkspaceCredentialUseCase } from '@/lib/credentials/application/credential-crud' -import { resolvePersonalToken } from '@/lib/credentials/application/resolve-personal-token' -import { deleteCredentialUseCase } from '@/lib/credentials/application/service-account' -import { decryptPersonalToken, encryptPersonalToken } from '@/lib/credentials/gitlab-personal-token' -import { - createPersonalTokenCredential, - getPersonalTokenCredentials, - updatePersonalTokenCredential, -} from '@/lib/credentials/personal-tokens' -import { listVisibleWorkspaceCredentials } from '@/lib/credentials/queries' -import { migrateGitLabPersonalTokens } from '@/scripts/migrate-gitlab-personal-tokens' - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - secureFetchWithValidation: async (url: string) => - Response.json( - url.endsWith('/user') - ? { id: 42, username: 'fixture', name: 'Fixture', state: 'active', bot: false } - : { user_id: 42, active: true, revoked: false, scopes: ['api'], expires_at: null } - ), -})) - -describe('organization personal tokens', () => { - let ids: ReturnType - const tokenSecret = 'isolated-gitlab-token-fixture' - function fixtureIds() { - return { - owner: generateId(), - other: generateId(), - org: generateId(), - foreignOrg: generateId(), - first: generateId(), - second: generateId(), - foreign: generateId(), - group: generateId(), - legacyGroup: generateId(), - enrollment: generateId(), - token: generateId(), - } - } - const identity = () => ({ - providerId: 'gitlab' as const, - ownerUserId: ids.owner, - subjectId: '42', - instanceUrl: 'https://gitlab.example.test', - }) - const principal = (userId = ids.owner) => ({ - kind: 'session' as const, - userId, - sessionId: 'isolated-session', - }) - const resolve = (workspaceId = ids.second, userId = ids.owner) => - resolvePersonalToken.execute({ - principal: principal(userId), - input: { - credentialId: ids.token, - assertedWorkspaceId: workspaceId, - expectedProviderId: 'gitlab', - }, - }) - const migrate = (apply = true) => migrateGitLabPersonalTokens({ organizationId: ids.org, apply }) - async function storedToken() { - const [row] = await db.select().from(credential).where(eq(credential.id, ids.token)) - if (!row) throw new Error('Fixture token missing') - return row - } - - beforeEach(async () => { - ids = fixtureIds() - const now = new Date() - await db.insert(user).values( - [ids.owner, ids.other].map((id) => ({ - id, - name: 'Token fixture', - email: `${id}@fixture.test`, - emailVerified: true, - createdAt: now, - updatedAt: now, - })) - ) - await db.insert(organization).values( - [ids.org, ids.foreignOrg].map((id) => ({ - id, - name: 'Token fixture organization', - slug: id, - })) - ) - await db.insert(member).values([ - { id: generateId(), organizationId: ids.org, userId: ids.owner, role: 'member' }, - { id: generateId(), organizationId: ids.org, userId: ids.other, role: 'admin' }, - ]) - await insertWorkspaceFixture( - db, - [ids.first, ids.second, ids.foreign].map((id) => ({ - id, - name: 'Token fixture workspace', - organizationId: id === ids.foreign ? ids.foreignOrg : ids.org, - ownerId: ids.owner, - billedAccountUserId: ids.owner, - })) - ) - await db.insert(permissions).values( - [ids.first, ids.second, ids.foreign].flatMap((id) => - [ids.owner, ids.other].map((userId) => ({ - id: generateId(), - userId, - entityId: id, - entityType: 'workspace' as const, - permissionType: 'admin' as const, - })) - ) - ) - await db.insert(credentialGroup).values([ - { - id: ids.group, - organizationId: ids.org, - publicId: generateId(), - name: 'Connected accounts', - options: [], - createdBy: ids.owner, - }, - { - id: ids.legacyGroup, - workspaceId: ids.first, - publicId: generateId(), - name: 'Legacy accounts', - options: [], - createdBy: ids.owner, - }, - ]) - await db.insert(resourcePolicy).values({ - id: generateId(), - organizationId: ids.org, - resourceType: 'credential_group', - resourceId: ids.group, - document: buildOrganizationAccountAccessPolicy(ids.group, []), - createdBy: ids.owner, - }) - await db.insert(credentialGroupEnrollment).values({ - id: ids.enrollment, - credentialGroupId: ids.legacyGroup, - userId: ids.owner, - email: `${ids.owner}@fixture.test`, - status: 'completed', - invitationTokenHash: sha256Hex(generateId()), - invitationExpiresAt: now, - invitedAt: now, - }) - await db.insert(credential).values({ - id: ids.token, - workspaceId: ids.first, - type: 'personal_token', - providerId: 'gitlab', - displayName: 'Personal GitLab', - createdBy: ids.owner, - providerSubjectId: '42', - providerTenantId: identity().instanceUrl, - grantedScopes: ['api'], - credentialGroupEnrollmentId: ids.enrollment, - encryptedPersonalToken: await encryptPersonalToken({ - ...identity(), - workspaceId: ids.first, - accessToken: tokenSecret, - }), - }) - }) - afterEach(async () => { - await db - .delete(permissions) - .where(inArray(permissions.entityId, [ids.first, ids.second, ids.foreign])) - await db - .delete(credentialGroup) - .where(inArray(credentialGroup.id, [ids.group, ids.legacyGroup])) - await deleteWorkspaceFixture(db, inArray(workspace.id, [ids.first, ids.second, ids.foreign])) - await db.delete(organization).where(inArray(organization.id, [ids.org, ids.foreignOrg])) - await db.delete(user).where(inArray(user.id, [ids.owner, ids.other])) - }) - - it('dry-runs without changing ownership, ciphertext, or enrollments', async () => { - const before = await storedToken() - expect(await migrate(false)).toEqual({ mode: 'dry-run', processed: 1 }) - expect(await storedToken()).toEqual(before) - expect( - await db - .select() - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, ids.group)) - ).toHaveLength(0) - }) - - it('creates one organization token and reconnects and rotates the same identity', async () => { - await db.delete(credential).where(eq(credential.id, ids.token)) - const input = { - userId: ids.owner, - accounts: { organizationId: ids.org, credentialGroupId: ids.group }, - providerId: 'gitlab', - apiToken: tokenSecret, - domain: 'gitlab.example.test', - } - const created = await createPersonalTokenCredential(input) - expect(created.credential).toMatchObject({ - workspaceId: null, - organizationId: ids.org, - createdBy: ids.owner, - }) - const reconnected = await createPersonalTokenCredential({ - ...input, - apiToken: 'reconnected-fixture', - }) - expect(reconnected).toMatchObject({ created: false, credential: { id: created.credential.id } }) - await updatePersonalTokenCredential({ - credential: reconnected.credential, - apiToken: 'rotated-fixture', - }) - const [rotated] = await db - .select() - .from(credential) - .where(eq(credential.id, created.credential.id)) - expect(rotated).toMatchObject({ workspaceId: null, organizationId: ids.org }) - await expect( - decryptPersonalToken(rotated!.encryptedPersonalToken!, { - ...identity(), - organizationId: ids.org, - }) - ).resolves.toBe('rotated-fixture') - }) - - it('preserves the credential ID, rebinds encryption, and is idempotent', async () => { - expect(await migrate()).toEqual({ mode: 'applied', processed: 1 }) - const current = await storedToken() - expect(current).toMatchObject({ - id: ids.token, - workspaceId: null, - organizationId: ids.org, - createdBy: ids.owner, - }) - const [enrollment] = await db - .select() - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.id, current.credentialGroupEnrollmentId!)) - expect(enrollment).toMatchObject({ userId: ids.owner, credentialGroupId: ids.group }) - await expect( - decryptPersonalToken(current.encryptedPersonalToken!, { - ...identity(), - organizationId: ids.org, - }) - ).resolves.toBe(tokenSecret) - await expect( - decryptPersonalToken(current.encryptedPersonalToken!, { - ...identity(), - workspaceId: ids.first, - }) - ).rejects.toThrow('binding') - expect(await migrate()).toEqual({ mode: 'applied', processed: 0 }) - }) - - it('uses the same connection across workspaces and survives deletion of the original workspace', async () => { - await migrate() - for (const workspaceId of [ids.first, ids.second]) { - expect( - (await getPersonalTokenCredentials(workspaceId, ids.owner)).map((row) => row.id) - ).toEqual([ids.token]) - await expect(resolve(workspaceId)).resolves.toMatchObject({ accessToken: tokenSecret }) - const listed = await listVisibleWorkspaceCredentials({ - workspaceId, - userId: ids.owner, - workspaceAccess: { canAdmin: false }, - types: ['personal_token'], - }) - expect(listed.data).toEqual([ - expect.objectContaining({ id: ids.token, workspaceId: null, organizationId: ids.org }), - ]) - } - await deleteWorkspaceFixture(db, eq(workspace.id, ids.first)) - await expect(resolve()).resolves.toMatchObject({ accessToken: tokenSecret }) - }) - - it('denies another person, another organization, and removed organization membership', async () => { - await migrate() - expect(await getPersonalTokenCredentials(ids.second, ids.other)).toEqual([]) - expect(await getPersonalTokenCredentials(ids.foreign, ids.owner)).toEqual([]) - await expect(resolve(ids.second, ids.other)).rejects.toThrow() - await expect(resolve(ids.foreign)).rejects.toThrow() - await db - .delete(member) - .where(and(eq(member.organizationId, ids.org), eq(member.userId, ids.owner))) - expect(await getPersonalTokenCredentials(ids.second, ids.owner)).toEqual([]) - await expect(resolve()).rejects.toThrow() - }) - - it('rechecks revocation before use and manages the token from another workspace', async () => { - await migrate() - await updateWorkspaceCredentialUseCase.execute({ - principal: principal(), - input: { - credentialId: ids.token, - assertedWorkspaceId: ids.second, - displayName: 'Renamed GitLab', - }, - }) - expect((await storedToken()).displayName).toBe('Renamed GitLab') - await db - .update(credentialGroupEnrollment) - .set({ revokedAt: new Date(), status: 'revoked' }) - .where(eq(credentialGroupEnrollment.credentialGroupId, ids.group)) - await expect(resolve()).rejects.toThrow() - await deleteCredentialUseCase.execute({ - principal: principal(), - input: { credentialId: ids.token, workspaceId: ids.second }, - }) - expect(await db.select().from(credential).where(eq(credential.id, ids.token))).toHaveLength(0) - }) - - it('refuses duplicate identities without choosing or overwriting a token', async () => { - const original = await storedToken() - await db.insert(credential).values({ - ...original, - id: generateId(), - workspaceId: ids.second, - encryptedPersonalToken: await encryptPersonalToken({ - ...identity(), - workspaceId: ids.second, - accessToken: 'second-fixture-token', - }), - }) - await expect(migrate()).rejects.toThrow('Duplicate GitLab identities') - expect(await storedToken()).toEqual(original) - }) - - it('refuses ciphertext bound to a different workspace without changing the row', async () => { - await db - .update(credential) - .set({ - encryptedPersonalToken: await encryptPersonalToken({ - ...identity(), - workspaceId: ids.second, - accessToken: tokenSecret, - }), - }) - .where(eq(credential.id, ids.token)) - await expect(migrate()).rejects.toThrow('binding') - expect((await storedToken()).workspaceId).toBe(ids.first) - }) - - it('does not revive a revoked source enrollment', async () => { - await db - .update(credentialGroupEnrollment) - .set({ status: 'revoked', revokedAt: new Date() }) - .where(eq(credentialGroupEnrollment.id, ids.enrollment)) - await expect(migrate()).rejects.toThrow('inactive or mismatched') - expect((await storedToken()).workspaceId).toBe(ids.first) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/confluence-enrollment.integration.ts b/apps/sim/lib/knowledge/__integration__/confluence-enrollment.integration.ts deleted file mode 100644 index d60d0e07bb8..00000000000 --- a/apps/sim/lib/knowledge/__integration__/confluence-enrollment.integration.ts +++ /dev/null @@ -1,332 +0,0 @@ -/** Real application authorization and invitation persistence; no external provider calls. */ -import { createHash } from 'node:crypto' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - type CredentialGroupOptionConfig, - credential, - credentialGroup, - credentialGroupEnrollment, - knowledgeBase, - knowledgeConnector, - knowledgeConnectorMember, - permissions, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { generateId } from '@sim/utils/id' -import { and, eq } from 'drizzle-orm' -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { env } from '@/lib/core/config/env' -import { listCredentialGroupEnrollments } from '@/lib/credential-groups/enrollments' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' -import { resolveViewerConnectorMemberships } from '@/lib/knowledge/connectors/member-provisioning' - -describe('Confluence mirrored-identity self-enrollment', () => { - const previousClient = { id: env.CONFLUENCE_CLIENT_ID, secret: env.CONFLUENCE_CLIENT_SECRET } - const ids = createKnowledgeAclFixtureIds() - const foreign = createKnowledgeAclFixtureIds() - const groupId = generateId() - const option: CredentialGroupOptionConfig = { - id: generateId(), - provider: 'confluence', - label: 'Confluence', - required: false, - status: 'active', - authorizationAppId: 'fixture-confluence-app', - requiredScopes: ['read:confluence-content.all'], - scopeVersion: 1, - } - const bob: Principal = { - kind: 'session', - userId: ids.bobId, - sessionId: 'fixture-self-enrollment', - } - const source = { knowledgeBaseId: ids.knowledgeBaseId, connectorId: ids.connectorId } - - beforeAll(async () => { - Object.assign(env, { - CONFLUENCE_CLIENT_ID: 'isolated-confluence-fixture-client', - CONFLUENCE_CLIENT_SECRET: 'isolated-confluence-fixture-secret', - }) - await seedKnowledgeAclFixture(ids) - await seedKnowledgeAclFixture(foreign) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true, name: 'Renamed search index' }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - await db.insert(credentialGroup).values({ - id: groupId, - workspaceId: ids.workspaceId, - publicId: generateId(), - name: 'Connected accounts', - options: [option], - createdBy: ids.aliceId, - }) - }) - - beforeEach(async () => { - await db - .delete(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, groupId)) - await db - .update(credentialGroup) - .set({ options: [option], status: 'active' }) - .where(eq(credentialGroup.id, groupId)) - await db.update(user).set({ emailVerified: true }).where(eq(user.id, ids.bobId)) - await db - .update(knowledgeConnector) - .set({ - connectorType: 'confluence', - accessMode: 'admin', - status: 'paused', - sourceConfig: { domain: 'fixture.atlassian.net', spaceId: 'fixture-space' }, - syncLockToken: null, - credentialGroupId: null, - credentialGroupOptionId: null, - }) - .where(eq(knowledgeConnector.id, ids.connectorId)) - }) - - afterAll(async () => { - Object.assign(env, { - CONFLUENCE_CLIENT_ID: previousClient.id, - CONFLUENCE_CLIENT_SECRET: previousClient.secret, - }) - for (const fixture of [ids, foreign]) { - await deleteWorkspaceFixture(db, eq(workspace.id, fixture.workspaceId)) - await db.delete(user).where(eq(user.id, fixture.aliceId)) - await db.delete(user).where(eq(user.id, fixture.bobId)) - } - await db.$client.end() - }) - - /** Bob's enrollment state for the fixture source, or null when none is offered. */ - async function viewerMembership() { - const [connector] = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, ids.connectorId)) - const memberships = await resolveViewerConnectorMemberships({ - userId: ids.bobId, - workspaceId: ids.workspaceId, - connectors: [connector!], - }) - return memberships.get(ids.connectorId) ?? null - } - - it('hides enrollment when the Confluence OAuth client is not configured', async () => { - const client = { id: env.CONFLUENCE_CLIENT_ID, secret: env.CONFLUENCE_CLIENT_SECRET } - try { - Object.assign(env, { CONFLUENCE_CLIENT_ID: undefined, CONFLUENCE_CLIENT_SECRET: undefined }) - await expect(viewerMembership()).resolves.toBeNull() - } finally { - Object.assign(env, { - CONFLUENCE_CLIENT_ID: client.id, - CONFLUENCE_CLIENT_SECRET: client.secret, - }) - } - }) - - function enroll(principal = bob) { - return startKnowledgeConnectorMemberEnrollment.execute({ principal, input: source }) - } - - function enrollments() { - return db - .select() - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, groupId)) - } - - async function crawlerAuthority() { - const [connectors, members, credentials, policies, roles] = await Promise.all([ - db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.knowledgeBaseId, ids.knowledgeBaseId)), - db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.workspaceId, ids.workspaceId)), - db.select().from(credential).where(eq(credential.workspaceId, ids.workspaceId)), - db.select().from(resourcePolicy).where(eq(resourcePolicy.workspaceId, ids.workspaceId)), - db - .select() - .from(permissions) - .where( - and(eq(permissions.entityType, 'workspace'), eq(permissions.entityId, ids.workspaceId)) - ), - ]) - return { connectors, members, credentials, policies, roles } - } - - it('lets a read-only member discover a paused source and obtain only their own identity invitation', async () => { - const before = await crawlerAuthority() - expect(before.members).toEqual([]) - expect(before.credentials).toEqual([]) - expect(before.policies).toHaveLength(1) - expect(before.roles.find((role) => role.userId === ids.bobId)?.permissionType).toBe('read') - await expect(viewerMembership()).resolves.toBe('not_enrolled') - - const { url } = await enroll() - const link = new URL(url) - expect(link.origin).toBe('http://localhost:3000') - expect(link.pathname.startsWith('/credential-groups/enroll/')).toBe(true) - const token = link.pathname.split('/').at(-1)! - const pending = await enrollments() - expect(pending).toHaveLength(1) - expect(pending[0]).toMatchObject({ - credentialGroupId: groupId, - email: `${ids.bobId}@fixture.test`, - status: 'invited', - createdBy: null, - sentAt: null, - invitationTokenHash: createHash('sha256').update(token).digest('hex'), - }) - expect(pending[0]!.invitationExpiresAt!.getTime()).toBeGreaterThan(Date.now()) - await expect(viewerMembership()).resolves.toBe('invited') - expect(await crawlerAuthority()).toEqual(before) - }) - - it('lists enrollment summaries across managed OAuth and personal token status types', async () => { - await enroll() - const [enrollment] = await enrollments() - expect(enrollment).toBeDefined() - const list = () => listCredentialGroupEnrollments(ids.workspaceId, groupId, 50) - await expect(list()).resolves.toMatchObject({ - enrollments: [{ id: enrollment!.id, connections: [] }], - nextCursor: null, - }) - - await db.insert(credential).values({ - id: generateId(), - workspaceId: ids.workspaceId, - type: 'managed_oauth', - displayName: 'Confluence fixture', - createdBy: ids.bobId, - providerId: 'confluence', - authorizationAppId: option.authorizationAppId, - credentialGroupEnrollmentId: enrollment!.id, - credentialGroupOptionId: option.id, - managedOauthScopeVersion: 1, - providerSubjectId: 'fixture-atlassian-account', - managedOauthStatus: 'active', - grantedScopes: ['read:confluence-content.all'], - encryptedOauthTokenSet: 'isolated-fixture-no-provider-token', - grantedAt: new Date(), - }) - await db.insert(credential).values( - (['active', 'needs_reauth', 'revoked'] as const).map((status) => ({ - id: generateId(), - workspaceId: ids.workspaceId, - type: 'personal_token' as const, - displayName: `GitLab ${status} fixture`, - providerId: 'gitlab', - createdBy: ids.bobId, - credentialGroupEnrollmentId: enrollment!.id, - providerSubjectId: `fixture-${status}`, - providerTenantId: 'gitlab.fixture.test', - encryptedPersonalToken: 'isolated-fixture-no-provider-token', - grantedScopes: ['read_api'], - accessTokenExpiresAt: status === 'active' ? null : new Date(0), - revokedAt: status === 'revoked' ? new Date(0) : null, - })) - ) - const result = await list() - expect(result.enrollments[0]!.connections).toHaveLength(4) - expect(result.enrollments[0]!.connections).toEqual( - expect.arrayContaining([ - { provider: 'confluence', status: 'active', count: 1 }, - { provider: 'gitlab', status: 'active', count: 1 }, - { provider: 'gitlab', status: 'needs_reauth', count: 1 }, - { provider: 'gitlab', status: 'revoked', count: 1 }, - ]) - ) - }) - - it.each(['missing', 'disabled', 'ambiguous', 'disabled-group'] as const)( - 'refuses enrollment when the canonical Confluence option is %s', - async (state) => { - const options = - state === 'missing' - ? [] - : state === 'disabled' - ? [{ ...option, status: 'disabled' as const }] - : state === 'ambiguous' - ? [option, { ...option, id: generateId() }] - : [option] - await db - .update(credentialGroup) - .set({ options, status: state === 'disabled-group' ? 'disabled' : 'active' }) - .where(eq(credentialGroup.id, groupId)) - const before = await crawlerAuthority() - await expect(viewerMembership()).resolves.toBeNull() - await expect(enroll()).rejects.toMatchObject({ code: 'validation' }) - expect(await enrollments()).toEqual([]) - expect(await crawlerAuthority()).toEqual(before) - } - ) - - it('preserves an administrator-revoked enrollment and its invitation instead of reactivating it', async () => { - await enroll() - await db - .update(credentialGroupEnrollment) - .set({ status: 'revoked', revokedAt: new Date() }) - .where(eq(credentialGroupEnrollment.credentialGroupId, groupId)) - const revoked = await enrollments() - const before = await crawlerAuthority() - await expect(viewerMembership()).resolves.toBe('revoked') - await expect(enroll()).rejects.toMatchObject({ code: 'forbidden' }) - expect(await enrollments()).toEqual(revoked) - expect(await crawlerAuthority()).toEqual(before) - }) - - it('requires the actor to verify their own email before issuing an invitation', async () => { - await db.update(user).set({ emailVerified: false }).where(eq(user.id, ids.bobId)) - await expect(viewerMembership()).resolves.toBe('unverified_email') - const before = await crawlerAuthority() - await expect(enroll()).rejects.toThrow('Verify your email address') - expect(await enrollments()).toEqual([]) - expect(await crawlerAuthority()).toEqual(before) - }) - - it.each([ - { connectorType: 'confluence', accessMode: 'workspace' }, - { connectorType: 'google_drive', accessMode: 'admin' }, - { connectorType: 'asana', accessMode: 'admin' }, - ])( - 'does not offer identity enrollment for $connectorType in $accessMode mode', - async (source) => { - await db - .update(knowledgeConnector) - .set(source) - .where(eq(knowledgeConnector.id, ids.connectorId)) - const before = await crawlerAuthority() - await expect(viewerMembership()).resolves.toBeNull() - await expect(enroll()).rejects.toMatchObject({ code: 'validation' }) - expect(await enrollments()).toEqual([]) - expect(await crawlerAuthority()).toEqual(before) - } - ) - - it('rejects another workspace actor and an asserted foreign workspace without changing grants', async () => { - const outsider: Principal = { ...bob, userId: foreign.bobId } - const before = await crawlerAuthority() - await expect(enroll(outsider)).rejects.toThrow('Insufficient workspace permissions') - await expect( - startKnowledgeConnectorMemberEnrollment.execute({ - principal: bob, - input: { ...source, assertedWorkspaceId: foreign.workspaceId }, - }) - ).rejects.toMatchObject({ code: 'not_found' }) - expect(await enrollments()).toEqual([]) - expect(await crawlerAuthority()).toEqual(before) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts b/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts deleted file mode 100644 index 2082c8467da..00000000000 --- a/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** Real PostgreSQL, member observations, authorization, and restoration; provider calls are fixtures. */ -import { db } from '@sim/db' -import { - document, - embedding, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - organization, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { generateId } from '@sim/utils/id' -import { eq, inArray } from 'drizzle-orm' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const provider = vi.hoisted(() => ({ list: vi.fn(), get: vi.fn(), changes: vi.fn() })) -vi.mock('@/connectors/registry.server', () => ({ - CONNECTOR_REGISTRY: { - google_drive: { - id: 'google_drive', - name: 'Fixture Drive', - auth: { mode: 'oauth', provider: 'google-drive' }, - permissionScopedListing: { capFieldIds: [] }, - listDocuments: provider.list, - getDocument: provider.get, - getChangeCursor: async () => 'fixture-start', - listChanges: provider.changes, - }, - }, -})) - -import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, - seedKnowledgeMemberFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { - listKnowledgeConnectorDocuments, - updateKnowledgeConnectorDocuments, -} from '@/lib/knowledge/application/connectors' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import * as memberAccess from '@/lib/knowledge/connectors/member-access' -import { recordMemberObservations } from '@/lib/knowledge/connectors/member-observations' -import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' - -describe('excluded member documents retain current source authorization', () => { - let ids: ReturnType - let members: Awaited> - let billing: Awaited> - let documentId: string - const principal = () => ({ - kind: 'session' as const, - userId: ids.aliceId, - sessionId: 'excluded-member-fixture', - }) - const target = () => ({ knowledgeBaseId: ids.knowledgeBaseId, connectorId: members.connectorId }) - const mutate = (operation: 'exclude' | 'restore') => - updateKnowledgeConnectorDocuments.execute({ - principal: principal(), - input: { ...target(), operation, documentIds: [documentId] }, - }) - const excluded = () => - listKnowledgeConnectorDocuments.execute({ - principal: principal(), - input: { ...target(), filter: 'excluded' }, - }) - const search = async () => { - const result = await searchKnowledge.execute({ - principal: principal(), - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - topK: 10, - tagFilters: [{ tagName: 'Fixture', operator: 'eq', value: 'excluded-member' }], - }, - }) - return result.results.map((row) => row.documentId) - } - const observations = () => - db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, documentId)) - - beforeAll(() => { - vi.stubGlobal('fetch', async () => { - throw new Error('Unexpected provider request in excluded member fixture') - }) - vi.spyOn(memberAccess, 'mintKnowledgeConnectorMemberToken').mockResolvedValue({ - accessToken: 'fixture-token', - refreshed: false, - }) - }) - - beforeEach(async () => { - provider.list.mockReset() - provider.get.mockReset() - provider.changes.mockReset() - ids = createKnowledgeAclFixtureIds() - documentId = generateId() - await seedKnowledgeAclFixture(ids) - members = await seedKnowledgeMemberFixture(ids) - billing = await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - await db - .insert(resourcePolicy) - .values({ - id: generateId(), - workspaceId: ids.workspaceId, - resourceType: 'credential_group', - resourceId: members.groupId, - document: compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: members.groupId, - allowedWorkflowIds: [], - }), - createdBy: ids.aliceId, - updatedBy: ids.aliceId, - }) - .onConflictDoNothing() - await memberAccess.grantKnowledgeConnectorCredentialAccess( - { - workspaceId: ids.workspaceId, - credentialGroupId: members.groupId, - credentialGroupOptionId: members.optionId, - connectorId: members.connectorId, - }, - ids.aliceId - ) - await db - .update(knowledgeConnector) - .set({ status: 'active', memberSyncStatus: 'idle', memberSyncLockToken: null }) - .where(eq(knowledgeConnector.id, members.connectorId)) - await db.insert(document).values({ - id: documentId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorId: members.connectorId, - externalId: 'excluded-file', - filename: 'Excluded fixture', - fileUrl: '', - fileSize: 0, - mimeType: 'text/plain', - processingStatus: 'completed', - contentHash: 'original-content', - acl: members.members.map((member) => member.subjectToken).sort(), - tag1: 'excluded-member', - }) - const content = 'Previously indexed source content remains excluded until explicitly restored.' - await db.insert(embedding).values({ - id: generateId(), - knowledgeBaseId: ids.knowledgeBaseId, - documentId, - chunkIndex: 0, - chunkHash: 'original-chunk', - content, - contentLength: content.length, - tokenCount: 12, - embedding: [1, ...Array(1535).fill(0)], - startOffset: 0, - endOffset: content.length, - tag1: 'excluded-member', - }) - for (const member of members.members) - await recordMemberObservations(db, member.id, [documentId], members.runId) - provider.list.mockResolvedValue({ - documents: [ - { - externalId: 'excluded-file', - title: 'Changed while excluded', - content: '', - contentDeferred: true, - contentHash: 'changed-content', - mimeType: 'text/plain', - }, - ], - hasMore: false, - }) - }) - - afterEach(async () => { - await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) - await db.delete(organization).where(eq(organization.id, ids.organizationId)) - await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) - }) - - afterAll(async () => { - vi.restoreAllMocks() - vi.unstubAllGlobals() - await db.$client.end() - }) - - it('keeps an excluded document restorable after a full sync without fetching or indexing its content', async () => { - expect(await search()).toEqual([documentId]) - expect((await mutate('exclude')).count).toBe(1) - const result = await executeMemberSync(members.connectorId, { billingAttribution: billing }) - expect(result.error).toBeUndefined() - expect(result.membersCompleted).toBe(2) - expect(result.docsAdded).toBe(0) - expect(result.docsUpdated).toBe(0) - expect(provider.get).not.toHaveBeenCalled() - const retained = await observations() - expect(retained).toHaveLength(2) - expect(retained.every((observation) => observation.runId !== members.runId)).toBe(true) - expect((await excluded()).documents.map((row) => row.id)).toEqual([documentId]) - expect(await search()).toEqual([]) - const [stored] = await db.select().from(document).where(eq(document.id, documentId)) - expect(stored).toMatchObject({ - userExcluded: true, - enabled: false, - contentHash: 'original-content', - }) - expect((await mutate('restore')).count).toBe(1) - expect(await search()).toEqual([documentId]) - }) - - it('withdraws excluded document access when a complete change feed revokes it', async () => { - expect((await mutate('exclude')).count).toBe(1) - await db - .update(knowledgeConnectorMember) - .set({ - changeCursor: 'fixture-start', - lastCompleteListingAt: new Date(), - memberSyncedThrough: new Date(), - }) - .where(eq(knowledgeConnectorMember.connectorId, members.connectorId)) - provider.changes.mockResolvedValue({ - changes: [{ kind: 'removed', externalId: 'excluded-file' }], - hasMore: false, - nextCursor: 'fixture-drained', - }) - const result = await executeMemberSync(members.connectorId, { billingAttribution: billing }) - expect(result.error).toBeUndefined() - expect(result.membersCompleted).toBe(2) - expect(provider.changes).toHaveBeenCalledTimes(2) - expect(provider.list).not.toHaveBeenCalled() - expect(provider.get).not.toHaveBeenCalled() - expect(await observations()).toEqual([]) - expect((await excluded()).documents).toEqual([]) - expect((await mutate('restore')).count).toBe(0) - expect(await search()).toEqual([]) - const [stored] = await db.select().from(document).where(eq(document.id, documentId)) - expect(stored).toMatchObject({ userExcluded: true, enabled: false, acl: [] }) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts deleted file mode 100644 index 03dcd80b13d..00000000000 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ /dev/null @@ -1,1367 +0,0 @@ -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -/** - * Fixture-backed GitHub API with real PostgreSQL, credential policy/token resolution, - * connector registry, member sync, storage, chunking, and application authorization. - * Provider replies and embeddings are deterministic; no live GitHub account is used. - */ - -import { createHash, generateKeyPairSync, verify } from 'node:crypto' -import { posix } from 'node:path' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - credentialMember, - document, - embedding, - knowledgeBase, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - member, - organization, - rateLimitBucket, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { readTestRedisUrl } from '@sim/db/testing/test-infrastructure' -import { sha256Hex } from '@sim/security/hash' -import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => ({ - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - }), -})) - -import { - resolveBillingAttribution, - resolveOrganizationBillingAttribution, -} from '@/lib/billing/core/billing-attribution' -import { env } from '@/lib/core/config/env' -import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' -import { encryptSecret } from '@/lib/core/security/encryption' -import { resetStorageMethod } from '@/lib/core/storage' -import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' -import { buildOrganizationAccountAccessPolicy } from '@/lib/credential-groups/application/workspace-access-policy' -import { - completeCredentialGroupEnrollment, - getCredentialGroupOAuthContext, - getCredentialGroupOAuthContextForEnrollment, -} from '@/lib/credential-groups/enrollments' -import { - completeCredentialGroupOAuth, - startCredentialGroupOAuth, -} from '@/lib/credential-groups/oauth' -import { consumeCredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' -import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' -import { - decryptManagedOAuthTokenSet, - encryptManagedOAuthTokenSet, -} from '@/lib/credentials/managed-oauth' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, - seedKnowledgeMemberFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import { subjectToken } from '@/lib/knowledge/access/tokens' -import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { - createKnowledgeConnector, - listKnowledgeConnectorDocuments, -} from '@/lib/knowledge/application/connectors' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import * as memberSyncEngine from '@/lib/knowledge/connectors/member-sync-engine' -import { - MEMBER_SUSPENDED_PURGE_DAYS, - MEMBER_TOMBSTONE_PURGE_DAYS, -} from '@/lib/knowledge/connectors/sync-limits' -import { getDocuments } from '@/lib/knowledge/documents/service' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' - -const redisUrl = readTestRedisUrl() - -/** Private repositories require the intersection of installation access and member access. */ -interface RepositoryFixture { - id: number - public: boolean - installed: boolean - stallRef: boolean - readers: Set - defaultBranch: string - files: Map - symlinks: Map - deniedStatus: 403 | 404 - throttledReaders: Set - throttledBlobReaders: Set - failedBlobs: Set - truncated: boolean -} - -describe('fixture-backed GitHub member search in PostgreSQL', () => { - let ids: ReturnType - let enrolled: Awaited> - let billing: Awaited> - const repositories = new Map() - const requests: { userId: string; path: string }[] = [] - const refreshedUsers = new Set() - const previousClient = { - id: env.GITHUB_APP_CLIENT_ID, - secret: env.GITHUB_APP_CLIENT_SECRET, - redis: env.REDIS_URL, - appId: env.GITHUB_APP_ID, - privateKey: env.GITHUB_APP_PRIVATE_KEY, - slug: env.GITHUB_APP_SLUG, - } - const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) - let organizationSource = false - let installationSuspended = false - const installation = () => ({ - id: 42, - app_id: 1, - client_id: 'github-fixture-client', - account: { id: 90, login: 'fixture', type: 'Organization' }, - repository_selection: 'selected', - permissions: { contents: 'read', metadata: 'read' }, - suspended_at: installationSuspended ? new Date().toISOString() : null, - }) - let oauthStateKey: string | undefined - let oauthVerification: { codeVerifier: string; redirectUri: string } | undefined - const tokenFor = (userId: string) => `ghu_fixture_${userId}` - const capacityKeyFor = (token: string) => - `provider:ocr:github-rest:${createHash('sha256').update(`Bearer ${token}`).digest('hex')}:capacity:v1` - const actor = (userId: string): Principal => ({ - kind: 'session', - userId, - sessionId: 'github-fixture', - }) - const workspaceKey = (): Principal => ({ - kind: 'workspace_api_key', - workspaceId: ids.workspaceId, - keyId: 'github-fixture', - }) - const shaFor = (content: string) => createHash('sha1').update(content).digest('hex') - - function repository(name: string, readers = [ids.aliceId, ids.bobId]) { - const value: RepositoryFixture = { - id: 9001 + repositories.size, - public: false, - installed: true, - stallRef: false, - readers: new Set(readers), - defaultBranch: 'trunk', - files: new Map([ - [ - 'docs/readme.md', - `Orion ${name}: repository documentation visible to authorized members.`, - ], - ]), - symlinks: new Map(), - deniedStatus: 404, - throttledReaders: new Set(), - throttledBlobReaders: new Set(), - failedBlobs: new Set(), - truncated: false, - } - repositories.set(name, value) - return value - } - - /** Implements only the documented GitHub endpoints requested by this fixture. */ - async function githubRequest(input: string | URL | Request, init?: RequestInit) { - const request = new Request(input, init) - const url = new URL(request.url) - if (url.href === 'https://github.com/login/oauth/access_token') { - expect(request.method).toBe('POST') - const body = new URLSearchParams(await request.text()) - expect(body.get('client_id')).toBe('github-fixture-client') - expect(body.get('client_secret')).toBe('github-fixture-client-secret') - if (body.has('code')) { - expect(oauthVerification).toBeDefined() - expect(body.get('code')).toBe('github-fixture-code') - expect(body.get('code_verifier')).toBe(oauthVerification!.codeVerifier) - expect(body.get('redirect_uri')).toBe(oauthVerification!.redirectUri) - return Response.json({ - access_token: tokenFor(ids.aliceId), - refresh_token: `ghr_fixture_${ids.aliceId}`, - token_type: 'bearer', - scope: '', - expires_in: 28800, - refresh_token_expires_in: 15897600, - }) - } - expect(body.get('grant_type')).toBe('refresh_token') - const member = enrolled.members.find( - (candidate) => body.get('refresh_token') === `ghr_fixture_${candidate.userId}` - ) - if (!member) throw new Error('Unexpected GitHub refresh credential') - refreshedUsers.add(member.userId) - return Response.json({ - access_token: `${tokenFor(member.userId)}_refreshed`, - refresh_token: `ghr_fixture_${member.userId}_refreshed`, - token_type: 'bearer', - scope: '', - expires_in: 28800, - refresh_token_expires_in: 15897600, - }) - } - if (url.origin !== 'https://api.github.com') - throw new Error(`Unexpected outbound request: ${request.method} ${url.origin}${url.pathname}`) - const bearer = request.headers.get('authorization')?.slice(7) ?? '' - if (url.pathname.startsWith('/app/installations/') || url.pathname.endsWith('/installation')) { - const [header, payload, signature] = bearer.split('.') - expect( - verify( - 'RSA-SHA256', - Buffer.from(`${header}.${payload}`), - publicKey, - Buffer.from(signature, 'base64url') - ) - ).toBe(true) - expect(JSON.parse(Buffer.from(payload, 'base64url').toString()).iss).toBe( - 'github-fixture-client' - ) - requests.push({ userId: 'app', path: url.pathname }) - if (url.pathname === '/app/installations/42/access_tokens') { - expect(request.method).toBe('POST') - const body = await request.json() - const contentToken = body.permissions.contents === 'read' - expect(body.permissions).toEqual( - contentToken ? { contents: 'read', metadata: 'read' } : { metadata: 'read' } - ) - if (contentToken) expect(body.repository_ids).toHaveLength(1) - else expect(body.repositories).toHaveLength(1) - const repositoryId = contentToken - ? body.repository_ids[0] - : repositories.get(body.repositories[0])?.id - expect( - [...repositories.values()].some( - (repository) => repository.id === repositoryId && repository.installed - ) - ).toBe(true) - return Response.json({ - token: `ghs_fixture_${contentToken ? 'installation' : 'metadata'}_${repositoryId}`, - expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), - permissions: body.permissions, - repositories: [{ id: repositoryId }], - }) - } - expect(request.method).toBe('GET') - const repositoryInstallation = url.pathname.match(/^\/repos\/fixture\/([^/]+)\/installation$/) - if (repositoryInstallation && !repositories.get(repositoryInstallation[1])?.installed) - return Response.json({ message: 'Not Found' }, { status: 404 }) - expect(url.pathname === '/app/installations/42' || Boolean(repositoryInstallation)).toBe(true) - return Response.json(installation()) - } - if (request.method !== 'GET') throw new Error(`Unexpected GitHub method: ${request.method}`) - const installationRepository = bearer.match( - /^ghs_fixture_(?:installation|metadata)_(\d+)$/ - )?.[1] - const metadataToken = bearer.startsWith('ghs_fixture_metadata_') - const installationToken = Boolean(installationRepository) - const member = enrolled.members.find((candidate) => - [tokenFor(candidate.userId), `${tokenFor(candidate.userId)}_refreshed`].some( - (token) => request.headers.get('authorization') === `Bearer ${token}` - ) - ) - if (!member && !installationToken) - throw new Error('GitHub request did not use an enrolled member or installation token') - const actingId = installationToken ? 'installation' : member!.userId - expect(request.headers.get('x-github-api-version')).toBe('2022-11-28') - requests.push({ userId: actingId, path: `${url.pathname}${url.search}` }) - if (url.pathname === '/user') { - expect(actingId).toBe(ids.aliceId) - return Response.json({ - id: 101, - login: 'github-fixture-alice', - type: 'User', - name: 'Alice Fixture', - }) - } - if (url.pathname === '/user/emails') { - expect(url.searchParams.get('per_page')).toBe('100') - expect(url.searchParams.get('page')).toBe('1') - return Response.json([ - { email: 'personal@github-fixture.test', primary: true, verified: true }, - { email: `${actingId}@fixture.test`, primary: false, verified: true }, - ]) - } - const match = url.pathname.match(/^\/repos\/fixture\/([^/]+)(.*)$/) - if (!match) throw new Error(`Unexpected GitHub endpoint: ${url.pathname}`) - const source = repositories.get(match[1]) - if (!source) throw new Error('Unexpected GitHub repository') - if (installationToken) expect(installationRepository).toBe(String(source.id)) - if (source.throttledReaders.has(actingId)) - return Response.json( - { message: 'You have exceeded a secondary rate limit.' }, - { status: 403 } - ) - if ( - (installationToken && !source.installed) || - (!installationToken && !source.public && (!source.installed || !source.readers.has(actingId))) - ) - return Response.json( - { message: 'Resource not accessible by integration' }, - { status: source.deniedStatus } - ) - if (!match[2]) - return Response.json({ - id: source.id, - owner: { id: 90 }, - full_name: `fixture/${match[1]}`, - private: !source.public, - default_branch: source.defaultBranch, - }) - expect(metadataToken).toBe(false) - if (match[2].startsWith('/branches/')) { - const branch = decodeURIComponent(match[2].slice('/branches/'.length)) - return branch === source.defaultBranch - ? Response.json({ name: branch, commit: { sha: shaFor(branch) }, protected: false }) - : Response.json({ message: 'Not Found' }, { status: 404 }) - } - if (match[2].startsWith('/git/ref/heads/')) { - if (source.stallRef) - return new Promise((_resolve, reject) => { - request.signal.addEventListener('abort', () => reject(request.signal.reason), { - once: true, - }) - }) - const ref = decodeURIComponent(match[2].slice('/git/ref/heads/'.length)) - return ref === source.defaultBranch - ? Response.json({ ref: `refs/heads/${ref}`, object: { type: 'commit', sha: shaFor(ref) } }) - : Response.json({ message: 'Not Found' }, { status: 404 }) - } - if (match[2].startsWith('/git/trees/')) { - const ref = decodeURIComponent(match[2].slice('/git/trees/'.length)) - const treeSha = shaFor(JSON.stringify([[...source.files], [...source.symlinks]])) - if (ref !== source.defaultBranch && ref !== treeSha) - return Response.json({ message: 'Not Found' }, { status: 404 }) - expect(url.searchParams.get('recursive')).toBe('1') - return Response.json({ - sha: treeSha, - tree: [ - ...[...source.files].map(([filePath, content]) => ({ - path: filePath, - mode: '100644', - type: 'blob', - sha: shaFor(content), - size: Buffer.byteLength(content), - })), - ...[...source.symlinks].map(([filePath, target]) => ({ - path: filePath, - mode: '120000', - type: 'blob', - sha: shaFor(target), - size: Buffer.byteLength(target), - })), - ], - truncated: source.truncated, - }) - } - if (match[2].startsWith('/git/blobs/')) { - if (source.throttledBlobReaders.has(actingId)) - return Response.json( - { message: 'You have exceeded a secondary rate limit.' }, - { status: 403 } - ) - const sha = decodeURIComponent(match[2].slice('/git/blobs/'.length)) - if (source.failedBlobs.has(sha)) { - return Response.json( - { message: 'Fixture provider unavailable' }, - { status: 503, headers: { 'Retry-After': '3600' } } - ) - } - const content = [...source.files.values(), ...source.symlinks.values()].find( - (value) => shaFor(value) === sha - ) - return content === undefined - ? Response.json({ message: 'Not Found' }, { status: 404 }) - : new Response(content) - } - if (match[2].startsWith('/contents/')) { - expect(url.searchParams.get('ref')).toBe(source.defaultBranch) - const filePath = decodeURIComponent(match[2].slice('/contents/'.length)) - const target = source.symlinks.get(filePath) - const content = - target === undefined - ? source.files.get(filePath) - : source.files.get(posix.join(posix.dirname(filePath), target)) - if (target !== undefined && content === undefined) { - return Response.json({ - type: 'symlink', - path: filePath, - sha: shaFor(target), - target, - size: Buffer.byteLength(target), - }) - } - if (content === undefined) return Response.json({ message: 'Not Found' }, { status: 404 }) - return Response.json({ - type: 'file', - path: filePath, - sha: shaFor(target ?? content), - size: Buffer.byteLength(content), - encoding: 'base64', - content: Buffer.from(content).toString('base64'), - }) - } - throw new Error(`Unexpected GitHub endpoint: ${url.pathname}`) - } - - beforeEach(async () => { - repositories.clear() - requests.length = 0 - refreshedUsers.clear() - organizationSource = false - installationSuspended = false - oauthStateKey = undefined - oauthVerification = undefined - Object.assign(env, { - GITHUB_APP_CLIENT_ID: 'github-fixture-client', - GITHUB_APP_CLIENT_SECRET: 'github-fixture-client-secret', - }) - ids = createKnowledgeAclFixtureIds() - await seedKnowledgeAclFixture(ids) - enrolled = await seedKnowledgeMemberFixture(ids) - const policy = await getCredentialGroupProviderAdapter('github-repositories').getPolicy( - undefined, - { - workspaceId: ids.workspaceId, - } - ) - expect(policy.requiredScopes).toEqual([]) - await db - .update(credentialGroup) - .set({ - options: [ - { - id: enrolled.optionId, - provider: 'github-repositories', - label: 'GitHub fixture', - authorizationAppId: policy.authorizationAppId, - requiredScopes: [], - scopeVersion: policy.scopeVersion, - required: false, - status: 'active', - }, - ], - }) - .where(eq(credentialGroup.id, enrolled.groupId)) - for (const [index, member] of enrolled.members.entries()) { - const identity = { - providerId: 'github-repositories', - providerSubjectId: String(index + 101), - providerTenantId: null, - } - await db - .update(credential) - .set({ - ...identity, - displayName: 'GitHub fixture', - authorizationAppId: policy.authorizationAppId, - managedOauthScopeVersion: policy.scopeVersion, - grantedScopes: [], - encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ - accessToken: tokenFor(member.userId), - refreshToken: `ghr_fixture_${member.userId}`, - }), - accessTokenExpiresAt: new Date(Date.now() + 8 * 60 * 60 * 1000), - refreshTokenExpiresAt: new Date(Date.now() + 180 * 24 * 60 * 60 * 1000), - }) - .where(eq(credential.id, member.credentialId)) - member.subjectToken = subjectToken(identity) - await db - .update(knowledgeConnectorMember) - .set({ subjectToken: member.subjectToken }) - .where(eq(knowledgeConnectorMember.id, member.id)) - } - await db - .insert(resourcePolicy) - .values({ - id: generateId(), - workspaceId: ids.workspaceId, - resourceType: 'credential_group', - resourceId: enrolled.groupId, - document: compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: enrolled.groupId, - allowedWorkflowIds: [], - }), - createdBy: ids.aliceId, - updatedBy: ids.aliceId, - }) - .onConflictDoNothing() - await db - .update(knowledgeConnector) - .set({ - connectorType: 'github', - sourceConfig: { repository: 'fixture/shared', maxFiles: 0 }, - memberSyncStatus: 'idle', - memberSyncLockToken: null, - }) - .where(eq(knowledgeConnector.id, enrolled.connectorId)) - await grant(enrolled.connectorId) - billing = await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - vi.stubGlobal('fetch', githubRequest) - repository('shared') - }) - - afterEach(async () => { - const files = await db - .select({ key: document.storageKey }) - .from(document) - .where(eq(document.knowledgeBaseId, ids.knowledgeBaseId)) - try { - for (const { key } of files) { - if (key) - await deleteFile({ key, context: 'knowledge-base' }).catch((error: unknown) => { - if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error - }) - } - } finally { - if (oauthStateKey) await getRedisClient()?.del(oauthStateKey) - await closeRedisConnection() - Object.assign(env, { REDIS_URL: previousClient.redis }) - resetStorageMethod() - await db.delete(rateLimitBucket).where( - inArray( - rateLimitBucket.key, - enrolled.members.flatMap(({ userId }) => - [tokenFor(userId), `${tokenFor(userId)}_refreshed`].map(capacityKeyFor) - ) - ) - ) - await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) - await db.delete(organization).where(eq(organization.id, ids.organizationId)) - await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) - vi.restoreAllMocks() - vi.unstubAllGlobals() - } - }) - afterAll(async () => { - Object.assign(env, { - GITHUB_APP_CLIENT_ID: previousClient.id, - GITHUB_APP_CLIENT_SECRET: previousClient.secret, - GITHUB_APP_ID: previousClient.appId, - GITHUB_APP_PRIVATE_KEY: previousClient.privateKey, - GITHUB_APP_SLUG: previousClient.slug, - }) - await db.$client.end() - }) - - async function grant(connectorId: string) { - await grantKnowledgeConnectorCredentialAccess( - { - connectorId, - workspaceId: ids.workspaceId, - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - }, - ids.aliceId - ) - } - - async function addSource(name: string) { - const connectorId = generateId() - await db.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - sourceConfig: { repository: `fixture/${name}`, maxFiles: 0 }, - accessMode: 'members', - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - }) - await grant(connectorId) - return connectorId - } - - async function sync(connectorId = enrolled.connectorId, forceContentRefresh = true) { - await db - .update(knowledgeConnectorMember) - .set({ nextAttemptAt: new Date(0) }) - .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - return memberSyncEngine.executeMemberSync(connectorId, { - billingAttribution: billing, - forceContentRefresh, - }) - } - - async function rows(connectorId = enrolled.connectorId) { - return db - .select() - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.deletedAt))) - .orderBy(document.externalId) - } - - async function search(principal: Principal, searchMode: 'hybrid' | 'vector' = 'hybrid') { - const result = await searchKnowledge.execute({ - principal, - input: { - ...(organizationSource - ? { organizationId: ids.organizationId } - : { workspaceId: ids.workspaceId }), - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - searchMode, - topK: 20, - }, - }) - return result.results.map((item) => item.documentId).sort() - } - - async function assertAccess( - principal: Principal, - row: typeof document.$inferSelect, - allowed: boolean - ) { - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - if (allowed) { - expect((await readKnowledgeDocument.execute({ principal, input })).document.id).toBe(row.id) - expect( - (await listKnowledgeChunks.execute({ principal, input })).chunks.length - ).toBeGreaterThan(0) - } else { - await expect(readKnowledgeDocument.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - await expect(listKnowledgeChunks.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - } - } - - async function useOrganizationInstallation() { - organizationSource = true - Object.assign(env, { - GITHUB_APP_ID: '1', - GITHUB_APP_PRIVATE_KEY: privateKey.export({ type: 'pkcs8', format: 'pem' }).toString(), - GITHUB_APP_SLUG: 'github-fixture', - }) - await db.insert(member).values([ - { id: generateId(), organizationId: ids.organizationId, userId: ids.aliceId, role: 'owner' }, - { id: generateId(), organizationId: ids.organizationId, userId: ids.bobId, role: 'member' }, - ]) - await db - .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - await db - .update(credentialGroup) - .set({ workspaceId: null, organizationId: ids.organizationId }) - .where(eq(credentialGroup.id, enrolled.groupId)) - await db - .update(resourcePolicy) - .set({ - workspaceId: null, - organizationId: ids.organizationId, - document: buildOrganizationAccountAccessPolicy(enrolled.groupId, []), - }) - .where( - and( - eq(resourcePolicy.resourceType, 'credential_group'), - eq(resourcePolicy.resourceId, enrolled.groupId) - ) - ) - await db - .update(credential) - .set({ workspaceId: null, organizationId: ids.organizationId }) - .where( - inArray( - credential.id, - enrolled.members.map((entry) => entry.credentialId) - ) - ) - await db - .update(knowledgeConnectorMember) - .set({ workspaceId: null, organizationId: ids.organizationId }) - .where(eq(knowledgeConnectorMember.connectorId, enrolled.connectorId)) - const installationCredentialId = generateId() - const { encrypted } = await encryptSecret( - JSON.stringify({ - type: 'github_app_installation', - version: 1, - appId: '1', - appClientId: 'github-fixture-client', - installationId: '42', - accountId: '90', - accountType: 'Organization', - accountLogin: 'fixture', - repositorySelection: 'selected', - }) - ) - await db.insert(credential).values({ - id: installationCredentialId, - organizationId: ids.organizationId, - type: 'service_account', - providerId: 'github-app-installation', - providerSubjectId: '42', - providerTenantId: '90', - encryptedServiceAccountKey: encrypted, - displayName: 'GitHub fixture installation', - createdBy: ids.aliceId, - }) - await db.insert(credentialMember).values({ - id: generateId(), - credentialId: installationCredentialId, - userId: ids.aliceId, - role: 'admin', - status: 'active', - }) - await db - .update(knowledgeConnector) - .set({ - credentialId: installationCredentialId, - sourceConfig: { repository: 'fixture/shared', githubRepositoryId: '9001', maxFiles: 0 }, - }) - .where(eq(knowledgeConnector.id, enrolled.connectorId)) - billing = await resolveOrganizationBillingAttribution({ - actorUserId: ids.aliceId, - organizationId: ids.organizationId, - }) - return installationCredentialId - } - - it('creates a live installation source without indexing or replacing connected member accounts', async () => { - const installationCredentialId = await useOrganizationInstallation() - const enrollmentsBefore = await db - .select({ - id: credentialGroupEnrollment.id, - userId: credentialGroupEnrollment.userId, - status: credentialGroupEnrollment.status, - }) - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, enrolled.groupId)) - .orderBy(credentialGroupEnrollment.id) - const credentialsBefore = await db - .select({ - id: credential.id, - enrollmentId: credential.credentialGroupEnrollmentId, - subjectId: credential.providerSubjectId, - }) - .from(credential) - .where(eq(credential.credentialGroupOptionId, enrolled.optionId)) - .orderBy(credential.id) - const later = repository('later', [ids.aliceId]) - const input = { - knowledgeBaseId: ids.knowledgeBaseId, - assertedOrganizationId: ids.organizationId, - connectorType: 'github', - accessMode: 'members' as const, - credentialId: installationCredentialId, - sourceConfig: { repository: 'fixture/later' }, - syncIntervalMinutes: 0, - } - await expect( - createKnowledgeConnector.execute({ principal: actor(ids.bobId), input }) - ).rejects.toThrow() - const { connector } = await createKnowledgeConnector.execute({ - principal: actor(ids.aliceId), - input, - }) - const connectorId = connector.id - expect(connector).toMatchObject({ - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - sourceConfig: { repository: 'fixture/later', githubRepositoryId: String(later.id) }, - }) - const [current] = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - expect(current).toMatchObject({ lastMemberSyncAt: null, nextMemberSyncAt: null }) - expect( - await db - .select({ - id: credentialGroupEnrollment.id, - userId: credentialGroupEnrollment.userId, - status: credentialGroupEnrollment.status, - }) - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, enrolled.groupId)) - .orderBy(credentialGroupEnrollment.id) - ).toEqual(enrollmentsBefore) - expect( - await db - .select({ - id: credential.id, - enrollmentId: credential.credentialGroupEnrollmentId, - subjectId: credential.providerSubjectId, - }) - .from(credential) - .where(eq(credential.credentialGroupOptionId, enrolled.optionId)) - .orderBy(credential.id) - ).toEqual(credentialsBefore) - expect(await rows(connectorId)).toEqual([]) - expect( - await db - .select({ id: embedding.id }) - .from(embedding) - .where(eq(embedding.knowledgeBaseId, ids.knowledgeBaseId)) - ).toEqual([]) - expect(requests.filter((entry) => entry.path.includes('/git/blobs/'))).toEqual([]) - }) - - it('keeps actual skips, legacy skips, and provider failures distinct in authorized document lists', async () => { - const source = repositories.get('shared')! - source.readers.delete(ids.bobId) - source.files.set('empty.txt', '') - source.files.set('image.png', 'binary\0contents') - expect((await sync()).error).toBeUndefined() - const initial = await rows() - const empty = initial.find((row) => row.externalId === 'empty.txt')! - const binary = initial.find((row) => row.externalId === 'image.png')! - expect(empty).toMatchObject({ processingStatus: 'failed', storageKey: null }) - expect(binary).toMatchObject({ processingStatus: 'failed', storageKey: null }) - expect(empty.contentHash).not.toBeNull() - await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, empty.id)) - source.files.set('unavailable.txt', 'Orion content whose blob cannot be fetched.') - source.failedBlobs.add(shaFor(source.files.get('unavailable.txt')!)) - await sync() - await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, empty.id)) - const failed = (await rows()).find((row) => row.externalId === 'unavailable.txt')! - expect(failed).toMatchObject({ - processingStatus: 'failed', - storageKey: null, - contentHash: null, - }) - for (const userId of [ids.aliceId, ids.bobId]) { - const provider = createKnowledgeAccessProvider(actor(userId), { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - }) - const listed = await getDocuments(ids.knowledgeBaseId, {}, 'github-skip-outcomes', provider) - expect(listed.pagination.total).toBe(userId === ids.aliceId ? 4 : 0) - if (userId === ids.aliceId) { - expect(listed.documents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: empty.id, - processingStatus: 'failed', - processingOutcome: 'skipped', - }), - expect.objectContaining({ - id: binary.id, - processingStatus: 'failed', - processingOutcome: 'skipped', - }), - expect.objectContaining({ id: failed.id, processingStatus: 'failed' }), - ]) - ) - } - for (const filter of ['failed', 'skipped'] as const) { - const outcomes = await listKnowledgeConnectorDocuments.execute({ - principal: actor(userId), - input: { - connectorId: enrolled.connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - filter, - }, - }) - expect(outcomes.counts).toMatchObject({ - failed: userId === ids.aliceId ? 1 : 0, - skipped: userId === ids.aliceId ? 2 : 0, - }) - expect(outcomes.documents.map((row) => row.id).sort()).toEqual( - userId === ids.aliceId - ? (filter === 'failed' ? [failed.id] : [empty.id, binary.id]).sort() - : [] - ) - } - } - }) - - it.runIf(Boolean(redisUrl))( - 'completes a PKCE OAuth attempt through Redis and persists a searchable scopeless credential', - async () => { - Object.assign(env, { REDIS_URL: redisUrl }) - const alice = enrolled.members[0] - const invitationToken = generateId() - await db - .update(knowledgeConnector) - .set({ status: 'paused' }) - .where(eq(knowledgeConnector.id, enrolled.connectorId)) - await db.delete(credential).where(eq(credential.id, alice.credentialId)) - await db - .update(credentialGroupEnrollment) - .set({ - status: 'invited', - completedAt: null, - invitationTokenHash: sha256Hex(invitationToken), - }) - .where(eq(credentialGroupEnrollment.id, alice.enrollmentId)) - const context = await getCredentialGroupOAuthContext(invitationToken, enrolled.optionId) - expect(context).not.toBeNull() - const authorization = new URL( - await startCredentialGroupOAuth(context!, invitationToken, { returnTo: 'search' }) - ) - expect(authorization.origin + authorization.pathname).toBe( - 'https://github.com/login/oauth/authorize' - ) - expect(authorization.searchParams.get('client_id')).toBe('github-fixture-client') - expect(authorization.searchParams.get('scope') ?? '').toBe('') - expect(authorization.searchParams.get('code_challenge_method')).toBe('S256') - const state = authorization.searchParams.get('state')! - oauthStateKey = `credential-group:oauth-attempt:${sha256Hex(state)}` - const storedAttempt = await getRedisClient()!.get(oauthStateKey) - expect(storedAttempt).not.toBeNull() - expect(storedAttempt).not.toContain(invitationToken) - expect(JSON.parse(storedAttempt!)).toMatchObject({ requiredScopes: [], returnTo: 'search' }) - const attempt = await consumeCredentialGroupOAuthAttempt(state) - expect(attempt).toMatchObject({ - requiredScopes: [], - invitationToken, - provider: 'github-repositories', - }) - expect(attempt!.codeVerifier).toBeTruthy() - expect(storedAttempt).not.toContain(attempt!.codeVerifier!) - expect(authorization.searchParams.get('code_challenge')).toBe( - createHash('sha256').update(attempt!.codeVerifier!).digest('base64url') - ) - expect(attempt!.redirectUri).toBe( - 'http://localhost:3000/api/auth/oauth2/callback/github-repositories' - ) - expect(await consumeCredentialGroupOAuthAttempt(state)).toBeNull() - oauthVerification = { - codeVerifier: attempt!.codeVerifier!, - redirectUri: attempt!.redirectUri, - } - const currentContext = await getCredentialGroupOAuthContextForEnrollment( - attempt!, - enrolled.optionId - ) - expect(currentContext).not.toBeNull() - const completion = await completeCredentialGroupOAuth( - currentContext!, - attempt!, - 'github-fixture-code' - ) - expect(completion).toMatchObject({ - created: true, - enrollmentStatus: 'in_progress', - providerId: 'github-repositories', - }) - const [stored] = await db - .select() - .from(credential) - .where(eq(credential.id, completion.credentialId)) - expect(stored).toMatchObject({ - type: 'managed_oauth', - grantedScopes: [], - managedOauthStatus: 'active', - authorizationAppId: attempt!.authorizationAppId, - providerSubjectId: '101', - providerTenantId: null, - credentialGroupEnrollmentId: alice.enrollmentId, - credentialGroupOptionId: enrolled.optionId, - }) - expect(stored.encryptedOauthTokenSet).not.toContain(tokenFor(ids.aliceId)) - expect(await decryptManagedOAuthTokenSet(stored.encryptedOauthTokenSet!)).toMatchObject({ - accessToken: tokenFor(ids.aliceId), - refreshToken: `ghr_fixture_${ids.aliceId}`, - }) - expect(stored.accessTokenExpiresAt!.getTime()).toBeGreaterThan(Date.now()) - expect(stored.refreshTokenExpiresAt!.getTime()).toBeGreaterThan( - stored.accessTokenExpiresAt!.getTime() - ) - expect(await completeCredentialGroupEnrollment(invitationToken)).toBe(true) - alice.credentialId = completion.credentialId - await db - .update(knowledgeConnector) - .set({ status: 'active' }) - .where(eq(knowledgeConnector.id, enrolled.connectorId)) - const result = await sync() - expect(result.error).toBeUndefined() - expect(result.membersCompleted).toBe(2) - const [shared] = await rows() - expect(await search(actor(ids.aliceId))).toEqual([shared.id]) - await assertAccess(actor(ids.aliceId), shared, true) - } - ) - - it('stores empty GitHub scopes but rejects missing required credential metadata in PostgreSQL', async () => { - for (const updates of [ - { grantedScopes: null }, - { authorizationAppId: null }, - { encryptedOauthTokenSet: null }, - ]) { - await expect( - db - .update(credential) - .set(updates) - .where(eq(credential.id, enrolled.members[0].credentialId)) - ).rejects.toMatchObject({ - cause: { code: '23514', constraint_name: 'credential_managed_oauth_source_check' }, - }) - } - const [stored] = await db - .select() - .from(credential) - .where(eq(credential.id, enrolled.members[0].credentialId)) - expect(stored.grantedScopes).toEqual([]) - expect(stored.authorizationAppId).toBeTruthy() - expect(stored.encryptedOauthTokenSet).toBeTruthy() - }) - - it('requires member access and App installation for private repository search, documents, chunks, and files', async () => { - repository('private', [ids.aliceId]) - const blocked = repository('uninstalled', [ids.bobId]) - blocked.installed = false - const privateId = await addSource('private') - const blockedId = await addSource('uninstalled') - for (const connectorId of [enrolled.connectorId, privateId, blockedId]) { - const result = await sync(connectorId) - expect(result.error).toBeUndefined() - expect(result.membersFailed).toBe(0) - } - const [shared] = await rows() - const [privateFile] = await rows(privateId) - expect(await rows(blockedId)).toEqual([]) - expect( - requests.filter((request) => request.path.startsWith('/repos/fixture/shared/git/blobs/')) - ).toHaveLength(1) - expect(shared.acl).toEqual( - expect.arrayContaining(enrolled.members.map((member) => member.subjectToken)) - ) - expect(privateFile.acl).toEqual([enrolled.members[0].subjectToken]) - expect(await search(actor(ids.aliceId))).toEqual([shared.id, privateFile.id].sort()) - expect(await search(actor(ids.bobId))).toEqual([shared.id]) - expect(await search(workspaceKey())).toEqual([]) - await assertAccess(actor(ids.bobId), shared, true) - await assertAccess(actor(ids.bobId), privateFile, false) - await assertAccess(workspaceKey(), shared, false) - expect( - ( - await downloadFileFromUrl(privateFile.fileUrl, { - userId: ids.aliceId, - knowledgeAccess: 'user', - }) - ).toString() - ).toContain('Orion private') - await expect( - downloadFileFromUrl(privateFile.fileUrl, { userId: ids.bobId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - const observations = await db - .select() - .from(knowledgeDocumentObservation) - .where(inArray(knowledgeDocumentObservation.documentId, [shared.id, privateFile.id])) - expect(observations).toHaveLength(3) - blocked.installed = true - await sync(blockedId) - const [newlyInstalled] = await rows(blockedId) - expect(await search(actor(ids.bobId))).toEqual([shared.id, newlyInstalled.id].sort()) - await assertAccess(actor(ids.aliceId), newlyInstalled, false) - blocked.installed = false - await sync(blockedId) - expect(await search(actor(ids.bobId))).toEqual([shared.id]) - await assertAccess(actor(ids.bobId), newlyInstalled, false) - }) - - it('preserves observations on secondary-rate-limit 403 and withdraws them on repository-access 403', async () => { - await sync() - const [shared] = await rows() - const source = repositories.get('shared')! - source.throttledReaders.add(ids.bobId) - const throttled = await sync() - expect(throttled.error).toBeUndefined() - expect(throttled.deferred).toMatchObject({ reason: 'rate_limit', providerId: 'github-rest' }) - expect(await search(actor(ids.bobId))).toEqual([shared.id]) - const [bob] = await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, enrolled.members[1].id)) - expect(bob.status).toBe('active') - source.throttledReaders.clear() - source.readers.delete(ids.bobId) - source.deniedStatus = 403 - /** A provider cooldown protects the shared credential even when another sync starts early. */ - const requestsBeforeRetry = requests.filter(({ userId }) => userId === ids.bobId).length - await sync() - expect(requests.filter(({ userId }) => userId === ids.bobId)).toHaveLength(requestsBeforeRetry) - expect(await search(actor(ids.bobId))).toEqual([shared.id]) - /** Expire only the fixture's cooldown so the next poll observes the subsequent access change. */ - await db - .update(rateLimitBucket) - .set({ - capacityState: sql`${rateLimitBucket.capacityState} || ${JSON.stringify({ cooldownUntil: 0, nextRequestAt: 0 })}::jsonb`, - }) - .where(eq(rateLimitBucket.key, capacityKeyFor(tokenFor(ids.bobId)))) - await sync() - expect(await search(actor(ids.bobId))).toEqual([]) - expect(await search(actor(ids.aliceId))).toEqual([shared.id]) - await assertAccess(actor(ids.bobId), shared, false) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, enrolled.members[1].id)) - ).toEqual([]) - }) - - it('revokes enrollment access immediately and purges suspended observations after the recovery window', async () => { - await sync() - const [shared] = await rows() - await db - .update(credentialGroupEnrollment) - .set({ status: 'revoked', revokedAt: new Date() }) - .where(eq(credentialGroupEnrollment.id, enrolled.members[1].enrollmentId)) - expect(await search(actor(ids.bobId))).toEqual([]) - await assertAccess(actor(ids.bobId), shared, false) - await sync() - const [suspended] = await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, enrolled.members[1].id)) - expect(suspended.status).toBe('suspended') - expect((await rows())[0].acl).not.toContain(suspended.subjectToken) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, suspended.id)) - ).toHaveLength(1) - await db - .update(knowledgeConnectorMember) - .set({ - suspendedAt: new Date(Date.now() - (MEMBER_SUSPENDED_PURGE_DAYS + 1) * 24 * 60 * 60 * 1000), - }) - .where(eq(knowledgeConnectorMember.id, suspended.id)) - await sync() - expect( - await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, suspended.id)) - ).toEqual([]) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, suspended.id)) - ).toEqual([]) - expect(await search(actor(ids.aliceId))).toEqual([shared.id]) - }) - - it('follows default-branch changes and persists updates and deletion revocations', async () => { - const source = repositories.get('shared')! - source.files.set( - 'docs/deleted.md', - 'Orion obsolete guide: this file will be removed from the repository.' - ) - await sync() - const before = await rows() - const removed = before.find((row) => row.externalId === 'docs/deleted.md')! - const updated = before.find((row) => row.externalId === 'docs/readme.md')! - const content = - 'Orion revised guide: the repository now follows the release branch and the previous instructions are replaced.' - source.defaultBranch = 'release/current' - source.files.set('docs/readme.md', content) - source.files.delete('docs/deleted.md') - const result = await sync() - expect(result.error).toBeUndefined() - expect((await rows()).map((row) => row.id)).toEqual([updated.id]) - const chunks = await listKnowledgeChunks.execute({ - principal: actor(ids.aliceId), - input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: updated.id }, - }) - expect(chunks.chunks.map((chunk) => chunk.content).join('\n')).toBe(content) - expect((await rows())[0].contentHash).toBe(`git-sha:${shaFor(content)}`) - await assertAccess(actor(ids.aliceId), removed, false) - expect(await search(actor(ids.bobId))).toEqual([updated.id]) - expect(requests.some((request) => request.path.includes('/git/trees/release%2Fcurrent'))).toBe( - true - ) - expect( - requests.some( - (request) => request.path === `/repos/fixture/shared/git/blobs/${shaFor(content)}` - ) - ).toBe(true) - const [tombstone] = await db.select().from(document).where(eq(document.id, removed.id)) - expect(tombstone.deletedAt).toBeInstanceOf(Date) - await db - .update(document) - .set({ - deletedAt: new Date(Date.now() - (MEMBER_TOMBSTONE_PURGE_DAYS + 1) * 24 * 60 * 60 * 1000), - }) - .where(eq(document.id, removed.id)) - expect((await sync()).docsPurged).toBe(1) - expect(await db.select().from(document).where(eq(document.id, removed.id))).toEqual([]) - expect(await db.select().from(embedding).where(eq(embedding.documentId, removed.id))).toEqual( - [] - ) - }) - - it('restarts an expired pinned tree on the current default branch and rechecks member repository access', async () => { - const source = repositories.get('shared')! - source.files.set('docs/deleted.md', 'Orion obsolete documentation.') - await sync() - const before = await rows() - const updated = before.find((row) => row.externalId === 'docs/readme.md')! - const removed = before.find((row) => row.externalId === 'docs/deleted.md')! - source.files.set('docs/readme.md', 'Orion intermediate revision awaiting provider capacity.') - const expiredTreeSha = shaFor(JSON.stringify([[...source.files], [...source.symlinks]])) - source.throttledBlobReaders.add(ids.aliceId) - await db - .update(knowledgeConnectorMember) - .set({ lastStartedAt: new Date(0) }) - .where(eq(knowledgeConnectorMember.id, enrolled.members[0].id)) - expect((await sync()).deferred).toMatchObject({ reason: 'rate_limit' }) - const [paused] = await db - .select({ checkpoint: knowledgeConnectorMember.listingCheckpoint }) - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, enrolled.members[0].id)) - expect(paused.checkpoint).toMatchObject({ - complete: false, - listedCount: 0, - cursor: JSON.stringify({ version: 1, treeSha: expiredTreeSha, branch: 'trunk', offset: 0 }), - }) - expect(await search(actor(ids.aliceId))).toEqual(before.map((row) => row.id).sort()) - expect(await search(actor(ids.bobId))).toEqual(before.map((row) => row.id).sort()) - source.throttledBlobReaders.clear() - source.defaultBranch = 'release/current' - const content = 'Orion current release documentation remains accessible to Alice.' - source.files.set('docs/readme.md', content) - source.files.delete('docs/deleted.md') - source.readers.delete(ids.bobId) - await db - .update(rateLimitBucket) - .set({ - capacityState: sql`${rateLimitBucket.capacityState} || ${JSON.stringify({ cooldownUntil: 0, nextRequestAt: 0 })}::jsonb`, - }) - .where(eq(rateLimitBucket.key, capacityKeyFor(tokenFor(ids.aliceId)))) - const requestOffset = requests.length - const resumed = await sync() - expect(resumed.error).toBeUndefined() - expect(resumed.deferred).toBeUndefined() - expect(resumed.membersFailed).toBe(0) - const [current] = await rows() - expect(current).toMatchObject({ - id: updated.id, - contentHash: `git-sha:${shaFor(content)}`, - acl: [enrolled.members[0].subjectToken], - }) - expect(await search(actor(ids.aliceId))).toEqual([updated.id]) - expect(await search(actor(ids.bobId))).toEqual([]) - await assertAccess(actor(ids.aliceId), current, true) - await assertAccess(actor(ids.bobId), current, false) - const [tombstone] = await db.select().from(document).where(eq(document.id, removed.id)) - expect(tombstone.deletedAt).toBeInstanceOf(Date) - const resumedPaths = requests.slice(requestOffset).map((request) => request.path) - expect(resumedPaths).toContain(`/repos/fixture/shared/git/trees/${expiredTreeSha}?recursive=1`) - expect(resumedPaths).toContain('/repos/fixture/shared/git/trees/release%2Fcurrent?recursive=1') - expect(resumedPaths).not.toContain('/repos/fixture/shared/git/trees/trunk?recursive=1') - expect(resumedPaths).toContain(`/repos/fixture/shared/git/blobs/${shaFor(content)}`) - }) - - it('updates linked target content, skips unchanged hydration, and removes old chunks after target deletion', async () => { - const source = repositories.get('shared')! - source.files.set('target.md', 'Orion linked instructions revision one.') - source.symlinks.set('docs/link.md', '../target.md') - await db - .update(knowledgeConnector) - .set({ - sourceConfig: { repository: 'fixture/shared', pathPrefix: 'docs/link.md' }, - }) - .where(eq(knowledgeConnector.id, enrolled.connectorId)) - expect((await sync()).error).toBeUndefined() - const [linked] = await rows() - expect(linked.externalId).toBe('docs/link.md') - expect(await search(actor(ids.aliceId))).toEqual([linked.id]) - const getChunks = () => - listKnowledgeChunks.execute({ - principal: actor(ids.aliceId), - input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: linked.id }, - }) - expect((await getChunks()).chunks.map((chunk) => chunk.content).join('\n')).toBe( - 'Orion linked instructions revision one.' - ) - - source.files.set( - 'target.md', - 'Orion linked instructions revision two replaces the first revision.' - ) - expect((await sync()).error).toBeUndefined() - expect((await rows())[0].id).toBe(linked.id) - expect((await rows())[0].contentHash).not.toBe(linked.contentHash) - expect((await getChunks()).chunks.map((chunk) => chunk.content).join('\n')).toBe( - source.files.get('target.md') - ) - const requestOffset = requests.length - expect((await sync(enrolled.connectorId, false)).error).toBeUndefined() - expect( - requests.slice(requestOffset).some((request) => request.path.includes('/git/blobs/')) - ).toBe(false) - - source.files.delete('target.md') - expect((await sync()).error).toBeUndefined() - const [skipped] = await rows() - expect(skipped).toMatchObject({ - id: linked.id, - processingStatus: 'failed', - processingError: 'Symbolic link target is not a repository file', - }) - expect(await db.select().from(embedding).where(eq(embedding.documentId, linked.id))).toEqual([]) - await expect(getChunks()).rejects.toBeInstanceOf(KnowledgeDocumentNotReadyError) - expect(await search(actor(ids.aliceId))).toEqual([]) - expect(await search(actor(ids.bobId))).toEqual([]) - }) - - it('keeps previously observed files when GitHub reports a truncated tree', async () => { - const source = repositories.get('shared')! - source.files.set( - 'docs/omitted.md', - 'Orion complete guide: a partial tree must not withdraw this document.' - ) - await sync() - const before = await rows() - expect(await search(actor(ids.bobId))).toEqual(before.map((row) => row.id).sort()) - for (const row of before) await assertAccess(actor(ids.bobId), row, true) - source.files.delete('docs/omitted.md') - source.truncated = true - const result = await sync() - expect(result.membersIncomplete).toBe(2) - expect((await rows()).map((row) => row.id).sort()).toEqual(before.map((row) => row.id).sort()) - for (const row of before) await assertAccess(actor(ids.bobId), row, true) - expect(await search(actor(ids.bobId))).toEqual(before.map((row) => row.id).sort()) - }) - - it('refreshes an expired scopeless GitHub App token before real member indexing', async () => { - await db - .update(credential) - .set({ accessTokenExpiresAt: new Date(0) }) - .where(eq(credential.id, enrolled.members[0].credentialId)) - const result = await sync() - expect(result.error).toBeUndefined() - expect(result.membersCompleted).toBe(2) - expect(refreshedUsers).toEqual(new Set([ids.aliceId])) - const [stored] = await db - .select() - .from(credential) - .where(eq(credential.id, enrolled.members[0].credentialId)) - expect(stored.grantedScopes).toEqual([]) - expect(stored.managedOauthStatus).toBe('active') - expect(stored.accessTokenExpiresAt!.getTime()).toBeGreaterThan(Date.now()) - const [shared] = await rows() - expect(await search(actor(ids.aliceId))).toEqual([shared.id]) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts b/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts deleted file mode 100644 index cc07d015cda..00000000000 --- a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts +++ /dev/null @@ -1,646 +0,0 @@ -/** - * Fixture-backed Gmail HTTP and embeddings; PostgreSQL, managed token resolution, - * member sync, document storage, ACL materialization, and application reads are real. - * Fixtures follow users.threads.list/get and users.labels.list, not a live mailbox. - */ -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - document, - embedding, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { generateId } from '@sim/utils/id' -import { eq, inArray } from 'drizzle-orm' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const counters = vi.hoisted(() => ({ embeddedTexts: 0 })) -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => { - counters.embeddedTexts += texts.length - return { - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - } - }, -})) - -import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { env } from '@/lib/core/config/env' -import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' -import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' -import { encryptManagedOAuthTokenSet } from '@/lib/credentials/managed-oauth' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, - seedKnowledgeMemberFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' - -interface ThreadFixture { - id: string - historyId: string - text: string -} - -interface MailboxFixture { - token: string - refreshToken: string - threads: ThreadFixture[] - rejectToken: boolean - failSecondPage: boolean -} - -describe('Gmail member ingestion and ACLs in PostgreSQL (provider fixtures)', () => { - let ids: ReturnType - let fixture: Awaited> - let billing: Awaited> - let mailboxes: MailboxFixture[] - const requests: { member: number; path: string; format: string | null; cursor: string | null }[] = - [] - const storageKeys = new Set() - const previousClient = { id: env.GOOGLE_CLIENT_ID, secret: env.GOOGLE_CLIENT_SECRET } - const actor = (userId: string): Principal => ({ - kind: 'session', - userId, - sessionId: 'gmail-fixture', - }) - - function providerFetch(input: string | URL | Request, init?: RequestInit): Promise { - const url = new URL(input instanceof Request ? input.url : input) - const headers = new Headers( - init?.headers ?? (input instanceof Request ? input.headers : undefined) - ) - if (url.origin === 'https://oauth2.googleapis.com' && url.pathname === '/token') { - const body = new URLSearchParams(String(init?.body)) - const mailbox = mailboxes.find((item) => item.refreshToken === body.get('refresh_token')) - if (!mailbox || !mailbox.rejectToken) throw new Error('Unexpected fixture OAuth refresh') - return Promise.resolve(Response.json({ error: 'invalid_grant' }, { status: 400 })) - } - if (url.origin !== 'https://gmail.googleapis.com') { - throw new Error(`Unexpected outbound request: ${url.origin}${url.pathname}`) - } - const member = mailboxes.findIndex( - (item) => `Bearer ${item.token}` === headers.get('authorization') - ) - if (member === -1) throw new Error('Gmail fixture received an unknown credential') - const mailbox = mailboxes[member] - const cursor = url.searchParams.get('pageToken') - const format = url.searchParams.get('format') - requests.push({ member, path: url.pathname, format, cursor }) - if (mailbox.rejectToken) { - return Promise.resolve( - Response.json({ error: { code: 401, message: 'Invalid Credentials' } }, { status: 401 }) - ) - } - if (url.pathname === '/gmail/v1/users/me/profile') { - return Promise.resolve( - Response.json({ emailAddress: `mailbox-${member}@example.com`, historyId: '900' }) - ) - } - if (url.pathname === '/gmail/v1/users/me/labels') { - return Promise.resolve( - Response.json({ labels: [{ id: 'INBOX', name: 'INBOX', type: 'system' }] }) - ) - } - if (url.pathname === '/gmail/v1/users/me/threads') { - expect(url.searchParams.get('maxResults')).toBe('100') - expect(url.searchParams.get('q')).toContain('subject:Orion') - if (cursor === 'empty' && mailbox.failSecondPage) { - return Promise.resolve( - Response.json( - { error: { code: 400, message: 'Fixture page unavailable' } }, - { status: 400 } - ) - ) - } - /** An empty intermediate page must not reconcile absent documents before EOF. */ - if (cursor === 'empty') return Promise.resolve(Response.json({ nextPageToken: 'last' })) - const threads = cursor === 'last' ? mailbox.threads.slice(1) : mailbox.threads.slice(0, 1) - return Promise.resolve( - Response.json({ - threads: threads.map((thread) => ({ id: thread.id, snippet: 'Orion mailbox fixture' })), - ...(!cursor ? { nextPageToken: 'empty' } : {}), - }) - ) - } - const threadId = url.pathname.match(/^\/gmail\/v1\/users\/me\/threads\/([^/]+)$/)?.[1] - const thread = mailbox.threads.find((item) => item.id === threadId) - if (!thread) return Promise.resolve(Response.json({ error: { code: 404 } }, { status: 404 })) - const metadata = { - id: thread.id, - historyId: thread.historyId, - snippet: 'Orion mailbox fixture', - } - if (format === 'minimal') return Promise.resolve(Response.json(metadata)) - expect(format).toBe('full') - return Promise.resolve( - Response.json({ - ...metadata, - messages: [ - { - id: `message-${thread.id}`, - threadId: thread.id, - internalDate: '1700000000000', - labelIds: ['INBOX'], - payload: { - mimeType: 'text/plain', - headers: [ - { name: 'Subject', value: `Orion ${thread.id}` }, - { name: 'From', value: 'sender@example.com' }, - ], - body: { - data: Buffer.from(thread.text).toString('base64url'), - size: Buffer.byteLength(thread.text), - }, - }, - }, - ], - }) - ) - } - - beforeAll(() => { - Object.assign(env, { - GOOGLE_CLIENT_ID: 'gmail-fixture-client', - GOOGLE_CLIENT_SECRET: 'gmail-fixture-secret', - }) - vi.stubGlobal('fetch', providerFetch) - }) - - beforeEach(async () => { - ids = createKnowledgeAclFixtureIds() - await seedKnowledgeAclFixture(ids) - fixture = await seedKnowledgeMemberFixture(ids) - mailboxes = ['Alice', 'Bob'].map((name, index) => ({ - token: `fixture-gmail-${index}`, - refreshToken: `fixture-refresh-${index}`, - threads: [ - { - id: 'shared-thread-id', - historyId: '100', - text: `Orion ${name} private reply. Only ${name} can read this mailbox content.`, - }, - { - id: `private-${index}`, - historyId: '200', - text: `Orion ${name} separate thread with confidential mailbox details.`, - }, - ], - rejectToken: false, - failSecondPage: false, - })) - requests.length = 0 - counters.embeddedTexts = 0 - const policy = await getCredentialGroupProviderAdapter('gmail').getPolicy(undefined, { - workspaceId: ids.workspaceId, - credentialGroupId: fixture.groupId, - credentialGroupOptionId: fixture.optionId, - }) - await db - .update(credentialGroup) - .set({ - options: [ - { - id: fixture.optionId, - provider: 'gmail', - label: 'Gmail fixture', - authorizationAppId: policy.authorizationAppId, - requiredScopes: policy.requiredScopes, - scopeVersion: policy.scopeVersion, - required: false, - status: 'active', - }, - ], - }) - .where(eq(credentialGroup.id, fixture.groupId)) - await db - .update(knowledgeConnector) - .set({ - connectorType: 'gmail', - /** A fixed query keeps this full-listing suite separate from history-feed ingestion. */ - sourceConfig: { maxThreads: 0, query: 'subject:Orion' }, - status: 'active', - memberSyncStatus: 'idle', - memberSyncLockToken: null, - }) - .where(eq(knowledgeConnector.id, fixture.connectorId)) - for (const [index, member] of fixture.members.entries()) { - member.subjectToken = `s:google-email:fixture-domain:${member.userId}` - await db - .update(credential) - .set({ - providerId: 'google-email', - authorizationAppId: policy.authorizationAppId, - managedOauthScopeVersion: policy.scopeVersion, - grantedScopes: policy.requiredScopes, - encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ - accessToken: mailboxes[index].token, - refreshToken: mailboxes[index].refreshToken, - }), - accessTokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000), - }) - .where(eq(credential.id, member.credentialId)) - await db - .update(knowledgeConnectorMember) - .set({ subjectToken: member.subjectToken }) - .where(eq(knowledgeConnectorMember.id, member.id)) - } - await db - .insert(resourcePolicy) - .values({ - id: generateId(), - workspaceId: ids.workspaceId, - resourceType: 'credential_group', - resourceId: fixture.groupId, - document: compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: fixture.groupId, - allowedWorkflowIds: [], - }), - createdBy: ids.aliceId, - updatedBy: ids.aliceId, - }) - .onConflictDoNothing() - await grantKnowledgeConnectorCredentialAccess( - { - workspaceId: ids.workspaceId, - credentialGroupId: fixture.groupId, - credentialGroupOptionId: fixture.optionId, - connectorId: fixture.connectorId, - }, - ids.aliceId - ) - billing = await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - }) - - afterEach(async () => { - if (!ids) return - if (fixture) await stored() - try { - for (const key of storageKeys) { - await deleteFile({ key, context: 'knowledge-base' }).catch((error: unknown) => { - if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error - }) - } - } finally { - storageKeys.clear() - await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) - await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) - } - }) - - afterAll(async () => { - Object.assign(env, { - GOOGLE_CLIENT_ID: previousClient.id, - GOOGLE_CLIENT_SECRET: previousClient.secret, - }) - vi.unstubAllGlobals() - await db.$client.end() - }) - - async function stored() { - const rows = await db - .select() - .from(document) - .where(eq(document.connectorId, fixture.connectorId)) - for (const row of rows) if (row.storageKey) storageKeys.add(row.storageKey) - return rows - } - - async function vectors() { - return db - .select({ id: embedding.id, documentId: embedding.documentId, content: embedding.content }) - .from(embedding) - .innerJoin(document, eq(document.id, embedding.documentId)) - .where(eq(document.connectorId, fixture.connectorId)) - .orderBy(embedding.id) - } - - async function sync(healthy = true) { - await db - .update(knowledgeConnectorMember) - .set({ nextAttemptAt: new Date(0) }) - .where(eq(knowledgeConnectorMember.connectorId, fixture.connectorId)) - const result = await executeMemberSync(fixture.connectorId, { billingAttribution: billing }) - expect(result.error).toBeUndefined() - expect(result.skipReason).toBeUndefined() - if (healthy) { - expect(result.membersFailed).toBe(0) - expect(result.docsFailed).toBe(0) - expect(result.membersRemaining).toBe(false) - } - await stored() - return result - } - - async function search(principal: Principal, searchMode: 'hybrid' | 'vector' = 'hybrid') { - return ( - await searchKnowledge.execute({ - principal, - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - topK: 100, - searchMode, - }, - }) - ).results - } - - async function assertReadable( - memberIndex: number, - row: typeof document.$inferSelect, - allowed: boolean - ) { - const member = fixture.members[memberIndex] - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - const principal = actor(member.userId) - if (allowed) { - expect((await readKnowledgeDocument.execute({ principal, input })).document.id).toBe(row.id) - expect( - (await listKnowledgeChunks.execute({ principal, input })).chunks.length - ).toBeGreaterThan(0) - expect( - (await downloadFileFromUrl(row.fileUrl, { userId: member.userId, knowledgeAccess: 'user' })) - .length - ).toBeGreaterThan(0) - } else { - await expect(readKnowledgeDocument.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - await expect(listKnowledgeChunks.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - await expect( - downloadFileFromUrl(row.fileUrl, { userId: member.userId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - } - } - - it('persists separate mailbox content, observations, and ACLs for identical provider thread IDs', async () => { - expect((await sync()).membersCompleted).toBe(2) - const rows = await stored() - expect(rows).toHaveLength(4) - expect(rows.every((row) => row.processingStatus === 'completed')).toBe(true) - for (const [index, member] of fixture.members.entries()) { - const own = rows.filter((row) => row.externalId?.startsWith(`member:${member.id}:`)) - expect(new Set(own.map((row) => row.externalId))).toEqual( - new Set([`member:${member.id}:shared-thread-id`, `member:${member.id}:private-${index}`]) - ) - for (const row of own) { - expect(row.acl).toEqual([member.subjectToken]) - expect(new URL(row.sourceUrl!).searchParams.get('Email')).toBe( - `mailbox-${index}@example.com` - ) - } - const observations = await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, member.id)) - expect(new Set(observations.map((row) => row.documentId))).toEqual( - new Set(own.map((row) => row.id)) - ) - for (const mode of ['hybrid', 'vector'] as const) { - const results = await search(actor(member.userId), mode) - expect(new Set(results.map((row) => row.documentId))).toEqual( - new Set(own.map((row) => row.id)) - ) - for (const result of results) { - expect(new URL(result.sourceUrl!).searchParams.get('Email')).toBe( - `mailbox-${index}@example.com` - ) - } - expect(results.map((row) => row.content).join('\n')).toContain( - index === 0 ? 'Alice private reply' : 'Bob private reply' - ) - expect(results.map((row) => row.content).join('\n')).not.toContain( - index === 0 ? 'Bob private reply' : 'Alice private reply' - ) - } - for (const row of rows) - await assertReadable( - index, - row, - own.some((item) => item.id === row.id) - ) - expect( - requests.some((request) => request.member === index && request.cursor === 'last') - ).toBe(true) - expect( - requests.some((request) => request.member === index && request.format === 'minimal') - ).toBe(true) - } - expect( - await search({ kind: 'workspace_api_key', workspaceId: ids.workspaceId, keyId: 'fixture' }) - ).toEqual([]) - }) - - it('keeps unchanged embeddings and replaces only the changed member’s revision', async () => { - await sync() - const before = await vectors() - const calls = counters.embeddedTexts - await sync() - expect(await vectors()).toEqual(before) - expect(counters.embeddedTexts).toBe(calls) - mailboxes[0].threads[0] = { - ...mailboxes[0].threads[0], - historyId: '101', - text: 'Orion Alice updated reply after editing the mailbox.', - } - await sync() - const rows = await stored() - const alice = rows.find( - (row) => row.externalId === `member:${fixture.members[0].id}:shared-thread-id` - )! - expect(alice.contentHash).toBe('gmail:shared-thread-id:101:body-v2') - const after = await vectors() - expect(after.filter((row) => row.documentId !== alice.id)).toEqual( - before.filter((row) => row.documentId !== alice.id) - ) - expect( - after - .filter((row) => row.documentId === alice.id) - .map((row) => row.content) - .join('\n') - ).toContain('Alice updated reply') - expect( - after - .filter((row) => row.documentId === alice.id) - .map((row) => row.content) - .join('\n') - ).not.toContain('Alice private reply') - }) - - it('removes only the deleting mailbox’s grant after complete pagination', async () => { - await sync() - const rows = await stored() - const removed = rows.find( - (row) => row.externalId === `member:${fixture.members[0].id}:shared-thread-id` - )! - mailboxes[0].threads.shift() - await sync() - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, removed.id)) - ).toEqual([]) - expect((await search(actor(ids.aliceId))).some((row) => row.documentId === removed.id)).toBe( - false - ) - await assertReadable(0, removed, false) - const bobCopy = rows.find( - (row) => row.externalId === `member:${fixture.members[1].id}:shared-thread-id` - )! - await assertReadable(1, bobCopy, true) - expect((await search(actor(ids.bobId))).map((row) => row.content).join('\n')).toContain( - 'Bob private reply' - ) - }) - - it('preserves grants through a failed continuation and reconciles only after resuming to EOF', async () => { - await sync() - const alice = fixture.members[0] - const old = (await stored()).find((row) => row.externalId === `member:${alice.id}:private-0`)! - mailboxes[0].threads.pop() - mailboxes[0].failSecondPage = true - expect((await sync(false)).membersFailed).toBe(1) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, old.id)) - ).toHaveLength(1) - await assertReadable(0, old, true) - const [pending] = await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, alice.id)) - expect(pending.listingCheckpoint).toMatchObject({ cursor: expect.any(String), complete: false }) - mailboxes[0].failSecondPage = false - requests.length = 0 - await sync() - expect( - requests.find((request) => request.member === 0 && request.path.endsWith('/threads'))?.cursor - ).toBe('empty') - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, old.id)) - ).toEqual([]) - await assertReadable(0, old, false) - }) - - it('fails closed on provider rejection and restores the member after replacement credentials are saved', async () => { - await sync() - const alice = fixture.members[0] - const own = (await stored()).filter((row) => row.externalId?.startsWith(`member:${alice.id}:`)) - mailboxes[0].rejectToken = true - expect((await sync(false)).membersFailed).toBe(1) - const [rejected] = await db - .select() - .from(credential) - .where(eq(credential.id, alice.credentialId)) - expect(rejected.managedOauthStatus).toBe('needs_reauth') - expect(await search(actor(ids.aliceId))).toEqual([]) - for (const row of own) await assertReadable(0, row, false) - expect(await search(actor(ids.bobId))).toHaveLength(2) - mailboxes[0].rejectToken = false - mailboxes[0].token = 'fixture-reconnected-gmail' - /** The provider callback is outside this fixture; persist its encrypted token replacement. */ - await db - .update(credential) - .set({ - managedOauthStatus: 'active', - encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ - accessToken: mailboxes[0].token, - refreshToken: mailboxes[0].refreshToken, - }), - accessTokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000), - grantedAt: new Date(), - }) - .where(eq(credential.id, alice.credentialId)) - await sync() - expect(new Set((await search(actor(ids.aliceId))).map((row) => row.documentId))).toEqual( - new Set(own.map((row) => row.id)) - ) - for (const row of own) await assertReadable(0, row, true) - }) - - it('denies revoked enrollment immediately and keeps the other mailbox accessible', async () => { - await sync() - const bob = fixture.members[1] - const own = (await stored()).filter((row) => row.externalId?.startsWith(`member:${bob.id}:`)) - await db - .update(credentialGroupEnrollment) - .set({ status: 'revoked', revokedAt: new Date() }) - .where(eq(credentialGroupEnrollment.id, bob.enrollmentId)) - expect(await search(actor(ids.bobId))).toEqual([]) - for (const row of own) await assertReadable(1, row, false) - await sync() - const [suspended] = await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.id, bob.id)) - expect(suspended.status).toBe('suspended') - expect( - (await stored()) - .filter((row) => own.some((item) => item.id === row.id)) - .every((row) => row.acl.length === 0) - ).toBe(true) - expect(await search(actor(ids.aliceId))).toHaveLength(2) - }) - - it('denies stale mailbox observations until a fresh sync renews access without re-embedding', async () => { - await sync() - const alice = fixture.members[0] - const own = (await stored()).filter((row) => row.externalId?.startsWith(`member:${alice.id}:`)) - const before = await vectors() - const expiredAt = new Date(Date.now() - 25 * 60 * 60 * 1000) - await db - .update(knowledgeConnectorMember) - .set({ memberSyncedThrough: expiredAt }) - .where(eq(knowledgeConnectorMember.id, alice.id)) - await db - .update(knowledgeDocumentObservation) - .set({ lastSeenAt: expiredAt }) - .where(eq(knowledgeDocumentObservation.memberId, alice.id)) - expect(await search(actor(ids.aliceId))).toEqual([]) - for (const row of own) await assertReadable(0, row, false) - expect(await search(actor(ids.bobId))).toHaveLength(2) - const embeddingCalls = counters.embeddedTexts - await sync() - expect(await vectors()).toEqual(before) - expect(counters.embeddedTexts).toBe(embeddingCalls) - expect(new Set((await search(actor(ids.aliceId))).map((row) => row.documentId))).toEqual( - new Set(own.map((row) => row.id)) - ) - for (const row of own) await assertReadable(0, row, true) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts b/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts deleted file mode 100644 index 574c67c19a3..00000000000 --- a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts +++ /dev/null @@ -1,650 +0,0 @@ -/** - * Fixture-backed Google Calendar API responses; real managed credential authorization, - * member sync, storage, parsing, PostgreSQL observations and authorized reads. - * Only provider HTTP and embedding generation are substituted. No live Google Calendar - * account or application database is used. - */ -import { createHash } from 'node:crypto' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - document, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull } from 'drizzle-orm' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => ({ - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - }), -})) - -import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { env } from '@/lib/core/config/env' -import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' -import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' -import { encryptManagedOAuthTokenSet } from '@/lib/credentials/managed-oauth' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' - -const SHARED_CALENDAR = 'team@group.calendar.google.com' -const UPDATED = '2026-09-01T12:00:00Z' - -function event(id: string, summary: string, description: string) { - const start = new Date(Date.now() + 3_600_000).toISOString() - const end = new Date(Date.now() + 7_200_000).toISOString() - return { - id, - status: 'confirmed', - summary, - description, - location: 'Private meeting room', - visibility: 'default', - htmlLink: `https://calendar.google.com/calendar/event?eid=${id}`, - updated: UPDATED, - created: UPDATED, - start: { dateTime: start, timeZone: 'UTC' }, - end: { dateTime: end, timeZone: 'UTC' }, - organizer: { email: 'organizer@fixture.test', displayName: 'Provider Organizer' }, - attendees: [{ email: 'attendee@fixture.test', displayName: 'Provider Attendee' }], - } -} - -type CalendarEventFixture = - | ReturnType - | ReturnType - | { id: string; status: 'cancelled'; recurringEventId?: string } - -/** Models only the allowed timing fields of a provider-redacted event, not live captured JSON. */ -function redactedEvent(value: ReturnType) { - return { - id: value.id, - status: value.status, - start: value.start, - end: value.end, - updated: value.updated, - created: value.created, - } -} - -describe('Google Calendar member indexing and authorization in PostgreSQL', () => { - const ids = createKnowledgeAclFixtureIds() - const groupId = generateId() - const optionId = generateId() - const connectorId = generateId() - const people = [ids.aliceId, ids.bobId].map((userId) => ({ - userId, - credentialId: generateId(), - enrollmentId: generateId(), - accessToken: `fixture-calendar-${generateId()}`, - refreshToken: `fixture-calendar-refresh-${generateId()}`, - })) - const [alice, bob] = people - const previousClient = { id: env.GOOGLE_CLIENT_ID, secret: env.GOOGLE_CLIENT_SECRET } - const storageKeys = new Set() - const listing = new Map>() - const failures = new Map>() - const refreshAttempts: string[] = [] - const requests: { - token: string - calendarId: string - page: string | null - timeMin: string | null - timeMax: string | null - }[] = [] - let readerRole: 'reader' | 'freeBusyReader' = 'reader' - let privateEvent: ReturnType - let planningEvent: ReturnType - let billing: Awaited> - const actor = (userId: string): Principal => ({ - kind: 'session', - userId, - sessionId: 'fixture-calendar-member', - }) - const workspaceKey: Principal = { - kind: 'workspace_api_key', - workspaceId: ids.workspaceId, - keyId: 'fixture-calendar-workspace-key', - } - - beforeAll(async () => { - Object.assign(env, { - GOOGLE_CLIENT_ID: 'isolated-calendar-fixture-client', - GOOGLE_CLIENT_SECRET: 'isolated-calendar-fixture-secret', - }) - vi.stubGlobal('fetch', async (input: string | URL | Request, init?: RequestInit) => { - const url = new URL(input instanceof Request ? input.url : input.toString()) - if (url.href === 'https://oauth2.googleapis.com/token') { - expect(init?.method).toBe('POST') - const body = new URLSearchParams(String(init?.body)) - expect(body.get('grant_type')).toBe('refresh_token') - expect(body.get('client_id')).toBe('isolated-calendar-fixture-client') - const person = people.find((person) => person.refreshToken === body.get('refresh_token')) - if (!person || failures.get(person.accessToken)?.get('primary') !== 401) - throw new Error('Unexpected Calendar fixture token refresh') - refreshAttempts.push(person.userId) - return Response.json( - { error: 'invalid_grant', error_description: 'Token has been expired or revoked.' }, - { status: 400 } - ) - } - const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : {})) - const token = headers.get('Authorization')?.replace(/^Bearer /, '') ?? '' - if (!people.some((person) => person.accessToken === token)) - throw new Error('Unexpected provider token in Calendar fixture') - const match = url.pathname.match(/^\/calendar\/v3\/calendars\/([^/]+)\/events$/) - if (url.origin !== 'https://www.googleapis.com' || !match) - throw new Error('Unexpected outbound request in Calendar fixture') - const calendarId = decodeURIComponent(match[1]) - if (!['primary', SHARED_CALENDAR].includes(calendarId)) - throw new Error('Unexpected calendar in fixture') - expect(init?.method).toBe('GET') - expect(url.searchParams.get('singleEvents')).toBe('true') - expect(url.searchParams.get('orderBy')).toBe('startTime') - expect(url.searchParams.get('maxResults')).toBe('250') - const page = url.searchParams.get('pageToken') - const timeMin = url.searchParams.get('timeMin') - const timeMax = url.searchParams.get('timeMax') - expect(Date.parse(timeMin!)).toBeLessThan(Date.parse(timeMax!)) - requests.push({ token, calendarId, page, timeMin, timeMax }) - const failure = failures.get(token)?.get(calendarId) - if (failure) - return Response.json( - { - error: { - code: failure, - message: failure === 404 ? 'Not Found' : 'Invalid Credentials', - }, - }, - { status: failure } - ) - const visible = listing.get(token)?.get(calendarId) ?? [] - const accessRole = - calendarId === 'primary' || token === alice.accessToken ? 'owner' : readerRole - if (page === null && visible.length > 0) - return Response.json({ kind: 'calendar#events', accessRole, items: [], nextPageToken: '0' }) - const offset = page === null ? 0 : Number(page) - if (!Number.isSafeInteger(offset) || offset < 0) - throw new Error('Unexpected Calendar fixture continuation') - const next = offset + 1 - return Response.json({ - kind: 'calendar#events', - accessRole, - items: visible.slice(offset, next), - ...(next < visible.length ? { nextPageToken: String(next) } : {}), - }) - }) - await seedKnowledgeAclFixture(ids) - const policy = await getCredentialGroupProviderAdapter('google-calendar').getPolicy(undefined, { - workspaceId: ids.workspaceId, - }) - await db.insert(credentialGroup).values({ - id: groupId, - workspaceId: ids.workspaceId, - publicId: generateId(), - name: 'Calendar fixture accounts', - createdBy: ids.aliceId, - options: [ - { ...policy, id: optionId, label: 'Google Calendar', required: false, status: 'active' }, - ], - }) - await db - .insert(resourcePolicy) - .values({ - id: generateId(), - workspaceId: ids.workspaceId, - resourceType: 'credential_group', - resourceId: groupId, - document: compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: groupId, - allowedWorkflowIds: [], - }), - createdBy: ids.aliceId, - updatedBy: ids.aliceId, - }) - .onConflictDoNothing() - for (const person of people) { - await db.insert(credentialGroupEnrollment).values({ - id: person.enrollmentId, - credentialGroupId: groupId, - email: `${person.userId}@fixture.test`, - status: 'completed', - invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), - invitationExpiresAt: new Date(Date.now() + 3_600_000), - invitedAt: new Date(), - }) - await db.insert(credential).values({ - id: person.credentialId, - workspaceId: ids.workspaceId, - type: 'managed_oauth', - displayName: 'Calendar fixture', - providerId: 'google-calendar', - authorizationAppId: policy.authorizationAppId, - credentialGroupEnrollmentId: person.enrollmentId, - credentialGroupOptionId: optionId, - managedOauthScopeVersion: policy.scopeVersion, - providerSubjectId: person.userId, - providerTenantId: null, - managedOauthStatus: 'active', - grantedScopes: policy.requiredScopes, - encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ - accessToken: person.accessToken, - refreshToken: person.refreshToken, - }), - accessTokenExpiresAt: new Date(Date.now() + 3_600_000), - grantedAt: new Date(), - createdBy: person.userId, - }) - } - await db.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'google_calendar', - sourceConfig: { calendarId: ['primary', SHARED_CALENDAR] }, - accessMode: 'members', - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - status: 'active', - memberSyncStatus: 'idle', - syncIntervalMinutes: 60, - }) - await grantKnowledgeConnectorCredentialAccess( - { - workspaceId: ids.workspaceId, - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - connectorId, - }, - ids.aliceId - ) - billing = await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - }) - - beforeEach(async () => { - failures.clear() - requests.length = 0 - refreshAttempts.length = 0 - readerRole = 'reader' - privateEvent = { - ...event('collision', 'Orion confidential renewal', '

Confidential renewal terms

'), - visibility: 'private', - } - planningEvent = event('planning', 'Orion planning meeting', '

Planning roadmap

') - listing.set( - alice.accessToken, - new Map([ - ['primary', [event('collision', 'Orion Alice personal event', 'Alice personal notes')]], - [SHARED_CALENDAR, [privateEvent, planningEvent]], - ]) - ) - listing.set( - bob.accessToken, - new Map([ - ['primary', [event('collision', 'Orion Bob personal event', 'Bob personal notes')]], - [SHARED_CALENDAR, [redactedEvent(privateEvent), planningEvent]], - ]) - ) - await db - .update(knowledgeConnector) - .set({ sourceConfig: { calendarId: ['primary', SHARED_CALENDAR] } }) - .where(eq(knowledgeConnector.id, connectorId)) - await db - .update(credential) - .set({ managedOauthStatus: 'active', accessTokenExpiresAt: new Date(Date.now() + 3_600_000) }) - .where( - inArray( - credential.id, - people.map((person) => person.credentialId) - ) - ) - const result = await sync() - expect(result.membersCompleted).toBe(2) - expect(result.membersFailed).toBe(0) - }) - - afterAll(async () => { - try { - await stored() - for (const key of storageKeys) - await deleteFile({ key, context: 'knowledge-base' }).catch((error: unknown) => { - if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error - }) - } finally { - await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) - await db.delete(user).where( - inArray( - user.id, - people.map((person) => person.userId) - ) - ) - Object.assign(env, { - GOOGLE_CLIENT_ID: previousClient.id, - GOOGLE_CLIENT_SECRET: previousClient.secret, - }) - vi.unstubAllGlobals() - await db.$client.end() - } - }) - - async function stored() { - const rows = await db.select().from(document).where(eq(document.connectorId, connectorId)) - for (const row of rows) if (row.storageKey) storageKeys.add(row.storageKey) - return rows - } - - async function sync() { - await db - .update(knowledgeConnectorMember) - .set({ nextAttemptAt: new Date(0) }) - .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - const result = await executeMemberSync(connectorId, { - forceContentRefresh: true, - billingAttribution: billing, - }) - expect(result.error).toBeUndefined() - expect(result.skipReason).toBeUndefined() - expect(result.docsFailed).toBe(0) - expect(result.membersRemaining).toBe(false) - await stored() - return result - } - - async function member(person: (typeof people)[number]) { - const [row] = await db - .select() - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, connectorId), - eq(knowledgeConnectorMember.credentialId, person.credentialId) - ) - ) - expect(row).toBeDefined() - return row - } - - async function assertAccess(person: (typeof people)[number], externalIds: string[]) { - const identity = await member(person) - const rows = await stored() - const expected = rows.filter((row) => - externalIds.some((id) => row.externalId === `member:${identity.id}:${id}`) - ) - expect(expected).toHaveLength(externalIds.length) - const expectedIds = new Set(expected.map((row) => row.id)) - const search = await searchKnowledge.execute({ - principal: actor(person.userId), - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - searchMode: 'hybrid', - topK: 100, - }, - }) - expect(new Set(search.results.map((row) => row.documentId))).toEqual(expectedIds) - for (const row of rows) { - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - if (expectedIds.has(row.id)) { - expect( - (await readKnowledgeDocument.execute({ principal: actor(person.userId), input })).document - .id - ).toBe(row.id) - expect( - (await listKnowledgeChunks.execute({ principal: actor(person.userId), input })).chunks - .length - ).toBeGreaterThan(0) - expect( - ( - await downloadFileFromUrl(row.fileUrl, { - userId: person.userId, - knowledgeAccess: 'user', - }) - ).length - ).toBeGreaterThan(0) - } else { - await expect( - readKnowledgeDocument.execute({ principal: actor(person.userId), input }) - ).rejects.toThrow('Document not found') - await expect( - listKnowledgeChunks.execute({ principal: actor(person.userId), input }) - ).rejects.toThrow('Document not found') - await expect( - downloadFileFromUrl(row.fileUrl, { userId: person.userId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - } - } - } - - it('isolates owner and reader projections across two calendars and follows empty pages', async () => { - const rows = await stored() - expect(rows).toHaveLength(5) - const expected = [ - 'primary:collision', - `${SHARED_CALENDAR}:collision`, - `${SHARED_CALENDAR}:planning`, - ] - await assertAccess(alice, expected) - await assertAccess(bob, ['primary:collision', `${SHARED_CALENDAR}:planning`]) - for (const person of people) { - const pages = requests.filter((request) => request.token === person.accessToken) - expect(pages.map(({ calendarId, page }) => [calendarId, page])).toEqual([ - ['primary', null], - ['primary', '0'], - [SHARED_CALENDAR, null], - [SHARED_CALENDAR, '0'], - [SHARED_CALENDAR, '1'], - ]) - expect(new Set(pages.map(({ timeMin, timeMax }) => `${timeMin}:${timeMax}`)).size).toBe(1) - } - const observations = await db - .select() - .from(knowledgeDocumentObservation) - .where( - inArray( - knowledgeDocumentObservation.documentId, - rows.map((row) => row.id) - ) - ) - expect(observations).toHaveLength(5) - for (const row of rows) { - const evidence = observations.filter((observation) => observation.documentId === row.id) - expect(evidence).toHaveLength(1) - expect(row.externalId).toContain(`member:${evidence[0].memberId}:`) - expect(row.processingStatus).toBe('completed') - } - const own = await content(alice, `${SHARED_CALENDAR}:collision`) - expect(own).toContain('Confidential renewal terms') - expect(own).not.toContain('

') - const reader = await member(bob) - expect( - rows.some((row) => row.externalId === `member:${reader.id}:${SHARED_CALENDAR}:collision`) - ).toBe(false) - expect(await content(alice, 'primary:collision')).not.toContain('Bob personal notes') - expect(await content(bob, 'primary:collision')).not.toContain('Alice personal notes') - const search = await searchKnowledge.execute({ - principal: workspaceKey, - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - searchMode: 'hybrid', - topK: 100, - }, - }) - expect(search.results).toEqual([]) - for (const row of rows) { - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - await expect( - readKnowledgeDocument.execute({ principal: workspaceKey, input }) - ).rejects.toThrow('Document not found') - await expect(listKnowledgeChunks.execute({ principal: workspaceKey, input })).rejects.toThrow( - 'Document not found' - ) - } - }) - - async function rowFor(person: (typeof people)[number], externalId: string) { - const identity = await member(person) - const row = (await stored()).find( - (row) => row.externalId === `member:${identity.id}:${externalId}` - ) - expect(row).toBeDefined() - return row! - } - - async function content(person: (typeof people)[number], externalId: string) { - const row = await rowFor(person, externalId) - const chunks = await listKnowledgeChunks.execute({ - principal: actor(person.userId), - input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id }, - }) - return chunks.chunks.map((chunk) => chunk.content).join('\n') - } - - it('withdraws a reader projection reduced to free/busy data without an event update', async () => { - const previous = await rowFor(bob, `${SHARED_CALENDAR}:planning`) - readerRole = 'freeBusyReader' - listing - .get(bob.accessToken)! - .set(SHARED_CALENDAR, [ - listing.get(bob.accessToken)!.get(SHARED_CALENDAR)![0], - redactedEvent(planningEvent), - ]) - await sync() - await assertAccess(bob, ['primary:collision']) - const withdrawn = await rowFor(bob, `${SHARED_CALENDAR}:planning`) - expect(withdrawn.acl).toEqual([]) - expect(withdrawn.contentHash).toBe(previous.contentHash) - expect(withdrawn.deletedAt).not.toBeNull() - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, previous.id)) - ).toEqual([]) - expect(await content(alice, `${SHARED_CALENDAR}:planning`)).toContain('Planning roadmap') - }) - - it('updates content and removes deleted events and cancelled recurring instances', async () => { - const cancelled: CalendarEventFixture = { - id: 'planning', - status: 'cancelled', - recurringEventId: 'series', - } - listing.get(alice.accessToken)!.set('primary', []) - listing.get(alice.accessToken)!.set(SHARED_CALENDAR, [ - { - ...privateEvent, - description: 'Updated confidential renewal', - updated: '2026-09-02T12:00:00Z', - }, - cancelled, - ]) - listing - .get(bob.accessToken)! - .set(SHARED_CALENDAR, [listing.get(bob.accessToken)!.get(SHARED_CALENDAR)![0], cancelled]) - await sync() - await assertAccess(alice, [`${SHARED_CALENDAR}:collision`]) - await assertAccess(bob, ['primary:collision']) - expect(await content(alice, `${SHARED_CALENDAR}:collision`)).toContain( - 'Updated confidential renewal' - ) - const rows = await stored() - const observations = await db - .select() - .from(knowledgeDocumentObservation) - .where( - inArray( - knowledgeDocumentObservation.documentId, - rows.map((row) => row.id) - ) - ) - expect(observations).toHaveLength(2) - }) - - it('withdraws a lost shared calendar while preserving the member primary calendar', async () => { - failures.set(bob.accessToken, new Map([[SHARED_CALENDAR, 404]])) - const result = await sync() - expect(result.membersFailed).toBe(0) - await assertAccess(bob, ['primary:collision']) - await assertAccess(alice, [ - 'primary:collision', - `${SHARED_CALENDAR}:collision`, - `${SHARED_CALENDAR}:planning`, - ]) - const identity = await member(bob) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, identity.id)) - ).toHaveLength(1) - }) - - it('removes organizer and attendee identifiers when the source disables them', async () => { - await db - .update(knowledgeConnector) - .set({ - sourceConfig: { calendarId: ['primary', SHARED_CALENDAR], includeAttendees: 'false' }, - }) - .where(eq(knowledgeConnector.id, connectorId)) - await sync() - const text = await content(alice, `${SHARED_CALENDAR}:collision`) - expect(text).toContain('Attendees: 1') - expect(text).not.toContain('organizer@fixture.test') - expect(text).not.toContain('Provider Organizer') - expect(text).not.toContain('Provider Attendee') - expect(text).toContain('Confidential renewal terms') - }) - - it('marks rejected OAuth credentials for reconnect and hides only the affected member', async () => { - failures.set(bob.accessToken, new Map([['primary', 401]])) - const result = await sync() - expect(result.membersFailed).toBe(1) - expect(refreshAttempts).toEqual([bob.userId]) - const [saved] = await db.select().from(credential).where(eq(credential.id, bob.credentialId)) - expect(saved.managedOauthStatus).toBe('needs_reauth') - await assertAccess(bob, []) - await assertAccess(alice, [ - 'primary:collision', - `${SHARED_CALENDAR}:collision`, - `${SHARED_CALENDAR}:planning`, - ]) - expect( - await db - .select() - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.deletedAt))) - ).toHaveLength(5) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts b/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts deleted file mode 100644 index 8672778309b..00000000000 --- a/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts +++ /dev/null @@ -1,655 +0,0 @@ -/** - * Fixture-backed Jira API responses; real managed credential authorization, - * member sync, storage, parsing, PostgreSQL observations and authorized reads. - * Only provider HTTP and embedding generation are substituted. No live Jira - * account or application database is used. - */ -import { createHash } from 'node:crypto' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - document, - embedding, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - resourcePolicy, - user, - workspace, -} from '@sim/db/schema' -import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' -import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNull } from 'drizzle-orm' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => ({ - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - }), -})) - -import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { env } from '@/lib/core/config/env' -import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' -import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' -import { encryptManagedOAuthTokenSet } from '@/lib/credentials/managed-oauth' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' - -const CLOUD_ID = 'jira-fixture-cloud' -const DOMAIN = 'fixture.atlassian.net' -const UPDATED = '2026-09-01T12:00:00.000+0000' - -function issue(id: string, description: string, projectKey = 'ENG') { - return { - id, - key: `${projectKey}-${id}`, - fields: { - summary: `Orion issue ${id}`, - description: { - type: 'doc', - version: 1, - content: [{ type: 'paragraph', content: [{ type: 'text', text: description }] }], - }, - project: { id: projectKey === 'ENG' ? '10000' : '20000', key: projectKey }, - updated: UPDATED, - created: UPDATED, - labels: ['orion'], - /** The indexed member projection must ignore comments even if returned. */ - comment: { total: 1, comments: [{ body: 'Never index this restricted comment' }] }, - }, - } -} - -describe('Jira member indexing and authorization in PostgreSQL', () => { - const ids = createKnowledgeAclFixtureIds() - const groupId = generateId() - const optionId = generateId() - const connectorId = generateId() - const people = [ids.aliceId, ids.bobId].map((userId) => ({ - userId, - credentialId: generateId(), - enrollmentId: generateId(), - accessToken: `fixture-jira-${generateId()}`, - refreshToken: `fixture-jira-refresh-${generateId()}`, - })) - const [alice, bob] = people - const previousClient = { id: env.JIRA_CLIENT_ID, secret: env.JIRA_CLIENT_SECRET } - const storageKeys = new Set() - const listing = new Map[]>() - const failures = new Map() - const projectFailures = new Map>() - const revokeOnPage = new Map() - const requests: { token: string; projectKey: string; page: string | null }[] = [] - const refreshAttempts: string[] = [] - let refreshError: 'invalid_grant' | 'unauthorized_client' = 'invalid_grant' - let billing: Awaited> - const actor = (userId: string): Principal => ({ - kind: 'session', - userId, - sessionId: 'fixture-jira-member', - }) - const workspaceKey: Principal = { - kind: 'workspace_api_key', - workspaceId: ids.workspaceId, - keyId: 'fixture-jira-workspace-key', - } - - beforeAll(async () => { - Object.assign(env, { - JIRA_CLIENT_ID: 'isolated-jira-fixture-client', - JIRA_CLIENT_SECRET: 'isolated-jira-fixture-secret', - }) - vi.stubGlobal('fetch', async (input: string | URL | Request, init?: RequestInit) => { - const url = new URL(input instanceof Request ? input.url : input.toString()) - if (url.href === 'https://auth.atlassian.com/oauth/token') { - expect(init?.method).toBe('POST') - const body = new URLSearchParams(String(init?.body)) - expect(body.get('grant_type')).toBe('refresh_token') - const person = people.find((person) => person.refreshToken === body.get('refresh_token')) - if (!person || failures.get(person.accessToken) !== 401) - throw new Error('Unexpected Jira fixture token refresh') - refreshAttempts.push(person.userId) - return Response.json({ error: refreshError }, { status: 403 }) - } - const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : {})) - const token = headers.get('Authorization')?.replace(/^Bearer /, '') ?? '' - if (!people.some((person) => person.accessToken === token)) - throw new Error('Unexpected provider token in Jira fixture') - if (url.origin !== 'https://api.atlassian.com') - throw new Error('Unexpected outbound origin in Jira fixture') - if (url.pathname === '/oauth/token/accessible-resources') - return Response.json([{ id: CLOUD_ID, url: `https://${DOMAIN}` }]) - if (url.pathname !== `/ex/jira/${CLOUD_ID}/rest/api/3/search/jql`) - throw new Error('Unexpected Jira fixture endpoint') - expect(init?.method).toBe('GET') - const projectMatch = url.searchParams - .get('jql') - ?.match(/^project = "([^"]+)" ORDER BY updated DESC$/) - if (!projectMatch) throw new Error('Unexpected Jira fixture project query') - const projectKey = projectMatch[1] - expect(url.searchParams.get('fields')?.split(',')).toContain('description') - expect(url.searchParams.get('fields')?.split(',')).not.toContain('comment') - const page = url.searchParams.get('nextPageToken') - requests.push({ token, projectKey, page }) - const revocation = revokeOnPage.get(token) - if (revocation?.projectKey === projectKey && revocation.page === page) { - projectFailures.set(token, { ...projectFailures.get(token), [projectKey]: 400 }) - revokeOnPage.delete(token) - } - const failure = failures.get(token) ?? projectFailures.get(token)?.[projectKey] - if (failure) - return Response.json( - { - errorMessages: [ - failure === 400 - ? `The value '${projectKey}' does not exist for the field 'project'.` - : 'Unauthorized', - ], - }, - { status: failure } - ) - const visible = (listing.get(token) ?? []).filter( - (issue) => issue.fields.project.key === projectKey - ) - const offset = page === null ? 0 : Number(page) - if (!Number.isSafeInteger(offset) || offset < 0) - throw new Error('Unexpected Jira fixture continuation') - const next = offset + 1 - return Response.json({ - issues: visible.slice(offset, next), - isLast: next >= visible.length, - ...(next < visible.length ? { nextPageToken: String(next) } : {}), - }) - }) - await seedKnowledgeAclFixture(ids) - const policy = await getCredentialGroupProviderAdapter('jira').getPolicy(undefined, { - workspaceId: ids.workspaceId, - }) - await db.insert(credentialGroup).values({ - id: groupId, - workspaceId: ids.workspaceId, - publicId: generateId(), - name: 'Jira fixture accounts', - createdBy: ids.aliceId, - options: [{ ...policy, id: optionId, label: 'Jira', required: false, status: 'active' }], - }) - await db - .insert(resourcePolicy) - .values({ - id: generateId(), - workspaceId: ids.workspaceId, - resourceType: 'credential_group', - resourceId: groupId, - document: compileCredentialGroupWorkflowAccessPolicy({ - credentialGroupId: groupId, - allowedWorkflowIds: [], - }), - createdBy: ids.aliceId, - updatedBy: ids.aliceId, - }) - .onConflictDoNothing() - for (const person of people) { - await db.insert(credentialGroupEnrollment).values({ - id: person.enrollmentId, - credentialGroupId: groupId, - email: `${person.userId}@fixture.test`, - status: 'completed', - invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), - invitationExpiresAt: new Date(Date.now() + 3_600_000), - invitedAt: new Date(), - }) - await db.insert(credential).values({ - id: person.credentialId, - workspaceId: ids.workspaceId, - type: 'managed_oauth', - displayName: 'Jira fixture', - providerId: 'jira', - authorizationAppId: policy.authorizationAppId, - credentialGroupEnrollmentId: person.enrollmentId, - credentialGroupOptionId: optionId, - managedOauthScopeVersion: policy.scopeVersion, - providerSubjectId: person.userId, - providerTenantId: null, - managedOauthStatus: 'active', - grantedScopes: policy.requiredScopes, - encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ - accessToken: person.accessToken, - refreshToken: person.refreshToken, - }), - accessTokenExpiresAt: new Date(Date.now() + 3_600_000), - grantedAt: new Date(), - createdBy: person.userId, - }) - } - await db.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'jira', - sourceConfig: { domain: DOMAIN, projectKey: 'ENG' }, - accessMode: 'members', - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - status: 'active', - memberSyncStatus: 'idle', - syncIntervalMinutes: 60, - }) - await grantKnowledgeConnectorCredentialAccess( - { - workspaceId: ids.workspaceId, - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - connectorId, - }, - ids.aliceId - ) - billing = await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - }) - - beforeEach(async () => { - refreshError = 'invalid_grant' - failures.clear() - projectFailures.clear() - revokeOnPage.clear() - requests.length = 0 - refreshAttempts.length = 0 - listing.set(alice.accessToken, [ - issue('1', 'Orion restricted Alice projection'), - issue('2', 'Orion private Alice issue'), - ]) - listing.set(bob.accessToken, [ - issue('1', 'Orion limited Bob projection'), - issue('3', 'Orion private Bob issue'), - ]) - await db - .update(knowledgeConnector) - .set({ sourceConfig: { domain: DOMAIN, projectKey: 'ENG' } }) - .where(eq(knowledgeConnector.id, connectorId)) - await db - .update(credential) - .set({ managedOauthStatus: 'active', accessTokenExpiresAt: new Date(Date.now() + 3_600_000) }) - .where( - inArray( - credential.id, - people.map((person) => person.credentialId) - ) - ) - const result = await sync() - expect(result.membersCompleted).toBe(2) - expect(result.membersFailed).toBe(0) - }) - - afterAll(async () => { - try { - await stored() - for (const key of storageKeys) - await deleteFile({ key, context: 'knowledge-base' }).catch((error: unknown) => { - if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error - }) - } finally { - await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId)) - await db.delete(user).where( - inArray( - user.id, - people.map((person) => person.userId) - ) - ) - Object.assign(env, { - JIRA_CLIENT_ID: previousClient.id, - JIRA_CLIENT_SECRET: previousClient.secret, - }) - vi.unstubAllGlobals() - await db.$client.end() - } - }) - - async function stored() { - const rows = await db.select().from(document).where(eq(document.connectorId, connectorId)) - for (const row of rows) if (row.storageKey) storageKeys.add(row.storageKey) - return rows - } - - async function sync() { - await db - .update(knowledgeConnectorMember) - .set({ nextAttemptAt: new Date(0) }) - .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - const result = await executeMemberSync(connectorId, { - forceContentRefresh: true, - billingAttribution: billing, - }) - expect(result.error).toBeUndefined() - expect(result.skipReason).toBeUndefined() - expect(result.docsFailed).toBe(0) - expect(result.membersRemaining).toBe(false) - await stored() - return result - } - - async function member(person: (typeof people)[number]) { - const [row] = await db - .select() - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, connectorId), - eq(knowledgeConnectorMember.credentialId, person.credentialId) - ) - ) - expect(row).toBeDefined() - return row - } - - async function assertAccess(person: (typeof people)[number], issueIds: string[]) { - const identity = await member(person) - const rows = await stored() - const expected = rows.filter((row) => - issueIds.some((id) => row.externalId === `member:${identity.id}:jira:${CLOUD_ID}:${id}`) - ) - expect(expected).toHaveLength(issueIds.length) - const expectedIds = new Set(expected.map((row) => row.id)) - const search = await searchKnowledge.execute({ - principal: actor(person.userId), - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - searchMode: 'hybrid', - topK: 100, - }, - }) - expect(new Set(search.results.map((row) => row.documentId))).toEqual(expectedIds) - for (const row of rows) { - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - if (expectedIds.has(row.id)) { - expect( - (await readKnowledgeDocument.execute({ principal: actor(person.userId), input })).document - .id - ).toBe(row.id) - expect( - (await listKnowledgeChunks.execute({ principal: actor(person.userId), input })).chunks - .length - ).toBeGreaterThan(0) - expect( - ( - await downloadFileFromUrl(row.fileUrl, { - userId: person.userId, - knowledgeAccess: 'user', - }) - ).length - ).toBeGreaterThan(0) - } else { - await expect( - readKnowledgeDocument.execute({ principal: actor(person.userId), input }) - ).rejects.toThrow('Document not found') - await expect( - listKnowledgeChunks.execute({ principal: actor(person.userId), input }) - ).rejects.toThrow('Document not found') - await expect( - downloadFileFromUrl(row.fileUrl, { userId: person.userId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - } - } - } - - it('persists both pages and isolates different projections of one Jira issue across all reads', async () => { - expect(await stored()).toHaveLength(4) - for (const person of people) - expect( - requests - .filter((request) => request.token === person.accessToken) - .map((request) => request.page) - ).toEqual([null, '1']) - await assertAccess(alice, ['1', '2']) - await assertAccess(bob, ['1', '3']) - const rows = await stored() - const observations = await db - .select() - .from(knowledgeDocumentObservation) - .where( - inArray( - knowledgeDocumentObservation.documentId, - rows.map((row) => row.id) - ) - ) - expect(observations).toHaveLength(4) - for (const row of rows) { - const evidence = observations.filter((observation) => observation.documentId === row.id) - expect(evidence).toHaveLength(1) - expect(row.externalId).toContain(`member:${evidence[0].memberId}:`) - expect(row.processingStatus).toBe('completed') - } - const chunks = await db - .select() - .from(embedding) - .where( - inArray( - embedding.documentId, - rows.map((row) => row.id) - ) - ) - expect(chunks.map((chunk) => chunk.content).join('\n')).not.toContain('restricted comment') - for (const [person, visibleText, hiddenText] of [ - [alice, 'restricted Alice projection', 'limited Bob projection'], - [bob, 'limited Bob projection', 'restricted Alice projection'], - ] as const) { - const identity = await member(person) - const projection = rows.find( - (row) => row.externalId === `member:${identity.id}:jira:${CLOUD_ID}:1` - )! - const content = chunks - .filter((chunk) => chunk.documentId === projection.id) - .map((chunk) => chunk.content) - .join('\n') - expect(content).toContain(visibleText) - expect(content).not.toContain(hiddenText) - } - const search = await searchKnowledge.execute({ - principal: workspaceKey, - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - searchMode: 'hybrid', - topK: 100, - }, - }) - expect(search.results).toEqual([]) - for (const row of rows) { - const input = { knowledgeBaseId: ids.knowledgeBaseId, documentId: row.id } - await expect( - readKnowledgeDocument.execute({ principal: workspaceKey, input }) - ).rejects.toThrow('Document not found') - await expect(listKnowledgeChunks.execute({ principal: workspaceKey, input })).rejects.toThrow( - 'Document not found' - ) - } - }) - - it('replaces previously visible fields without a Jira updated timestamp change', async () => { - const identity = await member(alice) - const previous = (await stored()).find( - (row) => row.externalId === `member:${identity.id}:jira:${CLOUD_ID}:1` - )! - listing.set(alice.accessToken, [ - issue('1', 'Orion reduced projection'), - issue('2', 'Orion private Alice issue'), - ]) - await sync() - const current = (await stored()).find((row) => row.id === previous.id)! - expect(current.contentHash).not.toBe(previous.contentHash) - const chunks = await db.select().from(embedding).where(eq(embedding.documentId, current.id)) - const content = chunks.map((chunk) => chunk.content).join('\n') - expect(content).toContain('reduced projection') - expect(content).not.toContain('restricted Alice projection') - await assertAccess(alice, ['1', '2']) - await assertAccess(bob, ['1', '3']) - }) - - it('withdraws an issue absent from a complete listing without affecting the other member', async () => { - const identity = await member(alice) - const removed = (await stored()).find( - (row) => row.externalId === `member:${identity.id}:jira:${CLOUD_ID}:2` - )! - listing.set(alice.accessToken, [issue('1', 'Orion restricted Alice projection')]) - await sync() - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, removed.id)) - ).toEqual([]) - await assertAccess(alice, ['1']) - await assertAccess(bob, ['1', '3']) - }) - - it('withdraws project access on Jira project-not-accessible responses', async () => { - failures.set(bob.accessToken, 400) - const result = await sync() - expect(result.membersFailed).toBe(0) - const identity = await member(bob) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.memberId, identity.id)) - ).toEqual([]) - await assertAccess(bob, []) - await assertAccess(alice, ['1', '2']) - }) - - it('retains accessible projects while withdrawing either denied project in a multi-project source', async () => { - await db - .update(knowledgeConnector) - .set({ sourceConfig: { domain: DOMAIN, projectKey: ['ENG', 'SUPPORT'] } }) - .where(eq(knowledgeConnector.id, connectorId)) - listing.get(alice.accessToken)!.push(issue('4', 'Orion Alice support issue', 'SUPPORT')) - listing.get(bob.accessToken)!.push(issue('5', 'Orion Bob support issue', 'SUPPORT')) - await sync() - await assertAccess(bob, ['1', '3', '5']) - - projectFailures.set(bob.accessToken, { SUPPORT: 400 }) - expect((await sync()).membersFailed).toBe(0) - await assertAccess(alice, ['1', '2', '4']) - await assertAccess(bob, ['1', '3']) - const identity = await member(bob) - const withdrawn = (await stored()).find( - (row) => row.externalId === `member:${identity.id}:jira:${CLOUD_ID}:5` - )! - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, withdrawn.id)) - ).toEqual([]) - - projectFailures.set(bob.accessToken, { ENG: 400 }) - expect((await sync()).membersFailed).toBe(0) - await assertAccess(bob, ['5']) - await assertAccess(alice, ['1', '2', '4']) - - projectFailures.clear() - await sync() - await assertAccess(bob, ['1', '3', '5']) - projectFailures.set(bob.accessToken, { ENG: 400, SUPPORT: 400 }) - expect((await sync()).membersFailed).toBe(0) - await assertAccess(bob, []) - await assertAccess(alice, ['1', '2', '4']) - }) - - it('withdraws earlier observations when project access is revoked during pagination', async () => { - await db - .update(knowledgeConnector) - .set({ sourceConfig: { domain: DOMAIN, projectKey: ['ENG', 'SUPPORT'] } }) - .where(eq(knowledgeConnector.id, connectorId)) - listing - .get(bob.accessToken)! - .push( - issue('5', 'Orion Bob support issue', 'SUPPORT'), - issue('6', 'Orion Bob second support issue', 'SUPPORT') - ) - await sync() - await assertAccess(bob, ['1', '3', '5', '6']) - revokeOnPage.set(bob.accessToken, { projectKey: 'SUPPORT', page: '1' }) - requests.length = 0 - - expect((await sync()).membersFailed).toBe(0) - await assertAccess(bob, ['1', '3']) - expect( - requests - .filter((request) => request.token === bob.accessToken) - .map(({ projectKey, page }) => ({ projectKey, page })) - ).toEqual([ - { projectKey: 'ENG', page: null }, - { projectKey: 'ENG', page: '1' }, - { projectKey: 'SUPPORT', page: null }, - { projectKey: 'SUPPORT', page: '1' }, - { projectKey: 'ENG', page: null }, - { projectKey: 'ENG', page: '1' }, - { projectKey: 'SUPPORT', page: null }, - ]) - }) - - it.each([403, 500] as const)( - 'preserves project observations after unconfirmed HTTP %s failure', - async (status) => { - await db - .update(knowledgeConnector) - .set({ sourceConfig: { domain: DOMAIN, projectKey: ['ENG', 'SUPPORT'] } }) - .where(eq(knowledgeConnector.id, connectorId)) - listing.get(bob.accessToken)!.push(issue('5', 'Orion Bob support issue', 'SUPPORT')) - await sync() - projectFailures.set(bob.accessToken, { SUPPORT: status }) - - expect((await sync()).membersFailed).toBe(1) - await assertAccess(bob, ['1', '3', '5']) - await assertAccess(alice, ['1', '2']) - } - ) - - it.each(['invalid_grant', 'unauthorized_client'] as const)( - 'marks a %s refresh rejection for reconnect and immediately hides that member corpus', - async (errorCode) => { - refreshError = errorCode - failures.set(alice.accessToken, 401) - const result = await sync() - expect(result.membersFailed).toBe(1) - expect(refreshAttempts).toEqual([alice.userId]) - const [saved] = await db - .select() - .from(credential) - .where(eq(credential.id, alice.credentialId)) - expect(saved.managedOauthStatus).toBe('needs_reauth') - await assertAccess(alice, []) - await assertAccess(bob, ['1', '3']) - expect( - await db - .select() - .from(document) - .where(and(eq(document.connectorId, connectorId), isNull(document.deletedAt))) - ).toHaveLength(4) - } - ) -}) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index 64a402616a9..e22c9dd3b19 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -37,25 +37,7 @@ export default defineConfig(({ mode }) => { : live ? ['**/*.live.test.ts'] : ['**/*.test.{ts,tsx}'], - exclude: [ - ...(integration || live ? [] : ['**/*.live.test.ts']), - /** - * Quarantined: these suites already failed against a freshly provisioned database before - * the integration layer was collected by glob (their fixtures predate connector and - * credential changes). Fix and remove them from this list; never add a passing suite here. - */ - ...(integration - ? [ - 'lib/credentials/__integration__/organization-personal-tokens.integration.ts', - 'lib/knowledge/__integration__/confluence-enrollment.integration.ts', - 'lib/knowledge/__integration__/excluded-member-documents.integration.ts', - 'lib/knowledge/__integration__/github-member.integration.ts', - 'lib/knowledge/__integration__/gmail-member.integration.ts', - 'lib/knowledge/__integration__/google-calendar-member.integration.ts', - 'lib/knowledge/__integration__/jira-member.integration.ts', - ] - : []), - ], + exclude: integration || live ? [] : ['**/*.live.test.ts'], setupFiles: integration ? ['./vitest.integration.setup.ts'] : ['./vitest.setup.ts'], pool: 'threads', fileParallelism: !integration, diff --git a/packages/testing/src/mocks/credential-groups-enrollments.mock.ts b/packages/testing/src/mocks/credential-groups-enrollments.mock.ts index 96c7f72278d..3ba0506dae8 100644 --- a/packages/testing/src/mocks/credential-groups-enrollments.mock.ts +++ b/packages/testing/src/mocks/credential-groups-enrollments.mock.ts @@ -46,7 +46,6 @@ export const credentialGroupsEnrollmentsMockFns = { mockGetAuthorizedPublicCredentialGroupEnrollment: vi.fn(), mockCompleteCredentialGroupEnrollment: vi.fn(), mockCompleteAuthorizedCredentialGroupEnrollment: vi.fn(), - mockGetCredentialGroupOAuthContext: vi.fn(), mockGetAuthorizedCredentialGroupOAuthContext: vi.fn(), mockGetCredentialGroupOAuthContextForEnrollment: vi.fn(), mockGetAuthorizedCredentialGroupMcpOAuthContext: vi.fn(), @@ -81,7 +80,6 @@ export const credentialGroupsEnrollmentsMock = { fns.mockGetAuthorizedPublicCredentialGroupEnrollment, completeCredentialGroupEnrollment: fns.mockCompleteCredentialGroupEnrollment, completeAuthorizedCredentialGroupEnrollment: fns.mockCompleteAuthorizedCredentialGroupEnrollment, - getCredentialGroupOAuthContext: fns.mockGetCredentialGroupOAuthContext, getAuthorizedCredentialGroupOAuthContext: fns.mockGetAuthorizedCredentialGroupOAuthContext, getCredentialGroupOAuthContextForEnrollment: fns.mockGetCredentialGroupOAuthContextForEnrollment, getAuthorizedCredentialGroupMcpOAuthContext: fns.mockGetAuthorizedCredentialGroupMcpOAuthContext, diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 5934cde0a94..ca5f3557445 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1407,7 +1407,6 @@ "apps/sim/lib/credential-groups/credentials.test.ts": 2, "apps/sim/lib/credential-groups/enrollments.ts": 1, "apps/sim/lib/credential-groups/oauth-state.test.ts": 2, - "apps/sim/lib/credentials/__integration__/organization-personal-tokens.integration.ts": 5, "apps/sim/lib/credentials/access.ts": 1, "apps/sim/lib/credentials/application/discover-managed-mcp-tools.test.ts": 1, "apps/sim/lib/credentials/application/organization-credentials.ts": 1, @@ -1457,7 +1456,6 @@ "apps/sim/lib/internal/tool-operations/file-result.ts": 1, "apps/sim/lib/internal/vision/operations.ts": 1, "apps/sim/lib/knowledge/__integration__/coda-live.integration.ts": 8, - "apps/sim/lib/knowledge/__integration__/confluence-enrollment.integration.ts": 9, "apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts": 1, "apps/sim/lib/knowledge/__integration__/confluence-live.integration.ts": 2, "apps/sim/lib/knowledge/__integration__/connector-deferral.integration.ts": 2, @@ -1468,16 +1466,12 @@ "apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts": 2, "apps/sim/lib/knowledge/__integration__/directory-sync.integration.ts": 5, "apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts": 1, - "apps/sim/lib/knowledge/__integration__/github-member.integration.ts": 39, "apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts": 42, "apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts": 11, "apps/sim/lib/knowledge/__integration__/gitlab-workspace.integration.ts": 2, - "apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts": 6, - "apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts": 11, "apps/sim/lib/knowledge/__integration__/google-drive-live.integration.ts": 17, "apps/sim/lib/knowledge/__integration__/google-drive-member-live.integration.ts": 5, "apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts": 2, - "apps/sim/lib/knowledge/__integration__/jira-member.integration.ts": 9, "apps/sim/lib/knowledge/__integration__/knowledge-base-deletion.integration.ts": 1, "apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts": 3, "apps/sim/lib/knowledge/__integration__/listing-continuation.integration.ts": 22,