OUT-4176: support Dropbox uploads over 150 MiB via upload sessions

src/features/sync/constant.ts

@@ -7,3 +7,9 @@ export const DBX_URL_PATH = {
   fileUpload: '/files/upload',
   fileDownload: '/files/download',
 }
+
+// Files above this use an upload session, not /files/upload.
+export const DROPBOX_SINGLE_UPLOAD_MAX_BYTES = 150 * 1024 * 1024
+
+// Bytes per upload-session append (a multiple of 4 MiB, per Dropbox).
+export const DROPBOX_UPLOAD_CHUNK_BYTES = 16 * 1024 * 1024

src/features/sync/lib/Sync.service.ts

@@ -16,7 +16,11 @@ import {
   fileFolderSync,
 } from '@/db/schema/fileFolderSync.schema'
 import APIError from '@/errors/APIError'
-import { DBX_URL_PATH } from '@/features/sync/constant'
+import {
+  DBX_URL_PATH,
+  DROPBOX_SINGLE_UPLOAD_MAX_BYTES,
+  DROPBOX_UPLOAD_CHUNK_BYTES,
+} from '@/features/sync/constant'
 import { MapFilesService } from '@/features/sync/lib/MapFiles.service'
 import type {
   AssemblyToDropboxSyncFilesPayload,
@@ -982,27 +986,62 @@ export class SyncService extends AuthenticatedDropboxService {
 
   private async uploadFileInDropbox(file: CopilotFileRetrieve, path: string) {
     logger.info('SyncService#uploadFileInDropbox :: Uploading file to', path)
-    if (file.downloadUrl) {
-      // download file from Assembly
-      const resp = await fetch(file.downloadUrl)
-      // upload file to dropbox
-      const dbxResponse = await this.dbxClient.uploadFile({
-        urlPath: DBX_URL_PATH.fileUpload,
-        filePath: path,
-        body: resp.body,
-        rootNamespaceId: z.string().parse(this.connectionToken.rootNamespaceId),
-        refreshToken: this.connectionToken.refreshToken,
-      })
-      logger.info('SyncService#uploadFileInDropbox :: File uploaded to', path)
-      return {
-        dbxFileId: dbxResponse.id,
-        contentHash: dbxResponse.contentHash,
-      }
+    if (!file.downloadUrl) {
+      logger.error(
+        `SyncService#uploadFileInDropbox. Assembly file with Id: ${file.id} has no download url. Channel ID: ${file.channelId}`,
+      )
+      throw new Error('File not found')
     }
-    logger.error(
-      `SyncService#uploadFileInDropbox. Assembly file with Id: ${file.id} has no download url. Channel ID: ${file.channelId}`,
-    )
-    throw new Error('File not found')
+
+    const downloadUrl = file.downloadUrl
+    const rootNamespaceId = z.string().parse(this.connectionToken.rootNamespaceId)
+    const refreshToken = this.connectionToken.refreshToken
+
+    // Over the single-shot limit (or unknown size) goes through an upload session.
+    const needsSession = file.size === undefined || file.size > DROPBOX_SINGLE_UPLOAD_MAX_BYTES
+
+    logger.info('SyncService#uploadFileInDropbox :: Upload method chosen', {
+      path,
+      method: needsSession ? 'upload session' : 'direct upload',
+      sizeBytes: file.size ?? null,
+    })
+
+    const dbxResponse = needsSession
+      ? await this.dbxClient.uploadFileSession({
+          filePath: path,
+          // node-fetch's body is a Node stream, an async iterable at runtime.
+          getBody: async () =>
+            (await this.fetchAssemblyBody(downloadUrl)) as unknown as AsyncIterable<Uint8Array>,
+          rootNamespaceId,
+          refreshToken,
+          chunkSize: DROPBOX_UPLOAD_CHUNK_BYTES,
+          expectedSize: file.size,
+        })
+      : await this.dbxClient.uploadFile({
+          urlPath: DBX_URL_PATH.fileUpload,
+          filePath: path,
+          body: await this.fetchAssemblyBody(downloadUrl),
+          rootNamespaceId,
+          refreshToken,
+        })
+
+    logger.info('SyncService#uploadFileInDropbox :: File uploaded to', path)
+    return {
+      dbxFileId: dbxResponse.id,
+      contentHash: dbxResponse.contentHash,
+    }
+  }
+
+  // Fetches the Assembly body, failing on a non-ok response so an error page is
+  // never streamed to Dropbox as the file content.
+  private async fetchAssemblyBody(downloadUrl: string): Promise<NodeJS.ReadableStream> {
+    const resp = await fetch(downloadUrl)
+    if (!resp.ok || !resp.body) {
+      throw new Error(
+        `SyncService#uploadFileInDropbox. Assembly download failed with status ${resp.status}`,
+      )
+    }
+    return resp.body
   }
 
   async removeChannelSyncMapping(channelSyncId: string) {

src/features/sync/lib/__tests__/Sync.uploadRouting.test.ts

@@ -0,0 +1,116 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  DROPBOX_SINGLE_UPLOAD_MAX_BYTES,
+  DROPBOX_UPLOAD_CHUNK_BYTES,
+} from '@/features/sync/constant'
+
+// The Assembly file body is fetched via node-fetch — mock it so no real request runs.
+vi.mock('node-fetch', () => ({ default: vi.fn() }))
+
+// SyncService builds a DropboxClient + CopilotAPI and imports a Trigger.dev task at
+// module load — stub all three so the service constructs in isolation.
+vi.mock('@/lib/dropbox/DropboxClient', () => ({
+  DropboxClient: class {
+    getDropboxClient() {
+      return {}
+    }
+    dbxAuthClient = { refreshAccessToken: vi.fn() }
+    uploadFile = vi.fn().mockResolvedValue({ id: 'single-id', contentHash: 'single-hash' })
+    uploadFileSession = vi.fn().mockResolvedValue({ id: 'session-id', contentHash: 'session-hash' })
+  },
+}))
+
+vi.mock('@/lib/copilot/CopilotAPI', () => ({
+  CopilotAPI: class {},
+  isCopilotApiError: () => false,
+}))
+
+vi.mock('@/trigger/processFileSync', () => ({
+  bidirectionalMasterSync: { trigger: vi.fn() },
+}))
+
+import fetch from 'node-fetch'
+import { SyncService } from '@/features/sync/lib/Sync.service'
+
+const mockedFetch = vi.mocked(fetch)
+
+const user = { portalId: 'p', token: 't', copilot: {} } as never
+const connectionToken = { accountId: 'a', refreshToken: 'r', rootNamespaceId: 'n' } as never
+
+type UploadInternals = {
+  uploadFileInDropbox(
+    file: unknown,
+    path: string,
+  ): Promise<{ dbxFileId: string; contentHash: string }>
+  dbxClient: {
+    uploadFile: ReturnType<typeof vi.fn>
+    uploadFileSession: ReturnType<typeof vi.fn>
+  }
+}
+
+const fileWithSize = (size: number | undefined) => ({
+  id: 'asm-1',
+  downloadUrl: 'https://assembly/download',
+  size,
+})
+
+let service: SyncService
+let internals: UploadInternals
+
+beforeEach(() => {
+  service = new SyncService(user, connectionToken)
+  internals = service as unknown as UploadInternals
+  mockedFetch.mockResolvedValue({ ok: true, status: 200, body: {} } as never)
+})
+
+afterEach(() => {
+  vi.restoreAllMocks()
+})
+
+describe('uploadFileInDropbox routing', () => {
+  it('uses the single-shot upload when the file is at or below the limit', async () => {
+    const result = await internals.uploadFileInDropbox(
+      fileWithSize(DROPBOX_SINGLE_UPLOAD_MAX_BYTES),
+      '/root/small.txt',
+    )
+
+    expect(internals.dbxClient.uploadFile).toHaveBeenCalledTimes(1)
+    expect(internals.dbxClient.uploadFileSession).not.toHaveBeenCalled()
+    expect(result).toEqual({ dbxFileId: 'single-id', contentHash: 'single-hash' })
+  })
+
+  it('uses an upload session when the file is over the limit', async () => {
+    const result = await internals.uploadFileInDropbox(
+      fileWithSize(DROPBOX_SINGLE_UPLOAD_MAX_BYTES + 1),
+      '/root/big.bin',
+    )
+
+    expect(internals.dbxClient.uploadFile).not.toHaveBeenCalled()
+    expect(internals.dbxClient.uploadFileSession).toHaveBeenCalledTimes(1)
+    expect(internals.dbxClient.uploadFileSession.mock.calls[0][0]).toMatchObject({
+      filePath: '/root/big.bin',
+      chunkSize: DROPBOX_UPLOAD_CHUNK_BYTES,
+      rootNamespaceId: 'n',
+      refreshToken: 'r',
+    })
+    expect(typeof internals.dbxClient.uploadFileSession.mock.calls[0][0].getBody).toBe('function')
+    expect(result).toEqual({ dbxFileId: 'session-id', contentHash: 'session-hash' })
+  })
+
+  it('uses an upload session when the size is unknown', async () => {
+    await internals.uploadFileInDropbox(fileWithSize(undefined), '/root/unknown.bin')
+
+    expect(internals.dbxClient.uploadFile).not.toHaveBeenCalled()
+    expect(internals.dbxClient.uploadFileSession).toHaveBeenCalledTimes(1)
+  })
+
+  it('throws instead of uploading when the Assembly download responds with a non-ok status', async () => {
+    mockedFetch.mockResolvedValue({ ok: false, status: 403, body: {} } as never)
+
+    await expect(
+      internals.uploadFileInDropbox(fileWithSize(1024), '/root/small.txt'),
+    ).rejects.toThrow(/download/i)
+
+    expect(internals.dbxClient.uploadFile).not.toHaveBeenCalled()
+  })
+})

src/lib/__tests__/logger.test.ts

@@ -4,17 +4,34 @@ import { logger } from '@/lib/logger'
 afterEach(() => {
   vi.restoreAllMocks()
   delete process.env.LOG_LEVEL
+  delete process.env.LOG_DEPTH
 })
 
 describe('logger', () => {
-  it('caps object depth so nested payloads are not fully dumped', () => {
+  it('shows nested payload fields like opts and file inside a batch array', () => {
     const spy = vi.spyOn(console, 'info').mockImplementation(() => undefined)
-    logger.info({ a: { b: { c: { secret: 'deep-value' } } } })
+    // Mirrors the batchTriggerAndWait item shape: [{ payload: { opts, file } }].
+    logger.info([{ payload: { opts: { channelId: 'ch-1' }, file: { id: 'f-1' } } }])
     const out = spy.mock.calls[0][0] as string
-    expect(out).not.toContain('deep-value')
+    expect(out).toContain("channelId: 'ch-1'")
+    expect(out).toContain("id: 'f-1'")
+  })
+
+  it('still caps extremely deep objects so a log line stays bounded', () => {
+    const spy = vi.spyOn(console, 'info').mockImplementation(() => undefined)
+    logger.info({ a: { b: { c: { d: { e: { secret: 'too-deep' } } } } } })
+    const out = spy.mock.calls[0][0] as string
+    expect(out).not.toContain('too-deep')
     expect(out).toContain('[Object]')
   })
 
+  it('honours LOG_DEPTH for deeper inspection', () => {
+    process.env.LOG_DEPTH = '6'
+    const spy = vi.spyOn(console, 'info').mockImplementation(() => undefined)
+    logger.info({ a: { b: { c: { d: { e: { secret: 'now-visible' } } } } } })
+    expect(spy.mock.calls[0][0] as string).toContain('now-visible')
+  })
+
   it('truncates long arrays instead of dumping every element', () => {
     const spy = vi.spyOn(console, 'info').mockImplementation(() => undefined)
     logger.info(Array.from({ length: 100 }, (_, i) => i))

src/lib/copilot/types.ts

@@ -139,6 +139,7 @@ export const CopilotFileRetrieveSchema = z.object({
   path: z.string(),
   status: z.string().optional(),
   downloadUrl: z.string().optional(),
+  size: z.number().optional(),
   previousAttributes: z
     .object({
       name: z.string().optional(),