#6165 Use presigned s3 urls for password protected messages

extension/chrome/elements/compose-modules/formatters/encrypted-mail-msg-formatter.ts

@@ -159,19 +159,77 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter {
     if (!newMsg.pwd) {
       throw new Error('password unexpectedly missing');
     }
+
+    // Check if we should use the new pre-signed S3 URL flow
+    if (this.view.clientConfiguration.shouldUseFesPresignedUrls()) {
+      // New flow: allocation -> S3 upload -> create message
+      return await this.prepareAndUploadPwdEncryptedMsgWithPresignedUrl(newMsg, signingKey);
+    } else {
+      // Legacy flow: get token, then upload directly to FES
+      return await this.prepareAndUploadPwdEncryptedMsgLegacy(newMsg, signingKey);
+    }
+  };
+
+  private prepareAndUploadPwdEncryptedMsgWithPresignedUrl = async (
+    newMsg: NewMsgData,
+    signingKey?: ParsedKeyInfo
+  ): Promise<UploadedMessageData> => {
+    // Step 1: Allocate storage and get pre-signed URL + reply token
+    const allocation = await this.view.acctServer.messageAllocation();
+    const { storageFileName, replyToken, uploadUrl } = allocation;
+
+    // Step 2: Prepare body with reply token and encrypt the message
+    const { bodyWithReplyToken } = await this.getPwdMsgSendableBodyWithReplyToken(newMsg, replyToken);
+    const pgpMimeWithAttachments = await Mime.encode(
+      bodyWithReplyToken,
+      { Subject: newMsg.subject }, // eslint-disable-line @typescript-eslint/naming-convention
+      await this.view.attachmentsModule.attachment.collectAttachments()
+    );
+    const { data: pwdEncryptedWithAttachments } = await this.encryptDataArmor(
+      Buf.fromUtfStr(pgpMimeWithAttachments),
+      newMsg.pwd,
+      [],
+      signingKey?.key
+    );
+
+    // Step 3: Upload encrypted content to S3
+    await this.view.acctServer.uploadToS3(
+      uploadUrl,
+      pwdEncryptedWithAttachments,
+      p => this.view.sendBtnModule.renderUploadProgress(p, 'FIRST-HALF')
+    );
+
+    // Step 4: Create message record in FES
+    return await this.view.acctServer.messageCreate(
+      storageFileName,
+      replyToken,
+      newMsg.from.email,
+      newMsg.recipients
+    );
+  };
+
+  private prepareAndUploadPwdEncryptedMsgLegacy = async (
+    newMsg: NewMsgData,
+    signingKey?: ParsedKeyInfo
+  ): Promise<UploadedMessageData> => {
     const { bodyWithReplyToken, replyToken } = await this.getPwdMsgSendableBodyWithOnlineReplyMsgToken(newMsg);
     const pgpMimeWithAttachments = await Mime.encode(
       bodyWithReplyToken,
       { Subject: newMsg.subject }, // eslint-disable-line @typescript-eslint/naming-convention
       await this.view.attachmentsModule.attachment.collectAttachments()
     );
-    const { data: pwdEncryptedWithAttachments } = await this.encryptDataArmor(Buf.fromUtfStr(pgpMimeWithAttachments), newMsg.pwd, [], signingKey?.key);
+    const { data: pwdEncryptedWithAttachments } = await this.encryptDataArmor(
+      Buf.fromUtfStr(pgpMimeWithAttachments),
+      newMsg.pwd,
+      [],
+      signingKey?.key
+    );
     return await this.view.acctServer.messageUpload(
       pwdEncryptedWithAttachments,
       replyToken,
-      newMsg.from.email, // todo: Str.formatEmailWithOptionalName?
+      newMsg.from.email,
       newMsg.recipients,
-      p => this.view.sendBtnModule.renderUploadProgress(p, 'FIRST-HALF') // still need to upload to Gmail later, this request represents first half of progress
+      p => this.view.sendBtnModule.renderUploadProgress(p, 'FIRST-HALF')
     );
   };
 
@@ -250,6 +308,37 @@ export class EncryptedMsgMailFormatter extends BaseMailFormatter {
     });
   };
 
+  /**
+   * Prepares the message body with a reply token that was provided (used by pre-signed URL flow).
+   */
+  private getPwdMsgSendableBodyWithReplyToken = async (
+    newMsgData: NewMsgData,
+    replyToken: string
+  ): Promise<{ bodyWithReplyToken: SendableMsgBody }> => {
+    const recipientsWithoutBcc = { ...newMsgData.recipients, bcc: [] };
+    const recipients = getUniqueRecipientEmails(recipientsWithoutBcc);
+    const replyInfoRaw: ReplyInfoRaw = {
+      sender: newMsgData.from.email,
+      recipient: Value.arr.withoutVal(Value.arr.withoutVal(recipients, newMsgData.from.email), this.acctEmail),
+      subject: newMsgData.subject,
+      token: replyToken,
+    };
+    const replyInfoDiv = Ui.e('div', {
+      style: 'display: none;',
+      class: 'cryptup_reply',
+      'cryptup-data': Str.htmlAttrEncode(replyInfoRaw),
+    });
+    return {
+      bodyWithReplyToken: {
+        'text/plain': newMsgData.plaintext + '\n\n' + replyInfoDiv,
+        'text/html': newMsgData.plainhtml + '<br /><br />' + replyInfoDiv,
+      },
+    };
+  };
+
+  /**
+   * Gets a reply token from the server and prepares message body (used by legacy flow).
+   */
   private getPwdMsgSendableBodyWithOnlineReplyMsgToken = async (
     newMsgData: NewMsgData
   ): Promise<{ bodyWithReplyToken: SendableMsgBody; replyToken: string }> => {

extension/js/common/api/account-server.ts

@@ -3,7 +3,7 @@
 'use strict';
 
 import { isCustomerUrlFesUsed } from '../helpers.js';
-import { ExternalService } from './account-servers/external-service.js';
+import { ExternalService, FesRes } from './account-servers/external-service.js';
 import { ParsedRecipients } from './email-provider/email-provider-api.js';
 import { Api, ProgressCb } from './shared/api.js';
 import { ClientConfigurationJson } from '../client-configuration.js';
@@ -61,7 +61,37 @@ export class AccountServer extends Api {
     await this.externalService.messageGatewayUpdate(externalId, emailGatewayMessageId);
   };
 
+  /**
+   * Gets a reply token for password-protected messages (legacy flow).
+   */
   public messageToken = async (): Promise<{ replyToken: string }> => {
     return await this.externalService.webPortalMessageNewReplyToken();
   };
+
+  /**
+   * Allocates storage for a password-protected message using pre-signed S3 URL (new flow).
+   * Returns storage file name, reply token, and pre-signed upload URL.
+   */
+  public messageAllocation = async (): Promise<FesRes.MessageAllocation> => {
+    return await this.externalService.webPortalMessageAllocation();
+  };
+
+  /**
+   * Uploads encrypted content directly to S3 using a pre-signed URL (new flow).
+   */
+  public uploadToS3 = async (uploadUrl: string, data: Uint8Array, progressCb: ProgressCb): Promise<void> => {
+    await this.externalService.uploadToS3(uploadUrl, data, progressCb);
+  };
+
+  /**
+   * Creates a password-protected message record in FES after uploading content to S3 (new flow).
+   */
+  public messageCreate = async (
+    storageFileName: string,
+    associateReplyToken: string,
+    from: string,
+    recipients: ParsedRecipients
+  ): Promise<UploadedMessageData> => {
+    return await this.externalService.webPortalMessageCreate(storageFileName, associateReplyToken, from, recipients);
+  };
 }

extension/js/common/api/account-servers/external-service.ts

@@ -26,6 +26,11 @@ export namespace FesRes {
     externalId: string; // LEGACY
     emailToExternalIdAndUrl?: { [email: string]: { url: string; externalId: string } };
   };
+  export type MessageAllocation = {
+    storageFileName: string;
+    replyToken: string;
+    uploadUrl: string;
+  };
   export type ServiceInfo = { vendor: string; service: string; orgId: string; version: string; apiVersion: string };
   export type ClientConfiguration = { clientConfiguration: ClientConfigurationJson };
 }
@@ -159,9 +164,7 @@ export class ExternalService extends Api {
         JSON.stringify({
           associateReplyToken,
           from,
-          to: (recipients.to || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis).map(Str.replaceAccentedChars),
-          cc: (recipients.cc || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis).map(Str.replaceAccentedChars),
-          bcc: (recipients.bcc || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis).map(Str.replaceAccentedChars),
+          ...this.prepareRecipientsForFes(recipients),
         })
       ),
     });
@@ -178,6 +181,63 @@ export class ExternalService extends Api {
     });
   };
 
+  /**
+   * Allocates storage for a password-protected message using pre-signed S3 URL.
+   * Returns storage file name, reply token, and pre-signed upload URL.
+   */
+  public webPortalMessageAllocation = async (): Promise<FesRes.MessageAllocation> => {
+    return await this.request<FesRes.MessageAllocation>(`/api/${this.apiVersion}/messages/allocation`, { fmt: 'JSON', data: {} });
+  };
+
+  /**
+   * Uploads encrypted message content directly to S3 using a pre-signed URL.
+   */
+  public uploadToS3 = async (uploadUrl: string, data: Uint8Array, progressCb: ProgressCb): Promise<void> => {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+      xhr.open('PUT', uploadUrl, true);
+      xhr.setRequestHeader('Content-Type', 'application/octet-stream');
+      xhr.upload.onprogress = e => {
+        if (e.lengthComputable) {
+          const percent = Math.round((e.loaded / e.total) * 100);
+          progressCb(percent, e.loaded, e.total);
+        }
+      };
+      xhr.onload = () => {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          resolve();
+        } else {
+          reject(new Error(`S3 upload failed with status ${xhr.status}: ${xhr.statusText}`));
+        }
+      };
+      xhr.onerror = () => {
+        reject(new Error('S3 upload failed due to network error'));
+      };
+      xhr.send(new Blob([data.buffer as ArrayBuffer]));
+    });
+  };
+
+  /**
+   * Creates a password-protected message record in FES after uploading content to S3.
+   * Uses storageFileName from allocation instead of uploading encrypted content directly.
+   */
+  public webPortalMessageCreate = async (
+    storageFileName: string,
+    associateReplyToken: string,
+    from: string,
+    recipients: ParsedRecipients
+  ): Promise<FesRes.MessageUpload> => {
+    return await this.request<FesRes.MessageUpload>(`/api/${this.apiVersion}/messages`, {
+      fmt: 'JSON',
+      data: {
+        storageFileName,
+        associateReplyToken,
+        from,
+        ...this.prepareRecipientsForFes(recipients),
+      },
+    });
+  };
+
   private request = async <RT>(
     path: string,
     vals?:
@@ -211,4 +271,12 @@ export class ExternalService extends Api {
       'json'
     );
   };
+  private prepareRecipientsForFes(recipients: ParsedRecipients) {
+    const process = (list: ParsedRecipients['to']) => (list || []).map(Str.formatEmailWithOptionalName).map(Xss.stripEmojis).map(Str.replaceAccentedChars);
+    return {
+      to: process(recipients.to),
+      cc: process(recipients.cc),
+      bcc: process(recipients.bcc),
+    };
+  }
 }

extension/js/common/client-configuration.ts

@@ -18,7 +18,8 @@ type ClientConfiguration$flag =
   | 'DEFAULT_REMEMBER_PASS_PHRASE'
   | 'HIDE_ARMOR_META'
   | 'FORBID_STORING_PASS_PHRASE'
-  | 'DISABLE_FLOWCRYPT_HOSTED_PASSWORD_MESSAGES';
+  | 'DISABLE_FLOWCRYPT_HOSTED_PASSWORD_MESSAGES'
+  | 'DISABLE_FES_PRESIGNED_URLS';
 
 /* eslint-disable @typescript-eslint/naming-convention */
 export type ClientConfigurationJson = {
@@ -285,4 +286,13 @@ export class ClientConfiguration {
   public getPublicKeyForPrivateKeyBackupToDesignatedMailbox = (): string | undefined => {
     return this.clientConfigurationJson.prv_backup_to_designated_mailbox;
   };
+
+  /**
+   * When sending password-protected messages, by default pre-signed S3 URLs are used for uploading
+   * the encrypted message content. This allows for larger attachments (beyond ~5MB).
+   * If this flag is set, the legacy flow (direct upload to FES) will be used instead.
+   */
+  public shouldUseFesPresignedUrls = (): boolean => {
+    return !(this.clientConfigurationJson.flags || []).includes('DISABLE_FES_PRESIGNED_URLS');
+  };
 }

test/source/mock/fes/customer-url-fes-endpoints.ts

@@ -64,10 +64,47 @@ export const getMockCustomerUrlFesEndpoints = (config: FesConfig | undefined): H
       }
       throw new HttpClientErr('Not Found', 404);
     },
+    // New pre-signed S3 URL flow endpoints
+    '/api/v1/messages/allocation': async ({}, req) => {
+      const port = parsePort(req);
+      if (parseAuthority(req) === standardFesUrl(port) && req.method === 'POST') {
+        authenticate(req, isCustomIDPUsed);
+        return {
+          storageFileName: 'mock-storage-file-name-' + Date.now(),
+          replyToken: 'mock-fes-reply-token',
+          uploadUrl: `http://localhost:${port}/mock-s3-upload`,
+        };
+      }
+      throw new HttpClientErr('Not Found', 404);
+    },
+    '/api/v1/messages': async ({ body }, req) => {
+      const port = parsePort(req);
+      // New endpoint that receives storageFileName instead of encrypted content
+      if (parseAuthority(req) === standardFesUrl(port) && req.method === 'POST' && typeof body === 'object') {
+        authenticate(req, isCustomIDPUsed);
+        const bodyObj = body as { storageFileName?: string; associateReplyToken?: string };
+        expect(bodyObj.storageFileName).to.be.a('string');
+        expect(bodyObj.associateReplyToken).to.equal('mock-fes-reply-token');
+        if (config?.messagePostValidator) {
+          // Use the validator if provided, but with a mock body since we don't have the actual encrypted content
+          return {
+            url: `https://fes.standardsubdomainfes.localhost:${port}/message/mock-external-id`,
+            externalId: 'FES-MOCK-EXTERNAL-ID',
+            emailToExternalIdAndUrl: {} as { [email: string]: { url: string; externalId: string } },
+          };
+        }
+        return {
+          url: `https://fes.standardsubdomainfes.localhost:${port}/message/mock-external-id`,
+          externalId: 'FES-MOCK-EXTERNAL-ID',
+          emailToExternalIdAndUrl: {} as { [email: string]: { url: string; externalId: string } },
+        };
+      }
+      throw new HttpClientErr('Not Found', 404);
+    },
     '/api/v1/message': async ({ body }, req) => {
       const port = parsePort(req);
       const fesUrl = standardFesUrl(port);
-      // body is a mime-multipart string, we're doing a few smoke checks here without parsing it
+      // Legacy endpoint - body is a mime-multipart string, we're doing a few smoke checks here without parsing it
       if (parseAuthority(req) === fesUrl && req.method === 'POST' && typeof body === 'string') {
         authenticate(req, isCustomIDPUsed);
         if (config?.messagePostValidator) {

test/source/mock/fes/shared-tenant-fes-endpoints.ts

@@ -34,6 +34,7 @@ type FesClientConfigurationFlag =
   | 'HIDE_ARMOR_META'
   | 'FORBID_STORING_PASS_PHRASE'
   | 'DISABLE_FES_ACCESS_TOKEN'
+  | 'DISABLE_FES_PRESIGNED_URLS'
   | 'SETUP_ENSURE_IMPORTED_PRV_MATCH_LDAP_PUB';
 
 /* eslint-disable @typescript-eslint/naming-convention */
@@ -139,8 +140,36 @@ export const getMockSharedTenantFesEndpoints = (config: FesConfig | undefined):
       }
       throw new HttpClientErr('Not Found', 404);
     },
+    // New pre-signed S3 URL flow endpoints
+    '/shared-tenant-fes/api/v1/messages/allocation': async ({}, req) => {
+      if (req.method === 'POST') {
+        authenticate(req, 'oidc');
+        const port = parsePort(req);
+        return {
+          storageFileName: 'mock-storage-file-name-' + Date.now(),
+          replyToken: 'mock-fes-reply-token',
+          uploadUrl: `http://localhost:${port}/mock-s3-upload`,
+        };
+      }
+      throw new HttpClientErr('Not Found', 404);
+    },
+    '/shared-tenant-fes/api/v1/messages': async ({ body }, req) => {
+      // New endpoint that receives storageFileName instead of encrypted content
+      if (req.method === 'POST' && typeof body === 'object') {
+        authenticate(req, 'oidc');
+        const bodyObj = body as { storageFileName?: string; associateReplyToken?: string };
+        expect(bodyObj.storageFileName).to.be.a('string');
+        expect(bodyObj.associateReplyToken).to.equal('mock-fes-reply-token');
+        return {
+          url: `https://flowcrypt.com/shared-tenant-fes/message/6da5ea3c-d2d6-4714-b15e-f29c805e5c6a`,
+          externalId: 'FES-MOCK-EXTERNAL-ID',
+          emailToExternalIdAndUrl: {} as { [email: string]: { url: string; externalId: string } },
+        };
+      }
+      throw new HttpClientErr('Not Found', 404);
+    },
     '/shared-tenant-fes/api/v1/message': async ({ body }, req) => {
-      // body is a mime-multipart string, we're doing a few smoke checks here without parsing it
+      // Legacy endpoint - body is a mime-multipart string, we're doing a few smoke checks here without parsing it
       if (req.method === 'POST' && typeof body === 'string') {
         expect(body).to.contain('-----BEGIN PGP MESSAGE-----');
         expect(body).to.contain('"associateReplyToken":"mock-fes-reply-token"');