From 86ced3e9e8e8ce8f65f5200e5f710a3ab851aaf9 Mon Sep 17 00:00:00 2001 From: Bennett Date: Thu, 10 Sep 2026 16:37:57 +0300 Subject: [PATCH 1/2] release: Add whatsapp retry --- .../WhatsAppInboxMessageRepository.java | 8 +++ .../services/WhatsAppInboxMessageService.java | 24 +++++-- .../services/WhatsAppMediaBackfillWorker.java | 69 +++++++++++++++++++ src/main/resources/application.properties | 7 ++ 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppMediaBackfillWorker.java diff --git a/src/main/java/com/flexcodelabs/flextuma/core/repositories/WhatsAppInboxMessageRepository.java b/src/main/java/com/flexcodelabs/flextuma/core/repositories/WhatsAppInboxMessageRepository.java index 610ca02..70d9e8c 100644 --- a/src/main/java/com/flexcodelabs/flextuma/core/repositories/WhatsAppInboxMessageRepository.java +++ b/src/main/java/com/flexcodelabs/flextuma/core/repositories/WhatsAppInboxMessageRepository.java @@ -4,11 +4,13 @@ import com.flexcodelabs.flextuma.core.entities.auth.User; import com.flexcodelabs.flextuma.core.entities.whatsapp.WhatsAppInboxMessage; import com.flexcodelabs.flextuma.modules.whatsapp.dtos.WhatsAppTenantStorageUsageDTO; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import java.time.LocalDateTime; import java.util.List; import java.util.UUID; @@ -17,6 +19,12 @@ public interface WhatsAppInboxMessageRepository extends BaseRepository { boolean existsByProviderMessageId(String providerMessageId); + /** Candidates for {@code WhatsAppMediaBackfillWorker}: messages that carry a Meta media id + * but never got a cached copy on disk, bounded to recent messages since Meta's CDN only + * keeps media retrievable for a limited window. */ + List findByMediaIdIsNotNullAndMediaPathIsNullAndReceivedAtAfter( + LocalDateTime receivedAfter, Pageable pageable); + /** Bytes of WhatsApp media stored for one tenant: everyone in {@code organisation} when it * is non-null, otherwise just {@code user} (the org-less-account fallback). */ @Query("SELECT COALESCE(SUM(m.mediaSize), 0) FROM WhatsAppInboxMessage m WHERE m.mediaSize IS NOT NULL AND " diff --git a/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppInboxMessageService.java b/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppInboxMessageService.java index 49d2110..ae1081e 100644 --- a/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppInboxMessageService.java +++ b/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppInboxMessageService.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; @Service @RequiredArgsConstructor @@ -56,14 +57,29 @@ public record MediaContent(byte[] bytes, String mimeType) {} throw new ResponseStatusException(HttpStatus.METHOD_NOT_ALLOWED, "Inbox messages cannot be updated manually"); } + /** Serves the cached copy of the message's media, re-downloading it from Meta on the fly if + * it was never successfully cached (e.g. the connector token was briefly unusable at + * ingestion time) or the cached file has since gone missing from disk. */ + @Transactional public MediaContent getMedia(UUID id) { WhatsAppInboxMessage message = findAccessibleById(id); - if (message.getMediaPath() == null) { + if (message.getMediaId() == null) { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "This message has no stored media"); } - byte[] bytes = mediaService.read(message.getMediaPath()) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Media file is no longer available")); - return new MediaContent(bytes, message.getMimeType()); + Optional bytes = message.getMediaPath() != null + ? mediaService.read(message.getMediaPath()) + : Optional.empty(); + if (bytes.isEmpty()) { + bytes = mediaService.download(message.getConfig(), message.getMediaId()).flatMap(downloaded -> { + message.setMediaPath(downloaded.path()); + message.setMediaSize(downloaded.size()); + repository.save(message); + return mediaService.read(downloaded.path()); + }); + } + return new MediaContent( + bytes.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Media file is no longer available")), + message.getMimeType()); } /** WhatsApp media storage usage. SUPER_ADMIN sees the breakdown across every tenant diff --git a/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppMediaBackfillWorker.java b/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppMediaBackfillWorker.java new file mode 100644 index 0000000..1456854 --- /dev/null +++ b/src/main/java/com/flexcodelabs/flextuma/modules/whatsapp/services/WhatsAppMediaBackfillWorker.java @@ -0,0 +1,69 @@ +package com.flexcodelabs.flextuma.modules.whatsapp.services; + +import com.flexcodelabs.flextuma.core.entities.whatsapp.WhatsAppInboxMessage; +import com.flexcodelabs.flextuma.core.repositories.WhatsAppInboxMessageRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.PageRequest; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * Retries downloading WhatsApp inbound media that failed to cache at webhook ingestion time + * (e.g. a briefly unusable connector token), so a message doesn't stay permanently unavailable + * just because nobody opened the conversation to trigger {@link WhatsAppInboxMessageService}'s + * on-read retry. Bounded to recently received messages -- Meta's CDN only keeps media + * retrievable for a limited window, so retrying older failures indefinitely would just waste + * batch slots on media that's gone for good. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WhatsAppMediaBackfillWorker { + + private final WhatsAppInboxMessageRepository inboxMessageRepository; + private final WhatsAppMediaService mediaService; + + @Value("${flextuma.whatsapp.media-backfill.batch-size:25}") + private int batchSize; + + @Value("${flextuma.whatsapp.media-backfill.max-age-days:3}") + private int maxAgeDays; + + @Scheduled(fixedDelayString = "${flextuma.whatsapp.media-backfill.interval-ms:300000}") + @Transactional + public void backfillMissingMedia() { + List candidates = inboxMessageRepository + .findByMediaIdIsNotNullAndMediaPathIsNullAndReceivedAtAfter( + LocalDateTime.now().minusDays(maxAgeDays), PageRequest.of(0, batchSize)); + + if (candidates.isEmpty()) { + return; + } + + log.info("WhatsAppMediaBackfillWorker: Retrying {} message(s) with missing media", candidates.size()); + + for (WhatsAppInboxMessage message : candidates) { + backfillOne(message); + } + } + + private void backfillOne(WhatsAppInboxMessage message) { + try { + mediaService.download(message.getConfig(), message.getMediaId()).ifPresentOrElse(downloaded -> { + message.setMediaPath(downloaded.path()); + message.setMediaSize(downloaded.size()); + inboxMessageRepository.save(message); + log.info("WhatsAppMediaBackfillWorker: Recovered media for message [{}]", message.getId()); + }, () -> log.debug("WhatsAppMediaBackfillWorker: Media for message [{}] still unavailable", message.getId())); + } catch (Exception e) { + log.error("WhatsAppMediaBackfillWorker: Error backfilling media for message [{}]: {}", + message.getId(), e.getMessage()); + } + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c309828..da3016d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -73,6 +73,13 @@ flextuma.app.frontend.directory=${APP_FRONTEND_DIRECTORY:/tmp/frontend} # persistent volume the deployed client assets use. flextuma.whatsapp.media.directory=${FLEXTUMA_WHATSAPP_MEDIA_DIRECTORY:client/media/whatsapp} +# Backfill job that retries downloading inbound WhatsApp media that failed to cache at ingestion +# time. Only considers messages received within max-age-days, since Meta's CDN stops serving +# media after a limited window anyway. +flextuma.whatsapp.media-backfill.interval-ms=${FLEXTUMA_WHATSAPP_MEDIA_BACKFILL_INTERVAL_MS:300000} +flextuma.whatsapp.media-backfill.batch-size=${FLEXTUMA_WHATSAPP_MEDIA_BACKFILL_BATCH_SIZE:25} +flextuma.whatsapp.media-backfill.max-age-days=${FLEXTUMA_WHATSAPP_MEDIA_BACKFILL_MAX_AGE_DAYS:3} + # Upload size limits. These can be overridden per deployment through environment variables. spring.servlet.multipart.max-file-size=${MAX_UPLOAD_FILE_SIZE:100MB} spring.servlet.multipart.max-request-size=${MAX_UPLOAD_REQUEST_SIZE:100MB} From ced77a30d406e5eb20dee92f5e39efe9343a8ff9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Sep 2026 13:38:38 +0000 Subject: [PATCH 2/2] Release v0.0.67 [skip ci] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 71e54e1..d8e3cca 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ plugins { } group = 'com.flexcodelabs' -version = '0.0.66' +version = '0.0.67' description = 'Flextuma App' java {