Skip to content

Time out slow image captions, and end each caption request once #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
trypsynth merged 1 commit into from
Oct 10, 2026
Merged

Time out slow image captions, and end each caption request once #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 70 additions & 31 deletions talkback/src/main/java/com/google/android/accessibility/talkback/actor/ImageCaptioner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ void startIconDetector() {
@VisibleForTesting
void shutdownIconDetector() {
removeMessages(MSG_RESULT_TIMEOUT);
captionResults.cleanUp();
abandonCaptionResults();

if (iconAnnotationsDetector != null) {
synchronized (this) {
Expand Down Expand Up @@ -689,6 +689,8 @@ private void shutdownImageDescription() {
}

public void shutdown() {
// Ends all requests, so that results that come later are ignored.
clearRequests();
shutdownIconDetector();
shutdownImageDescription();
moduleStateManager.shutdown();
Expand Down Expand Up @@ -927,8 +929,9 @@ public boolean captionWithGemini(AccessibilityNodeInfoCompat node) {
service,
node,
this::onScreenshotCapturePending,
(focusedNode, screenCapture, isUserRequested, captureByWindow) -> {
(request, focusedNode, screenCapture, isUserRequested, captureByWindow) -> {
processImageCaptioningWithGemini(
request,
focusedNode,
screenCapture,
isUserRequested,
Expand Down Expand Up @@ -969,8 +972,9 @@ public boolean captionWithOnDeviceGemini(AccessibilityNodeInfoCompat node) {
service,
node,
this::onScreenshotCapturePending,
(focusedNode, screenCapture, isUserRequested, captureByWindow) -> {
(request, focusedNode, screenCapture, isUserRequested, captureByWindow) -> {
processImageCaptioningWithGemini(
request,
focusedNode,
screenCapture,
isUserRequested,
Expand All @@ -983,13 +987,14 @@ public boolean captionWithOnDeviceGemini(AccessibilityNodeInfoCompat node) {
}

private void processImageCaptioningWithGemini(
ScreenshotCaptureRequest request,
AccessibilityNodeInfoCompat focusedNode,
Bitmap screenCapture,
boolean isUserRequested,
boolean captureByWindow,
boolean isToAiCore) {
if (screenCapture == null) {
handleScreenshotCaptureFailure(isUserRequested);
handleScreenshotCaptureFailure(request, isUserRequested);
return;
}

Expand All @@ -1002,6 +1007,7 @@ private void processImageCaptioningWithGemini(
captureByWindow);
CaptionResult captionResult =
new CaptionResult(
request,
AccessibilityNode.takeOwnership(focusedNode),
isUserRequested,
/* isImageDescriptionByGemini= */ true);
Expand Down Expand Up @@ -1066,21 +1072,22 @@ private void processImageCaptioningWithGemini(
return;
}

screenshotRequests.performNextRequest();
screenshotRequests.performNextRequest(request);
}

private void processScreenDescription(
ScreenshotCaptureRequest request,
AccessibilityNodeInfoCompat focusedNode,
Bitmap screenCapture,
boolean isUserRequested,
boolean captureByWindow) {
if (screenCapture == null) {
handleScreenshotCaptureFailure(isUserRequested);
handleScreenshotCaptureFailure(request, isUserRequested);
return;
}
byte[] imageBytes = DataFieldUtils.encodeImageToByteArray(screenCapture);
if (imageBytes == null) {
handleScreenshotCaptureFailure(isUserRequested);
handleScreenshotCaptureFailure(request, isUserRequested);
return;
}

Expand All @@ -1092,7 +1099,7 @@ private void processScreenDescription(
screenCapture.recycle();
}

screenshotRequests.performNextRequest();
screenshotRequests.performNextRequest(request);
}

/**
Expand Down Expand Up @@ -1304,12 +1311,13 @@ private static void blackenBlock(Rect bounds, int[] screenshotPixels, int width)

@VisibleForTesting
void handleScreenshotCaptureResponse(
ScreenshotCaptureRequest request,
AccessibilityNodeInfoCompat node,
@Nullable Bitmap screenCapture,
boolean isUserRequested,
boolean captureByWindow) {
if (screenCapture == null) {
handleScreenshotCaptureFailure(isUserRequested);
handleScreenshotCaptureFailure(request, isUserRequested);
return;
}

Expand All @@ -1320,6 +1328,7 @@ void handleScreenshotCaptureResponse(
if (!isUserRequested) {
LogUtils.v(TAG, "onScreenCaptureFinish(): auto-caption with Gemini");
processImageCaptioningWithGemini(
request,
node,
screenCapture,
/* isUserRequested= */ false,
Expand All @@ -1341,7 +1350,7 @@ void handleScreenshotCaptureResponse(
captureByWindow);

CaptionResult captionResult =
new CaptionResult(AccessibilityNode.takeOwnership(node), isUserRequested);
new CaptionResult(request, AccessibilityNode.takeOwnership(node), isUserRequested);
captionResults.addResultAndCleanOldOnes(requestId, captionResult, isUserRequested);
// For Bitmap recycle.
captionResult.addUsedScreenshot(screens.blockedScreenCapture());
Expand Down Expand Up @@ -1409,18 +1418,23 @@ void handleScreenshotCaptureResponse(
}

LogUtils.v(TAG, "No caption request for the screenshot. Perform the next screenshot request.");
screenshotRequests.performNextRequest();
// No result will finish the caption result, so it goes now, with its timeout.
captionResult.recycleAndClearScreenshots();
captionResults.removeCaptionResult(requestId);
removeMessages(MSG_RESULT_TIMEOUT);
screenshotRequests.performNextRequest(request);
}

private void handleScreenshotCaptureFailure(boolean isUserRequested) {
private void handleScreenshotCaptureFailure(
ScreenshotCaptureRequest request, boolean isUserRequested) {
analytics.onImageCaptionEvent(IMAGE_CAPTION_EVENT_SCREENSHOT_FAILED);
LogUtils.e(TAG, "onScreenCaptureFinish() taking screenshot has failed.");

if (isUserRequested) {
returnFeedback(R.string.gemini_screenshot_unavailable);
}

screenshotRequests.performNextRequest();
screenshotRequests.performNextRequest(request);
}

@VisibleForTesting
Expand All @@ -1430,9 +1444,9 @@ void onScreenshotCapturePending(boolean scheduled, Duration intervalTime) {
return;
}

// The list could not schedule the request, so it has dropped its requests.
analytics.onImageCaptionEvent(
TalkBackAnalytics.IMAGE_CAPTION_EVENT_SCHEDULE_SCREENSHOT_CAPTURE_FAILURE);
screenshotRequests.performNextRequest();
}

@VisibleForTesting
Expand All @@ -1448,7 +1462,7 @@ void onCharacterCaptionFinish(
+ StringBuilderUtils.joinFields(
StringBuilderUtils.optionalSubObj("result", result.text()),
StringBuilderUtils.optionalSubObj("node", node)));
characterCaptionRequests.performNextRequest();
characterCaptionRequests.performNextRequest(request);

handleResult(request.getRequestId(), node, result, isUserRequested);
imageCaptionStorage.updateCharacterCaptionResult(node, result);
Expand All @@ -1471,7 +1485,7 @@ void addCaptionRequest(
}
analytics.onImageCaptionEvent(IMAGE_CAPTION_EVENT_OCR_PERFORM_FAIL);
LogUtils.v(TAG, "onError(), error= %s", Request.errorName(errorCode));
characterCaptionRequests.performNextRequest();
characterCaptionRequests.performNextRequest(errorRequest);
handleResult(
errorRequest.getRequestId(),
AccessibilityNode.takeOwnership(node),
Expand All @@ -1489,7 +1503,7 @@ void onIconDetectionFinish(
}
analytics.onImageCaptionEvent(IMAGE_CAPTION_EVENT_ICON_DETECT_SUCCEED);
LogUtils.v(TAG, "onIconDetectionFinish() result=%s node=%s", result.text(), node);
iconDetectionRequests.performNextRequest();
iconDetectionRequests.performNextRequest(request);

handleResult(request.getRequestId(), node, result, isUserRequested);
imageCaptionStorage.updateDetectedIconLabel(node, result);
Expand All @@ -1513,7 +1527,7 @@ void addIconDetectionRequest(
}
analytics.onImageCaptionEvent(IMAGE_CAPTION_EVENT_ICON_DETECT_FAIL);
LogUtils.v(TAG, "onError(), error=%s", Request.errorName(errorCode));
iconDetectionRequests.performNextRequest();
iconDetectionRequests.performNextRequest(errorRequest);
handleResult(
errorRequest.getRequestId(),
AccessibilityNode.takeOwnership(node),
Expand All @@ -1536,7 +1550,7 @@ private void onImageDescriptionFinish(
+ StringBuilderUtils.joinFields(
StringBuilderUtils.optionalSubObj("result", result.text()),
StringBuilderUtils.optionalSubObj("node", node)));
imageDescriptionRequests.performNextRequest();
imageDescriptionRequests.performNextRequest(request);

handleResult(request.getRequestId(), node, result, isUserRequested);
imageCaptionStorage.updateImageDescriptionResult(node, result);
Expand All @@ -1560,7 +1574,7 @@ void addImageDescriptionRequest(
}
analytics.onImageCaptionEvent(IMAGE_CAPTION_EVENT_IMAGE_DESCRIBE_FAIL);
LogUtils.v(TAG, "onError(), error=%s", Request.errorName(errorCode));
imageDescriptionRequests.performNextRequest();
imageDescriptionRequests.performNextRequest(errorRequest);
handleResult(
errorRequest.getRequestId(),
AccessibilityNode.takeOwnership(node),
Expand Down Expand Up @@ -1642,10 +1656,23 @@ Locale getSpeechLocale() {

@VisibleForTesting
void clearRequests() {
removeMessages(MSG_RESULT_TIMEOUT);
screenshotRequests.clear();
characterCaptionRequests.clear();
iconDetectionRequests.clear();
captionResults.cleanUp();
imageDescriptionRequests.clear();
abandonCaptionResults();
}

/**
* Drops the caption results that wait for more results, and ends their screenshot requests, so
* that the next screenshot request is performed.
*/
private void abandonCaptionResults() {
for (CaptionResult captionResult : captionResults.removeAll()) {
captionResult.recycleAndClearScreenshots();
screenshotRequests.performNextRequest(captionResult.screenshotRequest);
}
}

@VisibleForTesting
Expand Down Expand Up @@ -1718,7 +1745,8 @@ void handleResult(int id, AccessibilityNode node, Result result, boolean isUserR
default -> {}
}
}
screenshotRequests.performNextRequest();
// Its screenshot request ended when its caption result was dropped. The first screenshot
// request is another one, so it stays.
} else {
switch (result.type()) {
case OCR -> {
Expand Down Expand Up @@ -1747,10 +1775,10 @@ void handleResult(int id, AccessibilityNode node, Result result, boolean isUserR
if (needFeedbackToUser) {
returnCaptionResult(id, captionResult);
}
screenshotRequests.performNextRequest();
captionResult.recycleAndClearScreenshots();
captionResults.removeCaptionResult(id);
removeMessages(MSG_RESULT_TIMEOUT);
screenshotRequests.performNextRequest(captionResult.screenshotRequest);
}
}

Expand Down Expand Up @@ -1958,7 +1986,7 @@ private void sendResultTimeoutMessage() {
message.what = MSG_RESULT_TIMEOUT;
message.arg1 = requestId;

sendMessageAtTime(message, System.currentTimeMillis() + RESULT_MAX_WAITING_TIME_MS);
sendMessageDelayed(message, RESULT_MAX_WAITING_TIME_MS);
}

@Override
Expand All @@ -1977,11 +2005,12 @@ public void handleMessage(@NonNull Message msg) {
}
}
LogUtils.w(TAG, "Caption request is timeout.");
screenshotRequests.clear();
// Ends the caption requests, so that their results are ignored when they come. The screenshot
// requests that wait stay, such as one the user has just made.
characterCaptionRequests.clear();
iconDetectionRequests.clear();
imageDescriptionRequests.clear();
captionResults.cleanUp();
abandonCaptionResults();
}
}

Expand Down Expand Up @@ -2108,6 +2137,9 @@ private static ProcessedScreens create(

/** Stores all caption results to announce them together. */
private static class CaptionResult {
/** The request that took the screenshot. It stays first in its list until this is finished. */
private final ScreenshotCaptureRequest screenshotRequest;

private final AccessibilityNode node;
private final boolean isUserRequest;
private boolean isOcrFinished;
Expand All @@ -2121,12 +2153,17 @@ private static class CaptionResult {
private boolean isGeminiFinishedWithError;
private boolean isNetworkError;

private CaptionResult(AccessibilityNode node, boolean isUserRequest) {
this(node, isUserRequest, /* isImageDescriptionByGemini= */ false);
private CaptionResult(
ScreenshotCaptureRequest screenshotRequest, AccessibilityNode node, boolean isUserRequest) {
this(screenshotRequest, node, isUserRequest, /* isImageDescriptionByGemini= */ false);
}

private CaptionResult(
AccessibilityNode node, boolean isUserRequest, boolean isImageDescriptionByGemini) {
ScreenshotCaptureRequest screenshotRequest,
AccessibilityNode node,
boolean isUserRequest,
boolean isImageDescriptionByGemini) {
this.screenshotRequest = screenshotRequest;
this.node = node;
this.isUserRequest = isUserRequest;
this.isImageDescriptionByGemini = isImageDescriptionByGemini;
Expand Down Expand Up @@ -2306,9 +2343,11 @@ public CaptionResults() {
captionResults = new HashMap<>();
}

public void cleanUp() {
captionResults.forEach((key, value) -> value.recycleAndClearScreenshots());
/** Removes all caption results and returns them. */
public List<CaptionResult> removeAll() {
List<CaptionResult> results = new ArrayList<>(captionResults.values());
captionResults.clear();
return results;
}

@Nullable
Expand Down
7 changes: 6 additions & 1 deletion .../src/main/java/com/google/android/accessibility/talkback/imagecaption/CaptionRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ protected void onCaptionStart() {
}

protected void onCaptionFinish(Result result) {
if (!finish()) {
return;
}
setEndTimestamp();
LogUtils.v(
TAG,
Expand All @@ -131,8 +134,10 @@ protected void onCaptionFinish(Result result) {

@Override
protected void onError(@ErrorCode int errorCode) {
if (!finish()) {
return;
}
setEndTimestamp();
stopTimeoutRunnable();
LogUtils.e(
TAG,
"onError() "
Expand Down
Loading
Loading