From e03732d623bc92a371466093e40f90109135341e Mon Sep 17 00:00:00 2001 From: Gabe Villalobos Date: Tue, 5 May 2026 16:31:50 -0700 Subject: [PATCH 1/5] Removes retry decorators from remaining core sentry tasks --- src/sentry/tasks/auth/auth.py | 4 ++-- src/sentry/tasks/codeowners/code_owners_auto_sync.py | 6 +++--- .../tasks/codeowners/update_code_owners_schema.py | 5 ++--- src/sentry/tasks/commits.py | 11 ++++++++--- src/sentry/tasks/email.py | 8 +++----- src/sentry/tasks/files.py | 6 +++--- src/sentry/tasks/groupowner.py | 5 ++--- src/sentry/tasks/reprocessing2.py | 5 ++--- src/sentry/tasks/summaries/weekly_reports.py | 8 +++----- src/sentry/tasks/weekly_escalating_forecast.py | 5 ++--- 10 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/sentry/tasks/auth/auth.py b/src/sentry/tasks/auth/auth.py index 3e000fa3c00c8a..bd01f8dbd6558f 100644 --- a/src/sentry/tasks/auth/auth.py +++ b/src/sentry/tasks/auth/auth.py @@ -15,7 +15,7 @@ from sentry.organizations.services.organization.service import organization_service from sentry.silo.base import SiloMode from sentry.silo.safety import unguarded_write -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import auth_control_tasks, auth_tasks from sentry.types.cell import CellMappingNotFound from sentry.users.services.user import RpcUser @@ -199,10 +199,10 @@ def call_to_action(self, org: Organization, user: RpcUser, member: OrganizationM namespace=auth_tasks, retry=Retry( delay=60 * 5, + on=(Exception,), ), silo_mode=SiloMode.CELL, ) -@retry def remove_2fa_non_compliant_members(org_id, actor_id=None, actor_key_id=None, ip_address=None): TwoFactorComplianceTask().remove_non_compliant_members( org_id, actor_id, actor_key_id, ip_address diff --git a/src/sentry/tasks/codeowners/code_owners_auto_sync.py b/src/sentry/tasks/codeowners/code_owners_auto_sync.py index 21a77c581a56a8..3e248cc9190b78 100644 --- a/src/sentry/tasks/codeowners/code_owners_auto_sync.py +++ b/src/sentry/tasks/codeowners/code_owners_auto_sync.py @@ -10,18 +10,18 @@ from sentry.models.projectownership import ProjectOwnership from sentry.notifications.notifications.codeowners_auto_sync import AutoSyncNotification from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import issues_tasks @instrumented_task( name="sentry.tasks.code_owners_auto_sync", namespace=issues_tasks, - retry=Retry(times=3, delay=60), + retry=Retry(times=3, delay=60, on=(Commit.DoesNotExist,)), processing_deadline_duration=60, silo_mode=SiloMode.CELL, + silenced_exceptions=(Commit.DoesNotExist,), ) -@retry(on=(), on_silent=(Commit.DoesNotExist,)) def code_owners_auto_sync(commit_id: int, **kwargs: Any) -> None: from django.db.models import BooleanField, Case, Exists, OuterRef, Subquery, When diff --git a/src/sentry/tasks/codeowners/update_code_owners_schema.py b/src/sentry/tasks/codeowners/update_code_owners_schema.py index 7e59b5ec1fbbe0..7a78315607889e 100644 --- a/src/sentry/tasks/codeowners/update_code_owners_schema.py +++ b/src/sentry/tasks/codeowners/update_code_owners_schema.py @@ -9,7 +9,7 @@ from sentry import features from sentry.models.organization import Organization, OrganizationStatus from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, load_model_from_db, retry +from sentry.tasks.base import instrumented_task, load_model_from_db from sentry.taskworker.namespaces import issues_tasks logger = logging.getLogger(__name__) @@ -18,10 +18,9 @@ @instrumented_task( name="sentry.tasks.update_code_owners_schema", namespace=issues_tasks, - retry=Retry(times=5, delay=5), + retry=Retry(times=5, delay=5, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def update_code_owners_schema( organization: int, integration: int | None = None, diff --git a/src/sentry/tasks/commits.py b/src/sentry/tasks/commits.py index edf92568aedfdb..9ff11a4f407ba2 100644 --- a/src/sentry/tasks/commits.py +++ b/src/sentry/tasks/commits.py @@ -24,7 +24,7 @@ from sentry.plugins.base import bindings from sentry.shared_integrations.exceptions import IntegrationError, IntegrationResourceNotFoundError from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import issues_tasks from sentry.users.models.user import User from sentry.users.services.user import RpcUser @@ -324,10 +324,15 @@ def fetch_commits_for_ref_with_lifecycle( name="sentry.tasks.commits.fetch_commits", namespace=issues_tasks, processing_deadline_duration=60 * 15 + 5, - retry=Retry(times=5, delay=60 * 5), + retry=Retry( + times=5, + delay=60 * 5, + on=(Exception,), + ignore=(Release.DoesNotExist, User.DoesNotExist), + ), silo_mode=SiloMode.CELL, + silenced_exceptions=(Release.DoesNotExist, User.DoesNotExist), ) -@retry(exclude=(Release.DoesNotExist, User.DoesNotExist)) def fetch_commits( release_id: int, user_id: int, diff --git a/src/sentry/tasks/email.py b/src/sentry/tasks/email.py index 304c3fc19c9729..9966beffac258a 100644 --- a/src/sentry/tasks/email.py +++ b/src/sentry/tasks/email.py @@ -7,7 +7,7 @@ from sentry.auth import access from sentry.models.group import Group from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import notifications_control_tasks, notifications_tasks from sentry.users.services.user.model import RpcUser from sentry.users.services.user.service import user_service @@ -74,10 +74,9 @@ def _send_email(message: dict[str, Any]) -> None: name="sentry.tasks.email.send_email", namespace=notifications_tasks, processing_deadline_duration=90, - retry=Retry(times=2, delay=60 * 5), + retry=Retry(times=2, delay=60 * 5, on=(TemporaryEmailError,)), silo_mode=SiloMode.CELL, ) -@retry(on=(TemporaryEmailError,)) def send_email(message: dict[str, Any]) -> None: _send_email(message) @@ -86,9 +85,8 @@ def send_email(message: dict[str, Any]) -> None: name="sentry.tasks.email.send_email_control", namespace=notifications_control_tasks, processing_deadline_duration=90, - retry=Retry(times=2, delay=60 * 5), + retry=Retry(times=2, delay=60 * 5, on=(TemporaryEmailError,)), silo_mode=SiloMode.CONTROL, ) -@retry(on=(TemporaryEmailError,)) def send_email_control(message: dict[str, Any]) -> None: _send_email(message) diff --git a/src/sentry/tasks/files.py b/src/sentry/tasks/files.py index 9a1fa8c500a7e5..9f1f3433ddbb98 100644 --- a/src/sentry/tasks/files.py +++ b/src/sentry/tasks/files.py @@ -5,7 +5,7 @@ from taskbroker_client.retry import Retry from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import deletion_control_tasks, deletion_tasks from sentry.utils.db import atomic_transaction @@ -60,10 +60,10 @@ def delete_file(file_blob_model, path, checksum, **kwargs): retry=Retry( times=MAX_RETRIES, delay=60 * 5, + on=(Exception,), ), silo_mode=SiloMode.CELL, ) -@retry def delete_unreferenced_blobs_region(blob_ids): from sentry.models.files.fileblob import FileBlob from sentry.models.files.fileblobindex import FileBlobIndex @@ -77,10 +77,10 @@ def delete_unreferenced_blobs_region(blob_ids): retry=Retry( times=MAX_RETRIES, delay=60 * 5, + on=(Exception,), ), silo_mode=SiloMode.CONTROL, ) -@retry def delete_unreferenced_blobs_control(blob_ids): from sentry.models.files.control_fileblob import ControlFileBlob from sentry.models.files.control_fileblobindex import ControlFileBlobIndex diff --git a/src/sentry/tasks/groupowner.py b/src/sentry/tasks/groupowner.py index 0b7a3ba6cb4c62..50de7e305289c8 100644 --- a/src/sentry/tasks/groupowner.py +++ b/src/sentry/tasks/groupowner.py @@ -15,7 +15,7 @@ from sentry.models.project import Project from sentry.models.release import Release from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import issues_tasks from sentry.users.api.serializers.user import UserSerializerResponse from sentry.utils import metrics @@ -187,10 +187,9 @@ def _process_suspect_commits( name="sentry.tasks.process_suspect_commits", namespace=issues_tasks, processing_deadline_duration=TASK_DURATION_S, - retry=Retry(times=5, delay=5), + retry=Retry(times=5, delay=5, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def process_suspect_commits( event_id, event_platform, diff --git a/src/sentry/tasks/reprocessing2.py b/src/sentry/tasks/reprocessing2.py index 3d4d9198c3c01a..d78bda2c249a98 100644 --- a/src/sentry/tasks/reprocessing2.py +++ b/src/sentry/tasks/reprocessing2.py @@ -13,7 +13,7 @@ from sentry.services import eventstore from sentry.services.eventstore.models import Event from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.tasks.process_buffer import buffer_incr from sentry.taskworker.namespaces import issues_tasks from sentry.types.activity import ActivityType @@ -141,10 +141,9 @@ def reprocess_group( name="sentry.tasks.reprocessing2.handle_remaining_events", namespace=issues_tasks, processing_deadline_duration=60 * 5, - retry=Retry(times=5), + retry=Retry(times=5, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def handle_remaining_events( project_id: int, new_group_id: int, diff --git a/src/sentry/tasks/summaries/weekly_reports.py b/src/sentry/tasks/summaries/weekly_reports.py index 5305395a37df00..3a78514a0a119a 100644 --- a/src/sentry/tasks/summaries/weekly_reports.py +++ b/src/sentry/tasks/summaries/weekly_reports.py @@ -27,7 +27,7 @@ from sentry.models.organizationmember import OrganizationMember from sentry.notifications.services import notifications_service from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.tasks.summaries.metrics import ( WeeklyReportHaltReason, WeeklyReportOperationType, @@ -101,11 +101,10 @@ def delete_min_org_id(self) -> None: @instrumented_task( name="sentry.tasks.summaries.weekly_reports.schedule_organizations", namespace=reports_tasks, - retry=Retry(times=5), + retry=Retry(times=5, on=(ProcessingDeadlineExceeded,)), processing_deadline_duration=timedelta(minutes=30), silo_mode=SiloMode.CELL, ) -@retry(timeouts=True) def schedule_organizations( dry_run: bool = False, timestamp: float | None = None, duration: int | None = None ) -> None: @@ -162,10 +161,9 @@ def schedule_organizations( name="sentry.tasks.summaries.weekly_reports.prepare_organization_report", namespace=reports_tasks, processing_deadline_duration=60 * 10, - retry=Retry(times=5, delay=5), + retry=Retry(times=5, delay=5, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def prepare_organization_report( timestamp: float, duration: int, diff --git a/src/sentry/tasks/weekly_escalating_forecast.py b/src/sentry/tasks/weekly_escalating_forecast.py index 7c9e1da400dc42..03b2b17ada0d9c 100644 --- a/src/sentry/tasks/weekly_escalating_forecast.py +++ b/src/sentry/tasks/weekly_escalating_forecast.py @@ -9,7 +9,7 @@ from sentry.models.group import Group, GroupStatus from sentry.models.project import Project from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import issues_tasks from sentry.types.group import GroupSubStatus from sentry.utils.iterators import chunked @@ -55,10 +55,9 @@ def run_escalating_forecast() -> None: name="sentry.tasks.weekly_escalating_forecast.generate_forecasts_for_projects", namespace=issues_tasks, processing_deadline_duration=60 * 2, - retry=Retry(times=3, delay=60), + retry=Retry(times=3, delay=60, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def generate_forecasts_for_projects(project_ids: list[int]) -> None: for until_escalating_groups in chunked( RangeQuerySetWrapper( From 7f14f41766bfe106981d143aaff48c6a1c91d136 Mon Sep 17 00:00:00 2001 From: Gabe Villalobos Date: Tue, 5 May 2026 16:31:21 -0700 Subject: [PATCH 2/5] Updates deletion tasks with new decorators --- src/sentry/deletions/tasks/groups.py | 9 ++++++--- src/sentry/deletions/tasks/nodestore.py | 7 ++++--- src/sentry/deletions/tasks/scheduled.py | 11 ++++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/sentry/deletions/tasks/groups.py b/src/sentry/deletions/tasks/groups.py index a02fb854476508..825db509d547df 100644 --- a/src/sentry/deletions/tasks/groups.py +++ b/src/sentry/deletions/tasks/groups.py @@ -3,6 +3,7 @@ import sentry_sdk from taskbroker_client.retry import Retry +from taskbroker_client.worker.workerchild import ProcessingDeadlineExceeded from sentry import deletions from sentry.deletions.defaults.group import GROUP_CHUNK_SIZE @@ -10,17 +11,19 @@ from sentry.exceptions import DeleteAborted from sentry.models.group import Group from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry, track_group_async_operation +from sentry.tasks.base import instrumented_task, track_group_async_operation from sentry.taskworker.namespaces import deletion_tasks @instrumented_task( name="sentry.deletions.tasks.groups.delete_groups_for_project", namespace=deletion_tasks, - retry=Retry(times=MAX_RETRIES, delay=60 * 5), + retry=Retry( + times=MAX_RETRIES, delay=60 * 5, on=(ProcessingDeadlineExceeded,), ignore=(DeleteAborted,) + ), silo_mode=SiloMode.CELL, + silenced_exceptions=(DeleteAborted,), ) -@retry(exclude=(DeleteAborted,), timeouts=True) @track_group_async_operation def delete_groups_for_project( project_id: int, diff --git a/src/sentry/deletions/tasks/nodestore.py b/src/sentry/deletions/tasks/nodestore.py index 0b57284fcd1682..602b5306f21471 100644 --- a/src/sentry/deletions/tasks/nodestore.py +++ b/src/sentry/deletions/tasks/nodestore.py @@ -24,7 +24,7 @@ from sentry.silo.base import SiloMode from sentry.snuba.dataset import Dataset from sentry.snuba.referrer import Referrer -from sentry.tasks.base import instrumented_task, retry, track_group_async_operation +from sentry.tasks.base import instrumented_task, track_group_async_operation from sentry.taskworker.namespaces import deletion_tasks from sentry.utils import metrics from sentry.utils.retries import ConditionalRetryPolicy, exponential_delay @@ -45,13 +45,14 @@ class RetryTask(Exception): namespace=deletion_tasks, processing_deadline_duration=60 * 20, retry=Retry( - on=(RetryTask,), + on=(Exception,), times=MAX_RETRIES, delay=60 * 5, + ignore=(DeleteAborted,), ), silo_mode=SiloMode.CELL, + silenced_exceptions=(DeleteAborted,), ) -@retry(exclude=(DeleteAborted,)) @track_group_async_operation def delete_events_for_groups_from_nodestore_and_eventstore( organization_id: int, diff --git a/src/sentry/deletions/tasks/scheduled.py b/src/sentry/deletions/tasks/scheduled.py index 6d250f64ab80cf..cbb86626660b7d 100644 --- a/src/sentry/deletions/tasks/scheduled.py +++ b/src/sentry/deletions/tasks/scheduled.py @@ -8,6 +8,7 @@ from django.utils import timezone from taskbroker_client.retry import LastAction, Retry from taskbroker_client.task import Task +from taskbroker_client.worker.workerchild import ProcessingDeadlineExceeded from sentry.deletions.models.scheduleddeletion import ( BaseScheduledDeletion, @@ -17,7 +18,7 @@ from sentry.exceptions import DeleteAborted from sentry.signals import pending_delete from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import deletion_control_tasks, deletion_tasks from sentry.utils.env import in_test_environment @@ -102,10 +103,12 @@ def _run_scheduled_deletions( times=MAX_RETRIES, times_exceeded=LastAction.Discard, delay=60 * 5, + on=(Exception, ProcessingDeadlineExceeded), + ignore=(DeleteAborted,), ), silo_mode=SiloMode.CONTROL, + silenced_exceptions=(DeleteAborted,), ) -@retry(exclude=(DeleteAborted,), timeouts=True) def run_deletion_control(deletion_id: int, first_pass: bool = True, **kwargs: Any) -> None: _run_deletion( deletion_id=deletion_id, @@ -123,10 +126,12 @@ def run_deletion_control(deletion_id: int, first_pass: bool = True, **kwargs: An times=MAX_RETRIES, times_exceeded=LastAction.Discard, delay=60 * 5, + on=(Exception, ProcessingDeadlineExceeded), + ignore=(DeleteAborted,), ), silo_mode=SiloMode.CELL, + silenced_exceptions=(DeleteAborted,), ) -@retry(exclude=(DeleteAborted,), timeouts=True) def run_deletion(deletion_id: int, first_pass: bool = True, **kwargs: Any) -> None: _run_deletion( deletion_id=deletion_id, From 2a171e2bc4316e6c9896c49154232c13e015ae9e Mon Sep 17 00:00:00 2001 From: Gabe Villalobos Date: Wed, 6 May 2026 15:14:37 -0700 Subject: [PATCH 3/5] Corrects a couple incorrect retries --- src/sentry/deletions/tasks/groups.py | 8 +++++++- src/sentry/tasks/summaries/weekly_reports.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/sentry/deletions/tasks/groups.py b/src/sentry/deletions/tasks/groups.py index 825db509d547df..bd6e77f9bf5a52 100644 --- a/src/sentry/deletions/tasks/groups.py +++ b/src/sentry/deletions/tasks/groups.py @@ -19,7 +19,13 @@ name="sentry.deletions.tasks.groups.delete_groups_for_project", namespace=deletion_tasks, retry=Retry( - times=MAX_RETRIES, delay=60 * 5, on=(ProcessingDeadlineExceeded,), ignore=(DeleteAborted,) + times=MAX_RETRIES, + delay=60 * 5, + on=( + Exception, + ProcessingDeadlineExceeded, + ), + ignore=(DeleteAborted,), ), silo_mode=SiloMode.CELL, silenced_exceptions=(DeleteAborted,), diff --git a/src/sentry/tasks/summaries/weekly_reports.py b/src/sentry/tasks/summaries/weekly_reports.py index 3a78514a0a119a..06b41f635453a5 100644 --- a/src/sentry/tasks/summaries/weekly_reports.py +++ b/src/sentry/tasks/summaries/weekly_reports.py @@ -101,7 +101,13 @@ def delete_min_org_id(self) -> None: @instrumented_task( name="sentry.tasks.summaries.weekly_reports.schedule_organizations", namespace=reports_tasks, - retry=Retry(times=5, on=(ProcessingDeadlineExceeded,)), + retry=Retry( + times=5, + on=( + Exception, + ProcessingDeadlineExceeded, + ), + ), processing_deadline_duration=timedelta(minutes=30), silo_mode=SiloMode.CELL, ) From 453057f8887157fd9e630f3d24fd44caf14c62c2 Mon Sep 17 00:00:00 2001 From: Gabe Villalobos Date: Wed, 6 May 2026 16:30:15 -0700 Subject: [PATCH 4/5] Removes test that assumes task retry decorator logic --- tests/sentry/tasks/test_code_owners.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/sentry/tasks/test_code_owners.py b/tests/sentry/tasks/test_code_owners.py index e0a87bb1687b0f..1ff620d6ac1953 100644 --- a/tests/sentry/tasks/test_code_owners.py +++ b/tests/sentry/tasks/test_code_owners.py @@ -1,8 +1,6 @@ from unittest.mock import MagicMock, patch -import pytest from rest_framework.exceptions import NotFound -from taskbroker_client.retry import RetryTaskError from sentry.api.validators.project_codeowners import build_codeowners_associations from sentry.integrations.models.external_actor import ExternalActor @@ -303,16 +301,6 @@ def test_post_bulk_create_commit_file_changes_triggers_auto_sync_task( mock_code_owners_auto_sync.delay.assert_called_once_with(commit_id=commit.id) - def test_codeowners_auto_sync_retries_on_missing_commit(self) -> None: - non_existent_commit_id = 999999999 - - with self.tasks() and self.feature({"organizations:integrations-codeowners": True}): - with pytest.raises(RetryTaskError): - code_owners_auto_sync(non_existent_commit_id) - - code_owners = ProjectCodeOwners.objects.get(id=self.code_owners.id) - assert code_owners.raw == self.data["raw"] - @patch( "sentry.integrations.github.integration.GitHubIntegration.get_codeowner_file", side_effect=NotImplementedError("Integration does not support CODEOWNERS"), From 799d45cdf768650775ab1f4e84dffc97a6b5f3b9 Mon Sep 17 00:00:00 2001 From: Gabe Villalobos Date: Thu, 7 May 2026 11:32:12 -0700 Subject: [PATCH 5/5] Updates tests --- tests/sentry/tasks/test_weekly_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sentry/tasks/test_weekly_reports.py b/tests/sentry/tasks/test_weekly_reports.py index ff82f73926c37c..86d5acdd8a462b 100644 --- a/tests/sentry/tasks/test_weekly_reports.py +++ b/tests/sentry/tasks/test_weekly_reports.py @@ -1208,7 +1208,7 @@ def test_email_override_invalid_target_user(self, logger: mock.MagicMock) -> Non org.id, batch_id=batch_id, dry_run=False, - target_user="dummy", + target_user=None, email_override="doesntmatter@smad.com", ) @@ -1217,7 +1217,7 @@ def test_email_override_invalid_target_user(self, logger: mock.MagicMock) -> Non extra={ "batch_id": str(batch_id), "organization": org.id, - "target_user": "dummy", + "target_user": None, "email_override": "doesntmatter@smad.com", }, )