From 5c2548a236d6c714f1314f5b88821ead8ba92258 Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Thu, 17 Sep 2026 14:10:20 -0700 Subject: [PATCH 1/5] feat(seer): Curate autofix issue labels Sample low-scoring issues and control cohorts after Night Shift, then use an Opus judge to record blinded, event-matched fixability reviews with a per-organization cost cap. --- src/sentry/conf/server.py | 1 + src/sentry/seer/night_shift/delivery.py | 16 +- src/sentry/tasks/seer/autofix_issue_data.py | 216 ++++++++++++++++++ src/sentry/testutils/factories.py | 13 ++ src/sentry/testutils/fixtures.py | 3 + .../sentry/seer/night_shift/test_delivery.py | 6 + tests/sentry/tasks/seer/test_autofix.py | 108 ++++++++- 7 files changed, 361 insertions(+), 2 deletions(-) create mode 100644 src/sentry/tasks/seer/autofix_issue_data.py diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 70085caf73ac..7e9287378aed 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1013,6 +1013,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "sentry.tasks.seer.lightweight_rca_cluster", "sentry.tasks.seer.investigation", "sentry.tasks.seer.night_shift.cron", + "sentry.tasks.seer.autofix_issue_data", "sentry.tasks.seer.backfill_supergroups_lightweight", # Used for tests "sentry.taskworker.tasks.examples", diff --git a/src/sentry/seer/night_shift/delivery.py b/src/sentry/seer/night_shift/delivery.py index 0bbfccbd0a08..f82570a36933 100644 --- a/src/sentry/seer/night_shift/delivery.py +++ b/src/sentry/seer/night_shift/delivery.py @@ -37,6 +37,7 @@ SeerWorkflowStrategy, ) from sentry.seer.night_shift.models import TriageResponse, TriageVerdict +from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.types.activity import ActivityType @@ -392,8 +393,10 @@ def _process_verdicts( SeerNightShiftRunResult.objects.bulk_create(rows, ignore_conflicts=True) captured_event_ids: dict[int, str] = {} + capture_enabled = False try: - if features.has("organizations:seer-fixability-training-data", organization): + capture_enabled = features.has("organizations:seer-fixability-training-data", organization) + if capture_enabled: captured_event_ids = _capture_autofix_issue_data( organization=organization, verdicts=verdicts, @@ -425,3 +428,14 @@ def _process_verdicts( ], }, ) + + if capture_enabled: + try: + schedule_judging_for_org.apply_async( + args=[organization.id], + headers={"sentry-propagate-traces": False}, + ) + except Exception: + logger.exception( + "night_shift.autofix_issue_data.judge_dispatch_failed", extra=log_extra + ) diff --git a/src/sentry/tasks/seer/autofix_issue_data.py b/src/sentry/tasks/seer/autofix_issue_data.py new file mode 100644 index 000000000000..e342fbedfccf --- /dev/null +++ b/src/sentry/tasks/seer/autofix_issue_data.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import logging +from enum import StrEnum +from typing import Any + +from django.db.models import F, Window +from django.db.models.functions import PercentRank +from django.utils import timezone +from pydantic import BaseModel, Field +from taskbroker_client.retry import Retry + +from sentry import features +from sentry.models.organization import Organization +from sentry.ratelimits import backend as ratelimiter +from sentry.seer.models import SeerApiError +from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData +from sentry.seer.signed_seer_api import ( + LlmGenerateRequest, + SeerViewerContext, + make_llm_generate_request, +) +from sentry.tasks.base import instrumented_task +from sentry.taskworker.namespaces import seer_tasks +from sentry.utils import json, metrics + +logger = logging.getLogger(__name__) + +FEATURE_FLAG = "organizations:seer-fixability-training-data" +MAX_REVIEWS_PER_ORG_PER_DAY = 20 +BOTTOM_SAMPLE_SIZE = 16 +MIDDLE_SAMPLE_SIZE = 2 +TOP_SAMPLE_SIZE = 2 +RATE_LIMIT_WINDOW = 24 * 60 * 60 +PROMPT_VERSION = "1" + +SYSTEM_PROMPT = """Night Shift reviews software issues and may trigger Autofix to investigate +and open a pull request. Your job is to identify issues where opening a pull request would be +wasteful because the issue cannot be fixed in the relevant codebase. + +An issue is fixable when it can reasonably be resolved with one or two pull requests to the +relevant codebase. It is not_fixable when it cannot. Choose uncertain only when the supplied +evidence is insufficient to decide. Use only the supplied issue and event evidence. Do not +assume an attempted fix, pull request, or prior automated decision. + +Return only a JSON object matching this shape, with a concise one-to-two-sentence reason: +{"verdict":"fixable|not_fixable|uncertain","confidence":"high|medium|low","reason":"..."} +""" + + +class JudgeVerdict(StrEnum): + FIXABLE = "fixable" + NOT_FIXABLE = "not_fixable" + UNCERTAIN = "uncertain" + + +class JudgeConfidence(StrEnum): + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + + +class JudgeResponse(BaseModel): + verdict: JudgeVerdict + confidence: JudgeConfidence + reason: str = Field(min_length=1, max_length=2048) + + +def _eligible_rows(organization_id: int): + return ( + SeerAutofixIssueData.objects.filter( + organization_id=organization_id, + judge_review__isnull=True, + group__seer_fixability_score__isnull=False, + ) + .exclude(raw_issue_data__status="pr_merged") + .select_related("group") + .annotate( + score_percentile=Window( + expression=PercentRank(), + order_by=F("group__seer_fixability_score").asc(), + ) + ) + ) + + +def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: + rows = _eligible_rows(organization_id) + bottom = list( + rows.filter(score_percentile__lte=0.1).order_by("group__seer_fixability_score", "id")[ + :BOTTOM_SAMPLE_SIZE + ] + ) + middle = list( + rows.filter(score_percentile__gte=0.4, score_percentile__lte=0.6).order_by("?")[ + :MIDDLE_SAMPLE_SIZE + ] + ) + top = list(rows.filter(score_percentile__gte=0.9).order_by("?")[:TOP_SAMPLE_SIZE]) + return [*bottom, *middle, *top] + + +def _dispatch_rate_limited(organization_id: int) -> bool: + return ratelimiter.is_limited( + f"autofix_issue_data_judge:org:{organization_id}", + limit=MAX_REVIEWS_PER_ORG_PER_DAY, + window=RATE_LIMIT_WINDOW, + ) + + +@instrumented_task( + name="sentry.tasks.seer.autofix_issue_data.schedule_judging_for_org", + namespace=seer_tasks, + processing_deadline_duration=5 * 60, +) +def schedule_judging_for_org(organization_id: int) -> None: + organization = Organization.objects.filter(id=organization_id).first() + if organization is None or not features.has(FEATURE_FLAG, organization): + return + + for issue_data in _select_candidates(organization.id): + event_id = issue_data.raw_issue_data.get("event_id") + if not isinstance(event_id, str): + continue + if _dispatch_rate_limited(organization.id): + break + judge_issue_data.apply_async( + args=[issue_data.id, event_id], + headers={"sentry-propagate-traces": False}, + ) + + +def _parse_response(content: str) -> JudgeResponse: + value = content.strip() + if value.startswith("```json") and value.endswith("```"): + value = value[7:-3].strip() + return JudgeResponse.parse_obj(json.loads(value)) + + +def _build_prompt(issue_data: SeerAutofixIssueData) -> str: + return json.dumps( + { + key: value + for key, value in issue_data.raw_issue_data.items() + if key not in {"status", "reason"} + } + ) + + +@instrumented_task( + name="sentry.tasks.seer.autofix_issue_data.judge", + namespace=seer_tasks, + processing_deadline_duration=60, + retry=Retry(times=2, delay=30, on=(Exception,)), +) +def judge_issue_data(issue_data_id: int, event_id: str) -> None: + issue_data = ( + SeerAutofixIssueData.objects.select_related("organization").filter(id=issue_data_id).first() + ) + if issue_data is None or not features.has(FEATURE_FLAG, issue_data.organization): + return + if issue_data.judge_review is not None: + return + if issue_data.raw_issue_data.get("event_id") != event_id: + metrics.incr("autofix_issue_data.judge.skipped", tags={"reason": "stale_event"}) + return + + body = LlmGenerateRequest( + provider="anthropic", + model="opus", + referrer="sentry.autofix_issue_data.judge", + prompt=_build_prompt(issue_data), + system_prompt=SYSTEM_PROMPT, + temperature=0.0, + max_tokens=1000, + timeout=25, + reasoning="high", + conversation_id=None, + ) + response = make_llm_generate_request( + body, + timeout=30, + viewer_context=SeerViewerContext(organization_id=issue_data.organization_id), + ) + if response.status >= 400: + raise SeerApiError("Seer autofix issue data judge request failed", response.status) + + response_data: dict[str, Any] = response.json() + content = response_data.get("content") + model = response_data.get("model") + if not isinstance(content, str) or not isinstance(model, str): + raise ValueError("Seer autofix issue data judge returned an invalid response") + result = _parse_response(content) + reviewed_at = timezone.now() + + updated = SeerAutofixIssueData.objects.filter( + id=issue_data.id, + judge_review__isnull=True, + raw_issue_data__event_id=event_id, + ).update( + judge_review={ + "reviewer": "llm_judge", + "verdict": result.verdict.value, + "confidence": result.confidence.value, + "reason": result.reason, + "model": model, + "prompt_version": PROMPT_VERSION, + "reviewed_at": reviewed_at.isoformat(), + "reviewed_event_id": event_id, + }, + date_updated=reviewed_at, + ) + metrics.incr( + "autofix_issue_data.judge.completed" if updated else "autofix_issue_data.judge.skipped", + tags={} if updated else {"reason": "stale_event"}, + ) diff --git a/src/sentry/testutils/factories.py b/src/sentry/testutils/factories.py index 1301fb59a0c8..7c8ef5725104 100644 --- a/src/sentry/testutils/factories.py +++ b/src/sentry/testutils/factories.py @@ -179,6 +179,7 @@ from sentry.replays.models import DeletionJobStatus, ReplayDeletionJobModel from sentry.seer.autofix.constants import CodingAgentStatus from sentry.seer.models.agent_write_grant import SeerAgentWriteGrant +from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData from sentry.seer.models.project_repository import SeerProjectRepository from sentry.seer.models.run import ( SeerAgentRun, @@ -3242,6 +3243,18 @@ def create_github_provider(**kwargs) -> IdentityProvider: ) return identity_provider + @staticmethod + @assume_test_silo_mode(SiloMode.CELL) + def create_seer_autofix_issue_data(group: Group, **kwargs) -> SeerAutofixIssueData: + kwargs.setdefault("organization_id", group.project.organization_id) + kwargs.setdefault("project_id", group.project_id) + kwargs.setdefault("source", "night_shift") + kwargs.setdefault( + "raw_issue_data", + {"event_id": "a" * 32, "event": {}, "issue": {}, "status": "skip"}, + ) + return SeerAutofixIssueData.objects.create(group=group, **kwargs) + @staticmethod @assume_test_silo_mode(SiloMode.CELL) def create_seer_agent_write_grant(organization, user, session_id: str = "s1", **kwargs): diff --git a/src/sentry/testutils/fixtures.py b/src/sentry/testutils/fixtures.py index 5d636c028002..1366761be957 100644 --- a/src/sentry/testutils/fixtures.py +++ b/src/sentry/testutils/fixtures.py @@ -1330,6 +1330,9 @@ def create_preprod_artifact_pair_for_comparison( return head_artifact, head_size_metrics, base_artifact, base_size_metrics + def create_seer_autofix_issue_data(self, group, **kwargs): + return Factories.create_seer_autofix_issue_data(group=group, **kwargs) + def create_seer_run(self, organization=None, **kwargs): if organization is None: organization = self.organization diff --git a/tests/sentry/seer/night_shift/test_delivery.py b/tests/sentry/seer/night_shift/test_delivery.py index 8ae4409259a1..1fd7d31e11f1 100644 --- a/tests/sentry/seer/night_shift/test_delivery.py +++ b/tests/sentry/seer/night_shift/test_delivery.py @@ -198,9 +198,15 @@ def test_autofix_issue_data_captured_when_enabled(self) -> None: "sentry.seer.night_shift.delivery._get_serialized_event", return_value=(event["eventID"], event), ), + patch( + "sentry.seer.night_shift.delivery.schedule_judging_for_org.apply_async" + ) as mock_schedule, ): self._deliver_dry_run_verdict(org, group.id, "fixable") + mock_schedule.assert_called_once_with( + args=[org.id], headers={"sentry-propagate-traces": False} + ) row = SeerAutofixIssueData.objects.get(group=group) assert row.organization_id == org.id assert row.project_id == group.project_id diff --git a/tests/sentry/tasks/seer/test_autofix.py b/tests/sentry/tasks/seer/test_autofix.py index c0debca86961..6c5bccd113be 100644 --- a/tests/sentry/tasks/seer/test_autofix.py +++ b/tests/sentry/tasks/seer/test_autofix.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch import pytest @@ -13,7 +13,15 @@ configure_seer_for_existing_org, generate_issue_summary_only, ) +from sentry.tasks.seer.autofix_issue_data import ( + FEATURE_FLAG, + _parse_response, + _select_candidates, + judge_issue_data, + schedule_judging_for_org, +) from sentry.testutils.cases import TestCase as SentryTestCase +from sentry.utils import json from sentry.utils.cache import cache @@ -56,6 +64,104 @@ def test_generates_fixability_score_after_summary( assert group.seer_fixability_score == 0.75 +class TestAutofixIssueDataJudge(SentryTestCase): + def test_selects_bottom_decile_and_control_samples(self) -> None: + for index in range(11): + group = self.create_group( + project=self.project, + seer_fixability_score=index / 10, + ) + self.create_seer_autofix_issue_data(group) + + candidates = _select_candidates(self.organization.id) + scores = [candidate.group.seer_fixability_score for candidate in candidates] + + assert len(scores) == 6 + assert set(scores) >= {0.0, 0.1, 0.9, 1.0} + assert len([score for score in scores if score is not None and 0.4 <= score <= 0.6]) == 2 + + @patch("sentry.tasks.seer.autofix_issue_data.judge_issue_data.apply_async") + @patch("sentry.tasks.seer.autofix_issue_data._dispatch_rate_limited") + @patch("sentry.tasks.seer.autofix_issue_data._select_candidates") + def test_limits_dispatches_to_twenty_per_organization( + self, + mock_select_candidates: MagicMock, + mock_rate_limited: MagicMock, + mock_apply_async: MagicMock, + ) -> None: + mock_select_candidates.return_value = [ + Mock(id=index, raw_issue_data={"event_id": str(index)}) for index in range(21) + ] + mock_rate_limited.side_effect = [False] * 20 + [True] + + with self.feature(FEATURE_FLAG): + schedule_judging_for_org(self.organization.id) + + assert mock_apply_async.call_count == 20 + + def test_accepts_all_verdicts(self) -> None: + for verdict in ("fixable", "not_fixable", "uncertain"): + response = _parse_response( + json.dumps({"verdict": verdict, "confidence": "high", "reason": "Evidence"}) + ) + assert response.verdict.value == verdict + + @patch("sentry.tasks.seer.autofix_issue_data.make_llm_generate_request") + def test_judges_blinded_issue_data_and_records_verdict( + self, + mock_request: MagicMock, + ) -> None: + verdict = "not_fixable" + event_id = "b" * 32 + group = self.create_group(project=self.project, seer_fixability_score=0.1) + issue_data = self.create_seer_autofix_issue_data( + group, + raw_issue_data={ + "event_id": event_id, + "event": {"entries": []}, + "issue": {"title": "Example"}, + "status": "skip", + "reason": "hidden", + }, + ) + response = Mock(status=200) + response.json.return_value = { + "content": json.dumps({"verdict": verdict, "confidence": "high", "reason": "Evidence"}), + "model": "claude-opus-4-8@default", + } + mock_request.return_value = response + + with self.feature(FEATURE_FLAG): + judge_issue_data(issue_data.id, event_id) + + prompt = json.loads(mock_request.call_args.args[0]["prompt"]) + assert prompt == { + "event_id": event_id, + "event": {"entries": []}, + "issue": {"title": "Example"}, + } + + issue_data.refresh_from_db() + assert issue_data.judge_review is not None + assert issue_data.judge_review["verdict"] == verdict + assert issue_data.judge_review["confidence"] == "high" + assert issue_data.judge_review["reviewed_event_id"] == event_id + assert issue_data.judge_review["model"] == "claude-opus-4-8@default" + assert issue_data.judge_review["prompt_version"] == "1" + + @patch("sentry.tasks.seer.autofix_issue_data.make_llm_generate_request") + def test_skips_stale_event(self, mock_request: MagicMock) -> None: + group = self.create_group(project=self.project, seer_fixability_score=0.1) + issue_data = self.create_seer_autofix_issue_data(group) + + with self.feature(FEATURE_FLAG): + judge_issue_data(issue_data.id, "stale-event") + + mock_request.assert_not_called() + issue_data.refresh_from_db() + assert issue_data.judge_review is None + + class TestConfigureSeerForExistingOrg(SentryTestCase): @patch("sentry.tasks.seer.autofix.logger") def test_missing_organization_returns_without_retry(self, mock_logger: MagicMock) -> None: From 711b69ed17a283c74a67100410316c87ecfaad8d Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Thu, 17 Sep 2026 14:19:39 -0700 Subject: [PATCH 2/5] ref(seer): Simplify autofix issue judging --- src/sentry/tasks/seer/autofix_issue_data.py | 72 ++++++--------------- tests/sentry/tasks/seer/test_autofix.py | 4 +- 2 files changed, 23 insertions(+), 53 deletions(-) diff --git a/src/sentry/tasks/seer/autofix_issue_data.py b/src/sentry/tasks/seer/autofix_issue_data.py index e342fbedfccf..81b96a785186 100644 --- a/src/sentry/tasks/seer/autofix_issue_data.py +++ b/src/sentry/tasks/seer/autofix_issue_data.py @@ -1,8 +1,6 @@ from __future__ import annotations -import logging -from enum import StrEnum -from typing import Any +from typing import Literal from django.db.models import F, Window from django.db.models.functions import PercentRank @@ -24,15 +22,11 @@ from sentry.taskworker.namespaces import seer_tasks from sentry.utils import json, metrics -logger = logging.getLogger(__name__) - FEATURE_FLAG = "organizations:seer-fixability-training-data" MAX_REVIEWS_PER_ORG_PER_DAY = 20 BOTTOM_SAMPLE_SIZE = 16 MIDDLE_SAMPLE_SIZE = 2 TOP_SAMPLE_SIZE = 2 -RATE_LIMIT_WINDOW = 24 * 60 * 60 -PROMPT_VERSION = "1" SYSTEM_PROMPT = """Night Shift reviews software issues and may trigger Autofix to investigate and open a pull request. Your job is to identify issues where opening a pull request would be @@ -48,26 +42,14 @@ """ -class JudgeVerdict(StrEnum): - FIXABLE = "fixable" - NOT_FIXABLE = "not_fixable" - UNCERTAIN = "uncertain" - - -class JudgeConfidence(StrEnum): - HIGH = "high" - MEDIUM = "medium" - LOW = "low" - - class JudgeResponse(BaseModel): - verdict: JudgeVerdict - confidence: JudgeConfidence + verdict: Literal["fixable", "not_fixable", "uncertain"] + confidence: Literal["high", "medium", "low"] reason: str = Field(min_length=1, max_length=2048) -def _eligible_rows(organization_id: int): - return ( +def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: + rows = ( SeerAutofixIssueData.objects.filter( organization_id=organization_id, judge_review__isnull=True, @@ -82,10 +64,6 @@ def _eligible_rows(organization_id: int): ) ) ) - - -def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: - rows = _eligible_rows(organization_id) bottom = list( rows.filter(score_percentile__lte=0.1).order_by("group__seer_fixability_score", "id")[ :BOTTOM_SAMPLE_SIZE @@ -100,14 +78,6 @@ def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: return [*bottom, *middle, *top] -def _dispatch_rate_limited(organization_id: int) -> bool: - return ratelimiter.is_limited( - f"autofix_issue_data_judge:org:{organization_id}", - limit=MAX_REVIEWS_PER_ORG_PER_DAY, - window=RATE_LIMIT_WINDOW, - ) - - @instrumented_task( name="sentry.tasks.seer.autofix_issue_data.schedule_judging_for_org", namespace=seer_tasks, @@ -122,7 +92,11 @@ def schedule_judging_for_org(organization_id: int) -> None: event_id = issue_data.raw_issue_data.get("event_id") if not isinstance(event_id, str): continue - if _dispatch_rate_limited(organization.id): + if ratelimiter.is_limited( + f"autofix_issue_data_judge:org:{organization.id}", + limit=MAX_REVIEWS_PER_ORG_PER_DAY, + window=24 * 60 * 60, + ): break judge_issue_data.apply_async( args=[issue_data.id, event_id], @@ -137,16 +111,6 @@ def _parse_response(content: str) -> JudgeResponse: return JudgeResponse.parse_obj(json.loads(value)) -def _build_prompt(issue_data: SeerAutofixIssueData) -> str: - return json.dumps( - { - key: value - for key, value in issue_data.raw_issue_data.items() - if key not in {"status", "reason"} - } - ) - - @instrumented_task( name="sentry.tasks.seer.autofix_issue_data.judge", namespace=seer_tasks, @@ -169,7 +133,13 @@ def judge_issue_data(issue_data_id: int, event_id: str) -> None: provider="anthropic", model="opus", referrer="sentry.autofix_issue_data.judge", - prompt=_build_prompt(issue_data), + prompt=json.dumps( + { + key: value + for key, value in issue_data.raw_issue_data.items() + if key not in {"status", "reason"} + } + ), system_prompt=SYSTEM_PROMPT, temperature=0.0, max_tokens=1000, @@ -185,7 +155,7 @@ def judge_issue_data(issue_data_id: int, event_id: str) -> None: if response.status >= 400: raise SeerApiError("Seer autofix issue data judge request failed", response.status) - response_data: dict[str, Any] = response.json() + response_data = response.json() content = response_data.get("content") model = response_data.get("model") if not isinstance(content, str) or not isinstance(model, str): @@ -200,11 +170,11 @@ def judge_issue_data(issue_data_id: int, event_id: str) -> None: ).update( judge_review={ "reviewer": "llm_judge", - "verdict": result.verdict.value, - "confidence": result.confidence.value, + "verdict": result.verdict, + "confidence": result.confidence, "reason": result.reason, "model": model, - "prompt_version": PROMPT_VERSION, + "prompt_version": "1", "reviewed_at": reviewed_at.isoformat(), "reviewed_event_id": event_id, }, diff --git a/tests/sentry/tasks/seer/test_autofix.py b/tests/sentry/tasks/seer/test_autofix.py index 6c5bccd113be..18512dd96f21 100644 --- a/tests/sentry/tasks/seer/test_autofix.py +++ b/tests/sentry/tasks/seer/test_autofix.py @@ -81,7 +81,7 @@ def test_selects_bottom_decile_and_control_samples(self) -> None: assert len([score for score in scores if score is not None and 0.4 <= score <= 0.6]) == 2 @patch("sentry.tasks.seer.autofix_issue_data.judge_issue_data.apply_async") - @patch("sentry.tasks.seer.autofix_issue_data._dispatch_rate_limited") + @patch("sentry.tasks.seer.autofix_issue_data.ratelimiter.is_limited") @patch("sentry.tasks.seer.autofix_issue_data._select_candidates") def test_limits_dispatches_to_twenty_per_organization( self, @@ -104,7 +104,7 @@ def test_accepts_all_verdicts(self) -> None: response = _parse_response( json.dumps({"verdict": verdict, "confidence": "high", "reason": "Evidence"}) ) - assert response.verdict.value == verdict + assert response.verdict == verdict @patch("sentry.tasks.seer.autofix_issue_data.make_llm_generate_request") def test_judges_blinded_issue_data_and_records_verdict( From 24275b0d1a44581d1e4252ed306ec1f754c16f13 Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Thu, 17 Sep 2026 14:42:34 -0700 Subject: [PATCH 3/5] fix(seer): Schedule issue judging once per Night Shift run --- src/sentry/seer/night_shift/delivery.py | 16 +--------------- src/sentry/tasks/seer/night_shift/cron.py | 11 +++++++++++ tests/sentry/seer/night_shift/test_delivery.py | 6 ------ tests/sentry/tasks/seer/test_night_shift.py | 12 ++++++++++++ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/sentry/seer/night_shift/delivery.py b/src/sentry/seer/night_shift/delivery.py index f82570a36933..0bbfccbd0a08 100644 --- a/src/sentry/seer/night_shift/delivery.py +++ b/src/sentry/seer/night_shift/delivery.py @@ -37,7 +37,6 @@ SeerWorkflowStrategy, ) from sentry.seer.night_shift.models import TriageResponse, TriageVerdict -from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.types.activity import ActivityType @@ -393,10 +392,8 @@ def _process_verdicts( SeerNightShiftRunResult.objects.bulk_create(rows, ignore_conflicts=True) captured_event_ids: dict[int, str] = {} - capture_enabled = False try: - capture_enabled = features.has("organizations:seer-fixability-training-data", organization) - if capture_enabled: + if features.has("organizations:seer-fixability-training-data", organization): captured_event_ids = _capture_autofix_issue_data( organization=organization, verdicts=verdicts, @@ -428,14 +425,3 @@ def _process_verdicts( ], }, ) - - if capture_enabled: - try: - schedule_judging_for_org.apply_async( - args=[organization.id], - headers={"sentry-propagate-traces": False}, - ) - except Exception: - logger.exception( - "night_shift.autofix_issue_data.judge_dispatch_failed", extra=log_extra - ) diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index b77ece8095fe..c73faa6fb3da 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -51,6 +51,7 @@ from sentry.seer.night_shift.models import NightShiftPayload, TriageCandidate, TriageTweaks from sentry.seer.workflows.schemas import WorkflowRunSource from sentry.tasks.base import instrumented_task +from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.simple_triage import ( fixability_score_strategy, fixability_score_strategy_per_project, @@ -639,6 +640,16 @@ def _complete_run(run: SeerWorkflowRun) -> None: extras.pop("error_type", None) locked_run.update(extras=extras, date_completed=timezone.now()) + try: + schedule_judging_for_org.apply_async( + args=[run.organization_id], headers={"sentry-propagate-traces": False} + ) + except Exception: + logger.exception( + "night_shift.autofix_issue_data.judge_dispatch_failed", + extra={"organization_id": run.organization_id, "night_shift_run_id": run.id}, + ) + def _record_run_error( run: SeerWorkflowRun, error_type: SeerNightShiftRunErrorType, message: str diff --git a/tests/sentry/seer/night_shift/test_delivery.py b/tests/sentry/seer/night_shift/test_delivery.py index 1fd7d31e11f1..8ae4409259a1 100644 --- a/tests/sentry/seer/night_shift/test_delivery.py +++ b/tests/sentry/seer/night_shift/test_delivery.py @@ -198,15 +198,9 @@ def test_autofix_issue_data_captured_when_enabled(self) -> None: "sentry.seer.night_shift.delivery._get_serialized_event", return_value=(event["eventID"], event), ), - patch( - "sentry.seer.night_shift.delivery.schedule_judging_for_org.apply_async" - ) as mock_schedule, ): self._deliver_dry_run_verdict(org, group.id, "fixable") - mock_schedule.assert_called_once_with( - args=[org.id], headers={"sentry-propagate-traces": False} - ) row = SeerAutofixIssueData.objects.get(group=group) assert row.organization_id == org.id assert row.project_id == group.project_id diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index 8950170885ee..e5ca63621f96 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -44,6 +44,7 @@ from sentry.tasks.seer.night_shift.skip_cache import key as skip_cache_key from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.testutils.cases import SnubaTestCase, TestCase +from sentry.testutils.factories import Factories from sentry.testutils.fixtures import Fixtures from sentry.testutils.helpers.datetime import before_now, freeze_time from sentry.testutils.helpers.features import with_feature @@ -644,6 +645,17 @@ def test_completed_run_ignores_stale_extras_update(self) -> None: assert run.extras.get("error_type") is None assert "num_candidates" not in run.extras + @patch("sentry.tasks.seer.night_shift.cron.schedule_judging_for_org.apply_async") + def test_complete_run_schedules_issue_data_judging_once(self, mock_schedule) -> None: + run = Factories.create_seer_workflow_run(organization=self.organization) + + _complete_run(run) + _complete_run(run) + + mock_schedule.assert_called_once_with( + args=[self.organization.id], headers={"sentry-propagate-traces": False} + ) + def test_extras_update_refreshes_run_instance(self) -> None: org = self.create_organization() From ccf72122420a0f0147cd7cb0386c617c47927577 Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Thu, 17 Sep 2026 16:32:30 -0700 Subject: [PATCH 4/5] fix(seer): Judge issue data after Night Shift delivery --- src/sentry/seer/night_shift/delivery.py | 43 +++++++++++++++++++ src/sentry/tasks/seer/night_shift/cron.py | 11 ----- .../sentry/seer/night_shift/test_delivery.py | 36 ++++++++++++++++ tests/sentry/tasks/seer/test_night_shift.py | 12 ------ 4 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/sentry/seer/night_shift/delivery.py b/src/sentry/seer/night_shift/delivery.py index 0bbfccbd0a08..688e8316ac94 100644 --- a/src/sentry/seer/night_shift/delivery.py +++ b/src/sentry/seer/night_shift/delivery.py @@ -8,6 +8,7 @@ from uuid import UUID import sentry_sdk +from django.db import router, transaction from sentry import features from sentry.api.serializers import EventSerializer, serialize @@ -37,6 +38,7 @@ SeerWorkflowStrategy, ) from sentry.seer.night_shift.models import TriageResponse, TriageVerdict +from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.types.activity import ActivityType @@ -131,6 +133,44 @@ def _capture_autofix_issue_data( return event_ids +def _schedule_judging_after_delivery( + shard: SeerWorkflowRunExecution, log_extra: Mapping[str, object] +) -> None: + judging_enabled = features.has( + "organizations:seer-fixability-training-data", shard.run.organization + ) + using = router.db_for_write(SeerWorkflowRun) + with transaction.atomic(using=using): + locked_run = SeerWorkflowRun.objects.select_for_update().get(id=shard.run_id) + locked_shard = SeerWorkflowRunExecution.objects.select_for_update().get(id=shard.id) + shard_extras = { + **(locked_shard.extras or {}), + "autofix_issue_data_delivery_completed": True, + } + locked_shard.update(extras=shard_extras) + + run_extras = dict(locked_run.extras or {}) + completed_deliveries = SeerWorkflowRunExecution.objects.filter( + run=locked_run, extras__autofix_issue_data_delivery_completed=True + ).count() + if ( + not judging_enabled + or run_extras.get("autofix_issue_data_judging_scheduled") + or completed_deliveries != locked_run.executions.count() + ): + return + + run_extras["autofix_issue_data_judging_scheduled"] = True + locked_run.update(extras=run_extras) + + try: + schedule_judging_for_org.apply_async( + args=[shard.run.organization_id], headers={"sentry-propagate-traces": False} + ) + except Exception: + logger.exception("night_shift.autofix_issue_data.judge_dispatch_failed", extra=log_extra) + + def deliver_night_shift_result( organization_id: int, run_uuid: UUID, @@ -183,6 +223,7 @@ def deliver_night_shift_result( attributes={"error_type": "delivery_error" if status == "error" else "no_artifact"}, ) logger.warning("night_shift.delivery.no_result", extra={**log_extra, "status": status}) + _schedule_judging_after_delivery(shard, log_extra) return try: @@ -192,6 +233,7 @@ def deliver_night_shift_result( "night_shift.triage_error", 1, attributes={"error_type": "invalid_artifact"} ) logger.exception("night_shift.delivery.invalid_result", extra=log_extra) + _schedule_judging_after_delivery(shard, log_extra) return options = (run.extras or {}).get("options") or {} @@ -212,6 +254,7 @@ def deliver_night_shift_result( prompt_version=prompt_version, log_extra=log_extra, ) + _schedule_judging_after_delivery(shard, log_extra) def _process_verdicts( diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index c73faa6fb3da..b77ece8095fe 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -51,7 +51,6 @@ from sentry.seer.night_shift.models import NightShiftPayload, TriageCandidate, TriageTweaks from sentry.seer.workflows.schemas import WorkflowRunSource from sentry.tasks.base import instrumented_task -from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.simple_triage import ( fixability_score_strategy, fixability_score_strategy_per_project, @@ -640,16 +639,6 @@ def _complete_run(run: SeerWorkflowRun) -> None: extras.pop("error_type", None) locked_run.update(extras=extras, date_completed=timezone.now()) - try: - schedule_judging_for_org.apply_async( - args=[run.organization_id], headers={"sentry-propagate-traces": False} - ) - except Exception: - logger.exception( - "night_shift.autofix_issue_data.judge_dispatch_failed", - extra={"organization_id": run.organization_id, "night_shift_run_id": run.id}, - ) - def _record_run_error( run: SeerWorkflowRun, error_type: SeerNightShiftRunErrorType, message: str diff --git a/tests/sentry/seer/night_shift/test_delivery.py b/tests/sentry/seer/night_shift/test_delivery.py index 8ae4409259a1..f09a7cbbae1f 100644 --- a/tests/sentry/seer/night_shift/test_delivery.py +++ b/tests/sentry/seer/night_shift/test_delivery.py @@ -107,6 +107,42 @@ def test_error_status_records_error_and_returns(self) -> None: assert shard.extras["error_type"] == SeerNightShiftRunErrorType.SHARD_DELIVERY_FAILED.value assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + def test_schedules_judging_after_final_shard_delivery(self) -> None: + org = self.create_organization() + run = self._create_night_shift_run(organization=org) + first_shard = run.executions.get() + assert first_shard.seer_run is not None + second_seer_run = self.create_seer_run(organization=org) + SeerWorkflowRunExecution.objects.create(run=run, seer_run=second_seer_run) + + with ( + self.feature("organizations:seer-fixability-training-data"), + patch( + "sentry.seer.night_shift.delivery.schedule_judging_for_org.apply_async" + ) as mock_schedule, + ): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=first_shard.seer_run.uuid, + status="error", + result=None, + error="Seer exploded", + ) + mock_schedule.assert_not_called() + + for _ in range(2): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=second_seer_run.uuid, + status="completed", + result={"verdicts": []}, + error=None, + ) + + mock_schedule.assert_called_once_with( + args=[org.id], headers={"sentry-propagate-traces": False} + ) + def test_sibling_shard_success_keeps_other_shard_error(self) -> None: """A successful shard delivery must not clear an error a sibling shard recorded on the same run.""" diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index e5ca63621f96..8950170885ee 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -44,7 +44,6 @@ from sentry.tasks.seer.night_shift.skip_cache import key as skip_cache_key from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.testutils.cases import SnubaTestCase, TestCase -from sentry.testutils.factories import Factories from sentry.testutils.fixtures import Fixtures from sentry.testutils.helpers.datetime import before_now, freeze_time from sentry.testutils.helpers.features import with_feature @@ -645,17 +644,6 @@ def test_completed_run_ignores_stale_extras_update(self) -> None: assert run.extras.get("error_type") is None assert "num_candidates" not in run.extras - @patch("sentry.tasks.seer.night_shift.cron.schedule_judging_for_org.apply_async") - def test_complete_run_schedules_issue_data_judging_once(self, mock_schedule) -> None: - run = Factories.create_seer_workflow_run(organization=self.organization) - - _complete_run(run) - _complete_run(run) - - mock_schedule.assert_called_once_with( - args=[self.organization.id], headers={"sentry-propagate-traces": False} - ) - def test_extras_update_refreshes_run_instance(self) -> None: org = self.create_organization() From 982224b9e9e6d7d9e89f06237efee9a765a18b07 Mon Sep 17 00:00:00 2001 From: Mihir-Mavalankar Date: Thu, 17 Sep 2026 16:44:42 -0700 Subject: [PATCH 5/5] fix(seer): Sample candidates from bottom 40% of fixability scores --- src/sentry/tasks/seer/autofix_issue_data.py | 17 ++--------------- tests/sentry/tasks/seer/test_autofix.py | 7 +++---- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/src/sentry/tasks/seer/autofix_issue_data.py b/src/sentry/tasks/seer/autofix_issue_data.py index 81b96a785186..aeb84b879650 100644 --- a/src/sentry/tasks/seer/autofix_issue_data.py +++ b/src/sentry/tasks/seer/autofix_issue_data.py @@ -24,9 +24,6 @@ FEATURE_FLAG = "organizations:seer-fixability-training-data" MAX_REVIEWS_PER_ORG_PER_DAY = 20 -BOTTOM_SAMPLE_SIZE = 16 -MIDDLE_SAMPLE_SIZE = 2 -TOP_SAMPLE_SIZE = 2 SYSTEM_PROMPT = """Night Shift reviews software issues and may trigger Autofix to investigate and open a pull request. Your job is to identify issues where opening a pull request would be @@ -49,6 +46,7 @@ class JudgeResponse(BaseModel): def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: + # Randomly sample 20 of the issues from bottom 40% of the fixability score rows = ( SeerAutofixIssueData.objects.filter( organization_id=organization_id, @@ -64,18 +62,7 @@ def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: ) ) ) - bottom = list( - rows.filter(score_percentile__lte=0.1).order_by("group__seer_fixability_score", "id")[ - :BOTTOM_SAMPLE_SIZE - ] - ) - middle = list( - rows.filter(score_percentile__gte=0.4, score_percentile__lte=0.6).order_by("?")[ - :MIDDLE_SAMPLE_SIZE - ] - ) - top = list(rows.filter(score_percentile__gte=0.9).order_by("?")[:TOP_SAMPLE_SIZE]) - return [*bottom, *middle, *top] + return list(rows.filter(score_percentile__lte=0.4).order_by("?")[:MAX_REVIEWS_PER_ORG_PER_DAY]) @instrumented_task( diff --git a/tests/sentry/tasks/seer/test_autofix.py b/tests/sentry/tasks/seer/test_autofix.py index 18512dd96f21..ed6a3e22900c 100644 --- a/tests/sentry/tasks/seer/test_autofix.py +++ b/tests/sentry/tasks/seer/test_autofix.py @@ -65,7 +65,7 @@ def test_generates_fixability_score_after_summary( class TestAutofixIssueDataJudge(SentryTestCase): - def test_selects_bottom_decile_and_control_samples(self) -> None: + def test_selects_bottom_forty_percent(self) -> None: for index in range(11): group = self.create_group( project=self.project, @@ -76,9 +76,8 @@ def test_selects_bottom_decile_and_control_samples(self) -> None: candidates = _select_candidates(self.organization.id) scores = [candidate.group.seer_fixability_score for candidate in candidates] - assert len(scores) == 6 - assert set(scores) >= {0.0, 0.1, 0.9, 1.0} - assert len([score for score in scores if score is not None and 0.4 <= score <= 0.6]) == 2 + assert len(scores) == 5 + assert all(score is not None and score <= 0.4 for score in scores) @patch("sentry.tasks.seer.autofix_issue_data.judge_issue_data.apply_async") @patch("sentry.tasks.seer.autofix_issue_data.ratelimiter.is_limited")