diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 70085caf73ac..7e9287378aed 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1013,6 +1013,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "sentry.tasks.seer.lightweight_rca_cluster", "sentry.tasks.seer.investigation", "sentry.tasks.seer.night_shift.cron", + "sentry.tasks.seer.autofix_issue_data", "sentry.tasks.seer.backfill_supergroups_lightweight", # Used for tests "sentry.taskworker.tasks.examples", diff --git a/src/sentry/seer/night_shift/delivery.py b/src/sentry/seer/night_shift/delivery.py index 0bbfccbd0a08..688e8316ac94 100644 --- a/src/sentry/seer/night_shift/delivery.py +++ b/src/sentry/seer/night_shift/delivery.py @@ -8,6 +8,7 @@ from uuid import UUID import sentry_sdk +from django.db import router, transaction from sentry import features from sentry.api.serializers import EventSerializer, serialize @@ -37,6 +38,7 @@ SeerWorkflowStrategy, ) from sentry.seer.night_shift.models import TriageResponse, TriageVerdict +from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import mark_skipped from sentry.types.activity import ActivityType @@ -131,6 +133,44 @@ def _capture_autofix_issue_data( return event_ids +def _schedule_judging_after_delivery( + shard: SeerWorkflowRunExecution, log_extra: Mapping[str, object] +) -> None: + judging_enabled = features.has( + "organizations:seer-fixability-training-data", shard.run.organization + ) + using = router.db_for_write(SeerWorkflowRun) + with transaction.atomic(using=using): + locked_run = SeerWorkflowRun.objects.select_for_update().get(id=shard.run_id) + locked_shard = SeerWorkflowRunExecution.objects.select_for_update().get(id=shard.id) + shard_extras = { + **(locked_shard.extras or {}), + "autofix_issue_data_delivery_completed": True, + } + locked_shard.update(extras=shard_extras) + + run_extras = dict(locked_run.extras or {}) + completed_deliveries = SeerWorkflowRunExecution.objects.filter( + run=locked_run, extras__autofix_issue_data_delivery_completed=True + ).count() + if ( + not judging_enabled + or run_extras.get("autofix_issue_data_judging_scheduled") + or completed_deliveries != locked_run.executions.count() + ): + return + + run_extras["autofix_issue_data_judging_scheduled"] = True + locked_run.update(extras=run_extras) + + try: + schedule_judging_for_org.apply_async( + args=[shard.run.organization_id], headers={"sentry-propagate-traces": False} + ) + except Exception: + logger.exception("night_shift.autofix_issue_data.judge_dispatch_failed", extra=log_extra) + + def deliver_night_shift_result( organization_id: int, run_uuid: UUID, @@ -183,6 +223,7 @@ def deliver_night_shift_result( attributes={"error_type": "delivery_error" if status == "error" else "no_artifact"}, ) logger.warning("night_shift.delivery.no_result", extra={**log_extra, "status": status}) + _schedule_judging_after_delivery(shard, log_extra) return try: @@ -192,6 +233,7 @@ def deliver_night_shift_result( "night_shift.triage_error", 1, attributes={"error_type": "invalid_artifact"} ) logger.exception("night_shift.delivery.invalid_result", extra=log_extra) + _schedule_judging_after_delivery(shard, log_extra) return options = (run.extras or {}).get("options") or {} @@ -212,6 +254,7 @@ def deliver_night_shift_result( prompt_version=prompt_version, log_extra=log_extra, ) + _schedule_judging_after_delivery(shard, log_extra) def _process_verdicts( diff --git a/src/sentry/tasks/seer/autofix_issue_data.py b/src/sentry/tasks/seer/autofix_issue_data.py new file mode 100644 index 000000000000..aeb84b879650 --- /dev/null +++ b/src/sentry/tasks/seer/autofix_issue_data.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from typing import Literal + +from django.db.models import F, Window +from django.db.models.functions import PercentRank +from django.utils import timezone +from pydantic import BaseModel, Field +from taskbroker_client.retry import Retry + +from sentry import features +from sentry.models.organization import Organization +from sentry.ratelimits import backend as ratelimiter +from sentry.seer.models import SeerApiError +from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData +from sentry.seer.signed_seer_api import ( + LlmGenerateRequest, + SeerViewerContext, + make_llm_generate_request, +) +from sentry.tasks.base import instrumented_task +from sentry.taskworker.namespaces import seer_tasks +from sentry.utils import json, metrics + +FEATURE_FLAG = "organizations:seer-fixability-training-data" +MAX_REVIEWS_PER_ORG_PER_DAY = 20 + +SYSTEM_PROMPT = """Night Shift reviews software issues and may trigger Autofix to investigate +and open a pull request. Your job is to identify issues where opening a pull request would be +wasteful because the issue cannot be fixed in the relevant codebase. + +An issue is fixable when it can reasonably be resolved with one or two pull requests to the +relevant codebase. It is not_fixable when it cannot. Choose uncertain only when the supplied +evidence is insufficient to decide. Use only the supplied issue and event evidence. Do not +assume an attempted fix, pull request, or prior automated decision. + +Return only a JSON object matching this shape, with a concise one-to-two-sentence reason: +{"verdict":"fixable|not_fixable|uncertain","confidence":"high|medium|low","reason":"..."} +""" + + +class JudgeResponse(BaseModel): + verdict: Literal["fixable", "not_fixable", "uncertain"] + confidence: Literal["high", "medium", "low"] + reason: str = Field(min_length=1, max_length=2048) + + +def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]: + # Randomly sample 20 of the issues from bottom 40% of the fixability score + rows = ( + SeerAutofixIssueData.objects.filter( + organization_id=organization_id, + judge_review__isnull=True, + group__seer_fixability_score__isnull=False, + ) + .exclude(raw_issue_data__status="pr_merged") + .select_related("group") + .annotate( + score_percentile=Window( + expression=PercentRank(), + order_by=F("group__seer_fixability_score").asc(), + ) + ) + ) + return list(rows.filter(score_percentile__lte=0.4).order_by("?")[:MAX_REVIEWS_PER_ORG_PER_DAY]) + + +@instrumented_task( + name="sentry.tasks.seer.autofix_issue_data.schedule_judging_for_org", + namespace=seer_tasks, + processing_deadline_duration=5 * 60, +) +def schedule_judging_for_org(organization_id: int) -> None: + organization = Organization.objects.filter(id=organization_id).first() + if organization is None or not features.has(FEATURE_FLAG, organization): + return + + for issue_data in _select_candidates(organization.id): + event_id = issue_data.raw_issue_data.get("event_id") + if not isinstance(event_id, str): + continue + if ratelimiter.is_limited( + f"autofix_issue_data_judge:org:{organization.id}", + limit=MAX_REVIEWS_PER_ORG_PER_DAY, + window=24 * 60 * 60, + ): + break + judge_issue_data.apply_async( + args=[issue_data.id, event_id], + headers={"sentry-propagate-traces": False}, + ) + + +def _parse_response(content: str) -> JudgeResponse: + value = content.strip() + if value.startswith("```json") and value.endswith("```"): + value = value[7:-3].strip() + return JudgeResponse.parse_obj(json.loads(value)) + + +@instrumented_task( + name="sentry.tasks.seer.autofix_issue_data.judge", + namespace=seer_tasks, + processing_deadline_duration=60, + retry=Retry(times=2, delay=30, on=(Exception,)), +) +def judge_issue_data(issue_data_id: int, event_id: str) -> None: + issue_data = ( + SeerAutofixIssueData.objects.select_related("organization").filter(id=issue_data_id).first() + ) + if issue_data is None or not features.has(FEATURE_FLAG, issue_data.organization): + return + if issue_data.judge_review is not None: + return + if issue_data.raw_issue_data.get("event_id") != event_id: + metrics.incr("autofix_issue_data.judge.skipped", tags={"reason": "stale_event"}) + return + + body = LlmGenerateRequest( + provider="anthropic", + model="opus", + referrer="sentry.autofix_issue_data.judge", + prompt=json.dumps( + { + key: value + for key, value in issue_data.raw_issue_data.items() + if key not in {"status", "reason"} + } + ), + system_prompt=SYSTEM_PROMPT, + temperature=0.0, + max_tokens=1000, + timeout=25, + reasoning="high", + conversation_id=None, + ) + response = make_llm_generate_request( + body, + timeout=30, + viewer_context=SeerViewerContext(organization_id=issue_data.organization_id), + ) + if response.status >= 400: + raise SeerApiError("Seer autofix issue data judge request failed", response.status) + + response_data = response.json() + content = response_data.get("content") + model = response_data.get("model") + if not isinstance(content, str) or not isinstance(model, str): + raise ValueError("Seer autofix issue data judge returned an invalid response") + result = _parse_response(content) + reviewed_at = timezone.now() + + updated = SeerAutofixIssueData.objects.filter( + id=issue_data.id, + judge_review__isnull=True, + raw_issue_data__event_id=event_id, + ).update( + judge_review={ + "reviewer": "llm_judge", + "verdict": result.verdict, + "confidence": result.confidence, + "reason": result.reason, + "model": model, + "prompt_version": "1", + "reviewed_at": reviewed_at.isoformat(), + "reviewed_event_id": event_id, + }, + date_updated=reviewed_at, + ) + metrics.incr( + "autofix_issue_data.judge.completed" if updated else "autofix_issue_data.judge.skipped", + tags={} if updated else {"reason": "stale_event"}, + ) diff --git a/src/sentry/testutils/factories.py b/src/sentry/testutils/factories.py index 1301fb59a0c8..7c8ef5725104 100644 --- a/src/sentry/testutils/factories.py +++ b/src/sentry/testutils/factories.py @@ -179,6 +179,7 @@ from sentry.replays.models import DeletionJobStatus, ReplayDeletionJobModel from sentry.seer.autofix.constants import CodingAgentStatus from sentry.seer.models.agent_write_grant import SeerAgentWriteGrant +from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData from sentry.seer.models.project_repository import SeerProjectRepository from sentry.seer.models.run import ( SeerAgentRun, @@ -3242,6 +3243,18 @@ def create_github_provider(**kwargs) -> IdentityProvider: ) return identity_provider + @staticmethod + @assume_test_silo_mode(SiloMode.CELL) + def create_seer_autofix_issue_data(group: Group, **kwargs) -> SeerAutofixIssueData: + kwargs.setdefault("organization_id", group.project.organization_id) + kwargs.setdefault("project_id", group.project_id) + kwargs.setdefault("source", "night_shift") + kwargs.setdefault( + "raw_issue_data", + {"event_id": "a" * 32, "event": {}, "issue": {}, "status": "skip"}, + ) + return SeerAutofixIssueData.objects.create(group=group, **kwargs) + @staticmethod @assume_test_silo_mode(SiloMode.CELL) def create_seer_agent_write_grant(organization, user, session_id: str = "s1", **kwargs): diff --git a/src/sentry/testutils/fixtures.py b/src/sentry/testutils/fixtures.py index 5d636c028002..1366761be957 100644 --- a/src/sentry/testutils/fixtures.py +++ b/src/sentry/testutils/fixtures.py @@ -1330,6 +1330,9 @@ def create_preprod_artifact_pair_for_comparison( return head_artifact, head_size_metrics, base_artifact, base_size_metrics + def create_seer_autofix_issue_data(self, group, **kwargs): + return Factories.create_seer_autofix_issue_data(group=group, **kwargs) + def create_seer_run(self, organization=None, **kwargs): if organization is None: organization = self.organization diff --git a/tests/sentry/seer/night_shift/test_delivery.py b/tests/sentry/seer/night_shift/test_delivery.py index 8ae4409259a1..f09a7cbbae1f 100644 --- a/tests/sentry/seer/night_shift/test_delivery.py +++ b/tests/sentry/seer/night_shift/test_delivery.py @@ -107,6 +107,42 @@ def test_error_status_records_error_and_returns(self) -> None: assert shard.extras["error_type"] == SeerNightShiftRunErrorType.SHARD_DELIVERY_FAILED.value assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + def test_schedules_judging_after_final_shard_delivery(self) -> None: + org = self.create_organization() + run = self._create_night_shift_run(organization=org) + first_shard = run.executions.get() + assert first_shard.seer_run is not None + second_seer_run = self.create_seer_run(organization=org) + SeerWorkflowRunExecution.objects.create(run=run, seer_run=second_seer_run) + + with ( + self.feature("organizations:seer-fixability-training-data"), + patch( + "sentry.seer.night_shift.delivery.schedule_judging_for_org.apply_async" + ) as mock_schedule, + ): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=first_shard.seer_run.uuid, + status="error", + result=None, + error="Seer exploded", + ) + mock_schedule.assert_not_called() + + for _ in range(2): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=second_seer_run.uuid, + status="completed", + result={"verdicts": []}, + error=None, + ) + + mock_schedule.assert_called_once_with( + args=[org.id], headers={"sentry-propagate-traces": False} + ) + def test_sibling_shard_success_keeps_other_shard_error(self) -> None: """A successful shard delivery must not clear an error a sibling shard recorded on the same run.""" diff --git a/tests/sentry/tasks/seer/test_autofix.py b/tests/sentry/tasks/seer/test_autofix.py index c0debca86961..ed6a3e22900c 100644 --- a/tests/sentry/tasks/seer/test_autofix.py +++ b/tests/sentry/tasks/seer/test_autofix.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch import pytest @@ -13,7 +13,15 @@ configure_seer_for_existing_org, generate_issue_summary_only, ) +from sentry.tasks.seer.autofix_issue_data import ( + FEATURE_FLAG, + _parse_response, + _select_candidates, + judge_issue_data, + schedule_judging_for_org, +) from sentry.testutils.cases import TestCase as SentryTestCase +from sentry.utils import json from sentry.utils.cache import cache @@ -56,6 +64,103 @@ def test_generates_fixability_score_after_summary( assert group.seer_fixability_score == 0.75 +class TestAutofixIssueDataJudge(SentryTestCase): + def test_selects_bottom_forty_percent(self) -> None: + for index in range(11): + group = self.create_group( + project=self.project, + seer_fixability_score=index / 10, + ) + self.create_seer_autofix_issue_data(group) + + candidates = _select_candidates(self.organization.id) + scores = [candidate.group.seer_fixability_score for candidate in candidates] + + assert len(scores) == 5 + assert all(score is not None and score <= 0.4 for score in scores) + + @patch("sentry.tasks.seer.autofix_issue_data.judge_issue_data.apply_async") + @patch("sentry.tasks.seer.autofix_issue_data.ratelimiter.is_limited") + @patch("sentry.tasks.seer.autofix_issue_data._select_candidates") + def test_limits_dispatches_to_twenty_per_organization( + self, + mock_select_candidates: MagicMock, + mock_rate_limited: MagicMock, + mock_apply_async: MagicMock, + ) -> None: + mock_select_candidates.return_value = [ + Mock(id=index, raw_issue_data={"event_id": str(index)}) for index in range(21) + ] + mock_rate_limited.side_effect = [False] * 20 + [True] + + with self.feature(FEATURE_FLAG): + schedule_judging_for_org(self.organization.id) + + assert mock_apply_async.call_count == 20 + + def test_accepts_all_verdicts(self) -> None: + for verdict in ("fixable", "not_fixable", "uncertain"): + response = _parse_response( + json.dumps({"verdict": verdict, "confidence": "high", "reason": "Evidence"}) + ) + assert response.verdict == verdict + + @patch("sentry.tasks.seer.autofix_issue_data.make_llm_generate_request") + def test_judges_blinded_issue_data_and_records_verdict( + self, + mock_request: MagicMock, + ) -> None: + verdict = "not_fixable" + event_id = "b" * 32 + group = self.create_group(project=self.project, seer_fixability_score=0.1) + issue_data = self.create_seer_autofix_issue_data( + group, + raw_issue_data={ + "event_id": event_id, + "event": {"entries": []}, + "issue": {"title": "Example"}, + "status": "skip", + "reason": "hidden", + }, + ) + response = Mock(status=200) + response.json.return_value = { + "content": json.dumps({"verdict": verdict, "confidence": "high", "reason": "Evidence"}), + "model": "claude-opus-4-8@default", + } + mock_request.return_value = response + + with self.feature(FEATURE_FLAG): + judge_issue_data(issue_data.id, event_id) + + prompt = json.loads(mock_request.call_args.args[0]["prompt"]) + assert prompt == { + "event_id": event_id, + "event": {"entries": []}, + "issue": {"title": "Example"}, + } + + issue_data.refresh_from_db() + assert issue_data.judge_review is not None + assert issue_data.judge_review["verdict"] == verdict + assert issue_data.judge_review["confidence"] == "high" + assert issue_data.judge_review["reviewed_event_id"] == event_id + assert issue_data.judge_review["model"] == "claude-opus-4-8@default" + assert issue_data.judge_review["prompt_version"] == "1" + + @patch("sentry.tasks.seer.autofix_issue_data.make_llm_generate_request") + def test_skips_stale_event(self, mock_request: MagicMock) -> None: + group = self.create_group(project=self.project, seer_fixability_score=0.1) + issue_data = self.create_seer_autofix_issue_data(group) + + with self.feature(FEATURE_FLAG): + judge_issue_data(issue_data.id, "stale-event") + + mock_request.assert_not_called() + issue_data.refresh_from_db() + assert issue_data.judge_review is None + + class TestConfigureSeerForExistingOrg(SentryTestCase): @patch("sentry.tasks.seer.autofix.logger") def test_missing_organization_returns_without_retry(self, mock_logger: MagicMock) -> None: