feat(seer): Curate autofix issue labels

src/sentry/conf/server.py

@@ -1013,6 +1013,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.tasks.seer.lightweight_rca_cluster",
     "sentry.tasks.seer.investigation",
     "sentry.tasks.seer.night_shift.cron",
+    "sentry.tasks.seer.autofix_issue_data",
     "sentry.tasks.seer.backfill_supergroups_lightweight",
     # Used for tests
     "sentry.taskworker.tasks.examples",

src/sentry/seer/night_shift/delivery.py

@@ -8,6 +8,7 @@
 from uuid import UUID
 
 import sentry_sdk
+from django.db import router, transaction
 
 from sentry import features
 from sentry.api.serializers import EventSerializer, serialize
@@ -37,6 +38,7 @@
     SeerWorkflowStrategy,
 )
 from sentry.seer.night_shift.models import TriageResponse, TriageVerdict
+from sentry.tasks.seer.autofix_issue_data import schedule_judging_for_org
 from sentry.tasks.seer.night_shift.models import TriageAction
 from sentry.tasks.seer.night_shift.skip_cache import mark_skipped
 from sentry.types.activity import ActivityType
@@ -131,6 +133,44 @@ def _capture_autofix_issue_data(
     return event_ids
 
 
+def _schedule_judging_after_delivery(
+    shard: SeerWorkflowRunExecution, log_extra: Mapping[str, object]
+) -> None:
+    judging_enabled = features.has(
+        "organizations:seer-fixability-training-data", shard.run.organization
+    )
+    using = router.db_for_write(SeerWorkflowRun)
+    with transaction.atomic(using=using):
+        locked_run = SeerWorkflowRun.objects.select_for_update().get(id=shard.run_id)
+        locked_shard = SeerWorkflowRunExecution.objects.select_for_update().get(id=shard.id)
+        shard_extras = {
+            **(locked_shard.extras or {}),
+            "autofix_issue_data_delivery_completed": True,
+        }
+        locked_shard.update(extras=shard_extras)
+
+        run_extras = dict(locked_run.extras or {})
+        completed_deliveries = SeerWorkflowRunExecution.objects.filter(
+            run=locked_run, extras__autofix_issue_data_delivery_completed=True
+        ).count()
+        if (
+            not judging_enabled
+            or run_extras.get("autofix_issue_data_judging_scheduled")
+            or completed_deliveries != locked_run.executions.count()
+        ):
+            return
+
+        run_extras["autofix_issue_data_judging_scheduled"] = True
+        locked_run.update(extras=run_extras)
+
+    try:
+        schedule_judging_for_org.apply_async(
+            args=[shard.run.organization_id], headers={"sentry-propagate-traces": False}
+        )
+    except Exception:
+        logger.exception("night_shift.autofix_issue_data.judge_dispatch_failed", extra=log_extra)
+
+
 def deliver_night_shift_result(
     organization_id: int,
     run_uuid: UUID,
@@ -183,6 +223,7 @@ def deliver_night_shift_result(
             attributes={"error_type": "delivery_error" if status == "error" else "no_artifact"},
         )
         logger.warning("night_shift.delivery.no_result", extra={**log_extra, "status": status})
+        _schedule_judging_after_delivery(shard, log_extra)
         return
 
     try:
@@ -192,6 +233,7 @@ def deliver_night_shift_result(
             "night_shift.triage_error", 1, attributes={"error_type": "invalid_artifact"}
         )
         logger.exception("night_shift.delivery.invalid_result", extra=log_extra)
+        _schedule_judging_after_delivery(shard, log_extra)
         return
 
     options = (run.extras or {}).get("options") or {}
@@ -212,6 +254,7 @@ def deliver_night_shift_result(
         prompt_version=prompt_version,
         log_extra=log_extra,
     )
+    _schedule_judging_after_delivery(shard, log_extra)
 
 
 def _process_verdicts(

src/sentry/tasks/seer/autofix_issue_data.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from django.db.models import F, Window
+from django.db.models.functions import PercentRank
+from django.utils import timezone
+from pydantic import BaseModel, Field
+from taskbroker_client.retry import Retry
+
+from sentry import features
+from sentry.models.organization import Organization
+from sentry.ratelimits import backend as ratelimiter
+from sentry.seer.models import SeerApiError
+from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData
+from sentry.seer.signed_seer_api import (
+    LlmGenerateRequest,
+    SeerViewerContext,
+    make_llm_generate_request,
+)
+from sentry.tasks.base import instrumented_task
+from sentry.taskworker.namespaces import seer_tasks
+from sentry.utils import json, metrics
+
+FEATURE_FLAG = "organizations:seer-fixability-training-data"
+MAX_REVIEWS_PER_ORG_PER_DAY = 20
+
+SYSTEM_PROMPT = """Night Shift reviews software issues and may trigger Autofix to investigate
+and open a pull request. Your job is to identify issues where opening a pull request would be
+wasteful because the issue cannot be fixed in the relevant codebase.
+
+An issue is fixable when it can reasonably be resolved with one or two pull requests to the
+relevant codebase. It is not_fixable when it cannot. Choose uncertain only when the supplied
+evidence is insufficient to decide. Use only the supplied issue and event evidence. Do not
+assume an attempted fix, pull request, or prior automated decision.
+
+Return only a JSON object matching this shape, with a concise one-to-two-sentence reason:
+{"verdict":"fixable|not_fixable|uncertain","confidence":"high|medium|low","reason":"..."}
+"""
+
+
+class JudgeResponse(BaseModel):
+    verdict: Literal["fixable", "not_fixable", "uncertain"]
+    confidence: Literal["high", "medium", "low"]
+    reason: str = Field(min_length=1, max_length=2048)
+
+
+def _select_candidates(organization_id: int) -> list[SeerAutofixIssueData]:
+    # Randomly sample 20 of the issues from bottom 40% of the fixability score
+    rows = (
+        SeerAutofixIssueData.objects.filter(
+            organization_id=organization_id,
+            judge_review__isnull=True,
+            group__seer_fixability_score__isnull=False,
+        )
+        .exclude(raw_issue_data__status="pr_merged")
+        .select_related("group")
+        .annotate(
+            score_percentile=Window(
+                expression=PercentRank(),
+                order_by=F("group__seer_fixability_score").asc(),
+            )
+        )
+    )
+    return list(rows.filter(score_percentile__lte=0.4).order_by("?")[:MAX_REVIEWS_PER_ORG_PER_DAY])
+
+
+@instrumented_task(
+    name="sentry.tasks.seer.autofix_issue_data.schedule_judging_for_org",
+    namespace=seer_tasks,
+    processing_deadline_duration=5 * 60,
+)
+def schedule_judging_for_org(organization_id: int) -> None:
+    organization = Organization.objects.filter(id=organization_id).first()
+    if organization is None or not features.has(FEATURE_FLAG, organization):
+        return
+
+    for issue_data in _select_candidates(organization.id):
+        event_id = issue_data.raw_issue_data.get("event_id")
+        if not isinstance(event_id, str):
+            continue
+        if ratelimiter.is_limited(
+            f"autofix_issue_data_judge:org:{organization.id}",
+            limit=MAX_REVIEWS_PER_ORG_PER_DAY,
+            window=24 * 60 * 60,
+        ):
+            break
+        judge_issue_data.apply_async(
+            args=[issue_data.id, event_id],
+            headers={"sentry-propagate-traces": False},
+        )
+
+
+def _parse_response(content: str) -> JudgeResponse:
+    value = content.strip()
+    if value.startswith("```json") and value.endswith("```"):
+        value = value[7:-3].strip()
+    return JudgeResponse.parse_obj(json.loads(value))
+
+
+@instrumented_task(
+    name="sentry.tasks.seer.autofix_issue_data.judge",
+    namespace=seer_tasks,
+    processing_deadline_duration=60,
+    retry=Retry(times=2, delay=30, on=(Exception,)),
+)
+def judge_issue_data(issue_data_id: int, event_id: str) -> None:
+    issue_data = (
+        SeerAutofixIssueData.objects.select_related("organization").filter(id=issue_data_id).first()
+    )
+    if issue_data is None or not features.has(FEATURE_FLAG, issue_data.organization):
+        return
+    if issue_data.judge_review is not None:
+        return
+    if issue_data.raw_issue_data.get("event_id") != event_id:
+        metrics.incr("autofix_issue_data.judge.skipped", tags={"reason": "stale_event"})
+        return
+
+    body = LlmGenerateRequest(
+        provider="anthropic",
+        model="opus",
+        referrer="sentry.autofix_issue_data.judge",
+        prompt=json.dumps(
+            {
+                key: value
+                for key, value in issue_data.raw_issue_data.items()
+                if key not in {"status", "reason"}
+            }
+        ),
+        system_prompt=SYSTEM_PROMPT,
+        temperature=0.0,
+        max_tokens=1000,
+        timeout=25,
+        reasoning="high",
+        conversation_id=None,
+    )
+    response = make_llm_generate_request(
+        body,
+        timeout=30,
+        viewer_context=SeerViewerContext(organization_id=issue_data.organization_id),
+    )
+    if response.status >= 400:
+        raise SeerApiError("Seer autofix issue data judge request failed", response.status)
+
+    response_data = response.json()
+    content = response_data.get("content")
+    model = response_data.get("model")
+    if not isinstance(content, str) or not isinstance(model, str):
+        raise ValueError("Seer autofix issue data judge returned an invalid response")
+    result = _parse_response(content)
+    reviewed_at = timezone.now()
+
+    updated = SeerAutofixIssueData.objects.filter(
+        id=issue_data.id,
+        judge_review__isnull=True,
+        raw_issue_data__event_id=event_id,
+    ).update(
+        judge_review={
+            "reviewer": "llm_judge",
+            "verdict": result.verdict,
+            "confidence": result.confidence,
+            "reason": result.reason,
+            "model": model,
+            "prompt_version": "1",
+            "reviewed_at": reviewed_at.isoformat(),
+            "reviewed_event_id": event_id,
+        },
+        date_updated=reviewed_at,
+    )
+    metrics.incr(
+        "autofix_issue_data.judge.completed" if updated else "autofix_issue_data.judge.skipped",
+        tags={} if updated else {"reason": "stale_event"},
+    )

src/sentry/testutils/factories.py

@@ -179,6 +179,7 @@
 from sentry.replays.models import DeletionJobStatus, ReplayDeletionJobModel
 from sentry.seer.autofix.constants import CodingAgentStatus
 from sentry.seer.models.agent_write_grant import SeerAgentWriteGrant
+from sentry.seer.models.autofix_issue_data import SeerAutofixIssueData
 from sentry.seer.models.project_repository import SeerProjectRepository
 from sentry.seer.models.run import (
     SeerAgentRun,
@@ -3242,6 +3243,18 @@ def create_github_provider(**kwargs) -> IdentityProvider:
         )
         return identity_provider
 
+    @staticmethod
+    @assume_test_silo_mode(SiloMode.CELL)
+    def create_seer_autofix_issue_data(group: Group, **kwargs) -> SeerAutofixIssueData:
+        kwargs.setdefault("organization_id", group.project.organization_id)
+        kwargs.setdefault("project_id", group.project_id)
+        kwargs.setdefault("source", "night_shift")
+        kwargs.setdefault(
+            "raw_issue_data",
+            {"event_id": "a" * 32, "event": {}, "issue": {}, "status": "skip"},
+        )
+        return SeerAutofixIssueData.objects.create(group=group, **kwargs)
+
     @staticmethod
     @assume_test_silo_mode(SiloMode.CELL)
     def create_seer_agent_write_grant(organization, user, session_id: str = "s1", **kwargs):

src/sentry/testutils/fixtures.py

@@ -1330,6 +1330,9 @@ def create_preprod_artifact_pair_for_comparison(
 
         return head_artifact, head_size_metrics, base_artifact, base_size_metrics
 
+    def create_seer_autofix_issue_data(self, group, **kwargs):
+        return Factories.create_seer_autofix_issue_data(group=group, **kwargs)
+
     def create_seer_run(self, organization=None, **kwargs):
         if organization is None:
             organization = self.organization

tests/sentry/seer/night_shift/test_delivery.py

@@ -107,6 +107,42 @@ def test_error_status_records_error_and_returns(self) -> None:
         assert shard.extras["error_type"] == SeerNightShiftRunErrorType.SHARD_DELIVERY_FAILED.value
         assert not SeerNightShiftRunResult.objects.filter(run=run).exists()
 
+    def test_schedules_judging_after_final_shard_delivery(self) -> None:
+        org = self.create_organization()
+        run = self._create_night_shift_run(organization=org)
+        first_shard = run.executions.get()
+        assert first_shard.seer_run is not None
+        second_seer_run = self.create_seer_run(organization=org)
+        SeerWorkflowRunExecution.objects.create(run=run, seer_run=second_seer_run)
+
+        with (
+            self.feature("organizations:seer-fixability-training-data"),
+            patch(
+                "sentry.seer.night_shift.delivery.schedule_judging_for_org.apply_async"
+            ) as mock_schedule,
+        ):
+            deliver_night_shift_result(
+                organization_id=org.id,
+                run_uuid=first_shard.seer_run.uuid,
+                status="error",
+                result=None,
+                error="Seer exploded",
+            )
+            mock_schedule.assert_not_called()
+
+            for _ in range(2):
+                deliver_night_shift_result(
+                    organization_id=org.id,
+                    run_uuid=second_seer_run.uuid,
+                    status="completed",
+                    result={"verdicts": []},
+                    error=None,
+                )
+
+            mock_schedule.assert_called_once_with(
+                args=[org.id], headers={"sentry-propagate-traces": False}
+            )
+
     def test_sibling_shard_success_keeps_other_shard_error(self) -> None:
         """A successful shard delivery must not clear an error a sibling shard
         recorded on the same run."""