From d10118810f4d3a91963c5395520f5334a33f95dd Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:08:54 +0530 Subject: [PATCH 1/3] FIX: use the canonical score-to-outcome mapping when a human score is removed remove_human_score_async compared score_value against the literal string true, so a float_scale automated score like 0.9 was stored as FAILURE and an undetermined score could never come back as UNDETERMINED. Route the recompute through attack_outcome_from_score, the same contract every attack strategy uses. Fixes #3181 Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- pyrit/backend/services/attack_service.py | 16 +++++-- tests/unit/backend/test_attack_service.py | 54 +++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/pyrit/backend/services/attack_service.py b/pyrit/backend/services/attack_service.py index 74eeb3d17a..ece5886d73 100644 --- a/pyrit/backend/services/attack_service.py +++ b/pyrit/backend/services/attack_service.py @@ -87,7 +87,9 @@ Message, MessagePiece, TargetIdentifier, + UndeterminedScoreError, ) +from pyrit.executor.attack import attack_outcome_from_score from pyrit.models.messages.tool_content import validate_tool_conversation from pyrit.prompt_target import PromptTarget @@ -867,13 +869,19 @@ async def remove_human_score_async(self, *, attack_result_id: str) -> AttackSumm return None automated_score = results[0].automated_score - if automated_score is None or automated_score.score_value is None: + # Follow the attack-side contract of attack_outcome_from_score (#3181): read the + # typed score value instead of comparing the raw string, and treat an undetermined + # score as neither achievement nor refutation. + if automated_score is None: outcome = AttackOutcome.UNDETERMINED + else: + try: + outcome = attack_outcome_from_score(automated_score) + except UndeterminedScoreError: + outcome = AttackOutcome.UNDETERMINED + if outcome is AttackOutcome.UNDETERMINED or automated_score is None: outcome_reason = None else: - outcome = ( - AttackOutcome.SUCCESS if automated_score.score_value.casefold() == "true" else AttackOutcome.FAILURE - ) outcome_reason = automated_score.score_rationale await self._memory.update_attack_result_by_id_async( diff --git a/tests/unit/backend/test_attack_service.py b/tests/unit/backend/test_attack_service.py index a0d82744fe..e1ab5294f6 100644 --- a/tests/unit/backend/test_attack_service.py +++ b/tests/unit/backend/test_attack_service.py @@ -3389,3 +3389,57 @@ def test_create_attack_request_normalizes_legacy_attribution_labels() -> None: def test_create_attack_request_rejects_overlength_values() -> None: with pytest.raises(ValueError, match="at most 128"): CreateAttackRequest(target_registry_name="target", operator="x" * 129) + + +class TestRemoveHumanScoreOutcomeByScoreType: + """Removing a human score must follow the canonical score-to-outcome contract (#3181).""" + + @pytest.mark.parametrize( + ("score_type", "score_value", "expected_outcome"), + [ + ("float_scale", "0.9", "success"), + ("float_scale", "0.0", "failure"), + ("true_false", "True", "success"), + ("true_false", "False", "failure"), + ], + ) + async def test_remove_human_score_maps_typed_score_values( + self, + attack_service, + mock_memory, + score_type: str, + score_value: str, + expected_outcome: str, + ) -> None: + attack = make_attack_result(conversation_id="test-id") + attack.automated_score = Score( + score_type=score_type, + score_value=score_value, + score_rationale="Automated rationale", + ) + mock_memory.get_attack_results_async.return_value = [attack] + + await attack_service.remove_human_score_async(attack_result_id="ar-test-id") + + update_fields = mock_memory.update_attack_result_by_id_async.call_args.kwargs["update_fields"] + assert update_fields["outcome"] == expected_outcome + + async def test_remove_human_score_with_undetermined_automated_score_is_undetermined( + self, + attack_service, + mock_memory, + ) -> None: + attack = make_attack_result(conversation_id="test-id") + attack.automated_score = Score( + score_type="true_false", + score_value=None, + score_rationale="No verdict", + status="undetermined", + ) + mock_memory.get_attack_results_async.return_value = [attack] + + await attack_service.remove_human_score_async(attack_result_id="ar-test-id") + + update_fields = mock_memory.update_attack_result_by_id_async.call_args.kwargs["update_fields"] + assert update_fields["outcome"] == "undetermined" + assert update_fields["outcome_reason"] is None From 99cb512875636851b75ffc24149469d42018c363 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 13 Oct 2026 23:19:26 -0700 Subject: [PATCH 2/3] FIX: sort attack service imports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 793a90d7-1cb3-4545-a760-4b5209c58e87 --- pyrit/backend/services/attack_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrit/backend/services/attack_service.py b/pyrit/backend/services/attack_service.py index ece5886d73..6ad069fc40 100644 --- a/pyrit/backend/services/attack_service.py +++ b/pyrit/backend/services/attack_service.py @@ -68,6 +68,7 @@ ) from pyrit.backend.services.target_service import get_target_service from pyrit.common.utils import to_sha256 +from pyrit.executor.attack import attack_outcome_from_score from pyrit.memory import ( AttackResultKeysetCursor, CentralMemory, @@ -89,7 +90,6 @@ TargetIdentifier, UndeterminedScoreError, ) -from pyrit.executor.attack import attack_outcome_from_score from pyrit.models.messages.tool_content import validate_tool_conversation from pyrit.prompt_target import PromptTarget From 71ef17baeb6f967768e34616cba7e24cab3c7353 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 13 Oct 2026 23:29:07 -0700 Subject: [PATCH 3/3] FIX: delegate undetermined scores directly to canonical helper The canonical helper already handles UndeterminedScoreError. Remove the unreachable duplicate handler so the PR meets the required changed-line coverage threshold without mocking an impossible error path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 793a90d7-1cb3-4545-a760-4b5209c58e87 --- pyrit/backend/services/attack_service.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pyrit/backend/services/attack_service.py b/pyrit/backend/services/attack_service.py index 6ad069fc40..62dad16c9e 100644 --- a/pyrit/backend/services/attack_service.py +++ b/pyrit/backend/services/attack_service.py @@ -88,7 +88,6 @@ Message, MessagePiece, TargetIdentifier, - UndeterminedScoreError, ) from pyrit.models.messages.tool_content import validate_tool_conversation from pyrit.prompt_target import PromptTarget @@ -872,13 +871,7 @@ async def remove_human_score_async(self, *, attack_result_id: str) -> AttackSumm # Follow the attack-side contract of attack_outcome_from_score (#3181): read the # typed score value instead of comparing the raw string, and treat an undetermined # score as neither achievement nor refutation. - if automated_score is None: - outcome = AttackOutcome.UNDETERMINED - else: - try: - outcome = attack_outcome_from_score(automated_score) - except UndeterminedScoreError: - outcome = AttackOutcome.UNDETERMINED + outcome = AttackOutcome.UNDETERMINED if automated_score is None else attack_outcome_from_score(automated_score) if outcome is AttackOutcome.UNDETERMINED or automated_score is None: outcome_reason = None else: