diff --git a/pyrit/memory/attack_analytics_query.py b/pyrit/memory/attack_analytics_query.py index 3c2678cfae..d58ec80db5 100644 --- a/pyrit/memory/attack_analytics_query.py +++ b/pyrit/memory/attack_analytics_query.py @@ -869,7 +869,9 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb A retained converter list takes precedence: migration backfills can have fewer edges than recorded members when a child has no usable hash. Edges remain a fallback for identifiers without a retained converter list. - A recorded [] means no converters, unlike absent identifier metadata. + A recorded [] means no converters, unlike absent identifier metadata. A recorded + attack identifier with no list or edges also means no converters, because + identifiers omit an empty converter list. Args: direction (AttackAnalyticsConverterDirection): Request or response pipeline; @@ -930,6 +932,11 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb self._from = self._from.outerjoin(arrays, arrays.c.attack_identifier_hash == self.attack.c.hash) path = f"children.{direction.value}_converters" retained = JsonContainer(self.attack.c.identifier_json, f"$.{path}") + # Identifiers leave out an empty converter list, so a recorded attack with no + # converter list or edges ran without converters, which is not missing metadata. + recorded_without_converters = case( + (self.attack.c.hash.isnot(None), literal("[]", UnicodeText())), else_=null() + ) self._converter_arrays[direction] = ( func.coalesce( retained, @@ -939,8 +946,9 @@ def _converter_source(self, *, direction: AttackAnalyticsConverterDirection, emb ), JsonContainer(self.root.c.atomic_attack_identifier, f"$.children.attack.{path}"), arrays.c.names, + recorded_without_converters, ), - func.coalesce(retained, arrays.c.names), + func.coalesce(retained, arrays.c.names, recorded_without_converters), ) value = self._converter_arrays[direction][0 if embedded else 1] return _Source(value=value, label=value, array=True, converters=True, insensitive=True) diff --git a/tests/unit/memory/test_attack_analytics.py b/tests/unit/memory/test_attack_analytics.py index b661528060..43918d0aaf 100644 --- a/tests/unit/memory/test_attack_analytics.py +++ b/tests/unit/memory/test_attack_analytics.py @@ -25,7 +25,9 @@ from sqlalchemy.sql.functions import Function from sqlalchemy.sql.selectable import CTE +from pyrit.converter import Base64Converter from pyrit.exceptions.analytics_exception import AnalyticsDataException, AnalyticsTimeoutException +from pyrit.executor.attack import AttackConverterConfig, PromptSendingAttack from pyrit.memory import MemoryInterface, SQLiteMemory from pyrit.memory.analytics_sql import ( JsonArrayEmpty, @@ -53,6 +55,8 @@ ConverterIdentifier, TargetIdentifier, ) +from pyrit.prompt_normalizer import ConverterConfiguration +from unit.mocks import MockPromptTarget if TYPE_CHECKING: from pathlib import Path @@ -400,6 +404,44 @@ async def test_known_empty_converters_differ_from_missing_identifier(sqlite_inst assert {group.option.key.kind.value for group in report.groups} == {"missing", "no_converters"} +@pytest.mark.parametrize("direction", ["request", "response"]) +async def test_real_attacks_without_converters_are_no_converters_async( + *, direction: str, sqlite_instance: SQLiteMemory +) -> None: + """Identifiers omit an empty converter list, so a real attack must still land in no_converters.""" + plain = await PromptSendingAttack(objective_target=MockPromptTarget()).execute_async(objective="plain") + converted = await PromptSendingAttack( + objective_target=MockPromptTarget(), + attack_converter_config=AttackConverterConfig( + request_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]), + response_converters=ConverterConfiguration.from_converters(converters=[Base64Converter()]), + ), + ).execute_async(objective="converted") + unidentified = make_result(index=3) + unidentified.atomic_attack_identifier = None + await sqlite_instance.add_attack_results_to_memory_async(attack_results=[unidentified]) + dimension = AttackAnalyticsDimension(name="converter_type", converter_direction=direction) + reader = AttackAnalyticsReader(memory=sqlite_instance) + + report = await reader.report_async(query=AttackAnalyticsQuery(group_by=dimension), control=control()) + filtered = await reader.report_async( + query=AttackAnalyticsQuery( + filters=AttackAnalyticsFilters.model_validate( + {"dimensions": [{"dimension": dimension.model_dump(), "values": [{"kind": "no_converters"}]}]} + ) + ), + control=control(), + ) + without_converters = await sqlite_instance.get_attack_results_async(has_converters=False) + + assert { + (group.option.key.kind.value, group.option.key.value): sum(group.counts.values()) for group in report.groups + } == {("no_converters", None): 1, ("value", "base64converter"): 1, ("missing", None): 1} + assert sum(filtered.counts.values()) == 1 + assert plain.attack_result_id in {result.attack_result_id for result in without_converters} + assert converted.attack_result_id not in {result.attack_result_id for result in without_converters} + + async def test_literal_label_paths_and_facet_search(sqlite_instance: SQLiteMemory) -> None: await sqlite_instance.add_attack_results_to_memory_async( attack_results=[