diff --git a/doc/code/analytics/0_attack_results.md b/doc/code/analytics/0_attack_results.md index 5a651e1e46..17964b0d17 100644 --- a/doc/code/analytics/0_attack_results.md +++ b/doc/code/analytics/0_attack_results.md @@ -253,11 +253,15 @@ is ASCII and native `lower` is equivalent. Unicode, embedded NULs, and non-text values retain the existing Python conversion and lowercasing. Objective-target keys are persisted frozen-v1 behavioral evaluation hashes. -Deployment changes need not split behaviorally equivalent targets. The +Deployment changes need not split behaviorally equivalent targets. Targets +without a model name, such as HTTP or Gandalf targets, are keyed by their +content hash instead, so different endpoints or levels stay separate. The `target_identifier_hash` in a result row is instead its exact content hash for inspection. Neither key is recomputed from current registry state. Scenario keys identify saved runs, not potentially repeated scenario names. The SDK adds short identity suffixes to target/scenario display labels and forwards reader warnings. +Whitespace-only model labels use the target class name and short identity suffix; +SQLite and SQL Server recognize the same Unicode whitespace as Python `str.strip`. ## Freshness and projection limits diff --git a/doc/contributing/11_memory_models.md b/doc/contributing/11_memory_models.md index cb43da35e2..77b7921495 100644 --- a/doc/contributing/11_memory_models.md +++ b/doc/contributing/11_memory_models.md @@ -138,6 +138,19 @@ their names. The frozen v1 evaluation rules match in place when evaluation rules evolve: add a new versioned column, backfill, and explicit selection policy instead of silently changing historical groups. +v1 had one pre-release correction before it shipped in a release. Targets +without a model name (HTTP, Gandalf and other non-LLM targets) carry no +behavioral settings, so the original rules merged every such target of a +class into one group. v1 now keys a target with no nonempty +`underlying_model_name` or `model_name` by its content hash, after unwrapping +a wrapper to its first inner target. Targets with a model name keep their +original key. Revision `3f6a9d2c8b1e` recomputes stored keys for databases +that already applied `901e6c7bf9d4`, so existing databases, fresh installs, and +new writes agree. Its downgrade keeps the corrected keys. Objective-target +labels skip blank model names and fall back to the class name, which the SDK +suffixes with a short key. With this correction the v1 definition is frozen +again. + Filters OR values within each predicate (converter `ALL` is the exception) and AND separate predicates, including repeated dimensions. Missing metadata, recorded empty converter pipelines, and real empty strings retain different typed keys. diff --git a/pyrit/memory/alembic/versions/3f6a9d2c8b1e_recompute_model_less_target_identity.py b/pyrit/memory/alembic/versions/3f6a9d2c8b1e_recompute_model_less_target_identity.py new file mode 100644 index 0000000000..3f874319e2 --- /dev/null +++ b/pyrit/memory/alembic/versions/3f6a9d2c8b1e_recompute_model_less_target_identity.py @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Recompute frozen v1 objective-target keys after the model-less target correction. + +Revision ``901e6c7bf9d4`` grouped every target without a model name (HTTP, Gandalf and +other non-LLM targets) of one class into a single key. The v1 rules now fall back to the +target content hash for those targets. Databases that already applied ``901e6c7bf9d4`` +keep the old keys, because Alembic does not rerun an applied revision, so this revision +recomputes them with the same bounded backfill. Keys for targets with a model name come +out unchanged. + +Revision ID: 3f6a9d2c8b1e +Revises: 901e6c7bf9d4 +Create Date: 2026-10-13 09:00:00.000000 +""" + +import importlib +from collections.abc import Sequence + +from alembic import op + +revision: str = "3f6a9d2c8b1e" +down_revision: str | None = "901e6c7bf9d4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_ANALYTICS_REVISION = importlib.import_module("pyrit.memory.alembic.versions.901e6c7bf9d4_index_attack_analytics") + + +def upgrade() -> None: + """Recompute every stored v1 objective-target key with the corrected rules.""" + _ANALYTICS_REVISION._backfill(connection=op.get_bind()) + + +def downgrade() -> None: + """ + Keep the corrected keys. + + The pre-correction keys merged distinct targets and can't be told apart again, + so downgrading leaves the stored values as they are. + """ diff --git a/pyrit/memory/analytics_identity_v1.py b/pyrit/memory/analytics_identity_v1.py index c06f704ae1..0fb49e3e12 100644 --- a/pyrit/memory/analytics_identity_v1.py +++ b/pyrit/memory/analytics_identity_v1.py @@ -14,6 +14,11 @@ class ObjectiveTargetAnalyticsIdentityV1: """ The objective-target evaluation identity used for both writes and historical backfills. + Targets with a model name group by the behavioral evaluation rules (model, temperature, + top_p). Targets without one (HTTP, Gandalf and other non-LLM targets) group by their + content hash instead, so distinct endpoints or levels stay apart. That fallback is a + pre-release correction to v1, applied by revision ``3f6a9d2c8b1e``. + Keep these rules unchanged when ``Evaluate`` markers evolve. A future rule change needs a new versioned column and migration so stored chart identities never acquire a new meaning merely because code was upgraded. @@ -43,6 +48,11 @@ def hash(cls, *, identifier: ComponentIdentifier) -> str: str: The 64-character v1 evaluation hash. """ target = TargetIdentifier.from_component_identifier(identifier) + evaluated = cls._evaluated_target(target=target) + if not cls._has_model_name(target=evaluated): + # The behavioral rules only see model settings, so they would merge every + # model-less target of a class into one group. + return evaluated.hash return compute_eval_hash( target, child_eval_rules=cls._CHILD_RULES, @@ -50,6 +60,33 @@ def hash(cls, *, identifier: ComponentIdentifier) -> str: root_unwrap_child="targets", ) + @staticmethod + def _evaluated_target(*, target: ComponentIdentifier) -> ComponentIdentifier: + """ + Return the target the rules evaluate: a wrapper's first inner target, or the target itself. + + Args: + target (ComponentIdentifier): The recorded objective target. + + Returns: + ComponentIdentifier: The unwrapped target. + """ + inner = target.get_child_list("targets") + return inner[0] if inner else target + + @classmethod + def _has_model_name(cls, *, target: ComponentIdentifier) -> bool: + """ + Check for a nonempty underlying model name or its ``model_name`` fallback. + + Args: + target (ComponentIdentifier): The unwrapped target. + + Returns: + bool: True if the target records a model name the rules can group by. + """ + return any(str(target.params.get(name) or "").strip() for name in ("underlying_model_name", "model_name")) + @classmethod def from_atomic_document(cls, *, document: dict[str, object] | None) -> str | None: """ diff --git a/pyrit/memory/analytics_sql.py b/pyrit/memory/analytics_sql.py index d4d10c62c8..6eacfee282 100644 --- a/pyrit/memory/analytics_sql.py +++ b/pyrit/memory/analytics_sql.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar -from sqlalchemy import Boolean, String, UnicodeText +from sqlalchemy import Boolean, String, Unicode, UnicodeText, literal from sqlalchemy.exc import CompileError from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.elements import BindParameter @@ -23,6 +23,7 @@ from sqlalchemy.sql.selectable import Join, Lateral if TYPE_CHECKING: + from sqlalchemy.sql import ColumnElement from sqlalchemy.sql.compiler import SQLCompiler @@ -77,6 +78,51 @@ def _mssql_unicode_lower(element: UnicodeLower, compiler: Any, **kwargs: Any) -> return f"LOWER({value})" +class UnicodeStrip(FunctionElement[str]): + """Strip Python's Unicode whitespace set using native database expressions.""" + + _WHITESPACE: ClassVar[str] = ( + "\t\n\v\f\r\x1c\x1d\x1e\x1f \x85\xa0\u1680" + "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a" + "\u2028\u2029\u202f\u205f\u3000" + ) + type = UnicodeText() + inherit_cache = True + + def __init__(self, value: ColumnElement[Any]) -> None: + """ + Bind the whitespace set alongside the value to preserve statement caching. + + Args: + value (ColumnElement[Any]): Nullable text to strip. + """ + super().__init__(value, literal(self._WHITESPACE, Unicode(len(self._WHITESPACE)))) + + +@compiles(UnicodeStrip, "sqlite") +def _sqlite_unicode_strip(element: UnicodeStrip, compiler: Any, **kwargs: Any) -> str: + """ + Strip the explicit character set instead of SQLite's space-only default. + + Returns: + str: A native trim expression preserving NULL. + """ + value, whitespace = _arguments(element, compiler, **kwargs) + return f"trim({value}, {whitespace})" + + +@compiles(UnicodeStrip, "mssql") +def _mssql_unicode_strip(element: UnicodeStrip, compiler: Any, **kwargs: Any) -> str: + """ + Strip exact whitespace code points independently of database collation. + + Returns: + str: A native TRIM expression with a bounded Unicode character argument. + """ + value, whitespace = _arguments(element, compiler, **kwargs) + return f"TRIM({whitespace} FROM {value} COLLATE Latin1_General_100_BIN2)" + + class JsonClassNamePresent(FunctionElement[bool]): """ Whether an identifier object explicitly has ``class_name``, even when null. diff --git a/pyrit/memory/attack_analytics_query.py b/pyrit/memory/attack_analytics_query.py index d12076169b..bbad779b00 100644 --- a/pyrit/memory/attack_analytics_query.py +++ b/pyrit/memory/attack_analytics_query.py @@ -53,6 +53,7 @@ JsonObjectAggregate, JsonScalar, UnicodeLower, + UnicodeStrip, ) from pyrit.memory.memory_models import ( AtomicAttackIdentifierEntry, @@ -759,9 +760,11 @@ def _create_source(self, *, dimension: AttackAnalyticsDimension, embedded: bool return _Source(value=value, label=value, insensitive=True) if name is AttackAnalyticsDimensionName.OBJECTIVE_TARGET: value = self.root.c.objective_target_eval_hash_v1 + # Model-less targets record a blank model name; fall through to the class name so + # the SDK's short-hash suffix makes the label readable without endpoint details. label = func.coalesce( - self.target.c.model_name, - self._target_property(path="params.model_name", embedded=embedded), + func.nullif(UnicodeStrip(self.target.c.model_name), ""), + func.nullif(UnicodeStrip(self._target_property(path="params.model_name", embedded=embedded)), ""), self.target.c.class_name, self._target_property(path="class_name", embedded=embedded), value, diff --git a/tests/unit/analytics/test_attack_result_analytics.py b/tests/unit/analytics/test_attack_result_analytics.py index 7504f81ad8..8ed1564ecb 100644 --- a/tests/unit/analytics/test_attack_result_analytics.py +++ b/tests/unit/analytics/test_attack_result_analytics.py @@ -10,6 +10,7 @@ import pytest from pydantic import ValidationError +from sqlalchemy import update from pyrit.analytics import AttackResultAnalytics, compute_scenario_statistics from pyrit.common.task_utils import gather_with_cleanup_async @@ -17,6 +18,7 @@ from pyrit.memory import CentralMemory, MemoryInterface, SQLiteMemory from pyrit.memory.analytics_identity_v1 import ObjectiveTargetAnalyticsIdentityV1 from pyrit.memory.attack_analytics import RawAnalyticsOption +from pyrit.memory.memory_models import TargetIdentifierEntry from pyrit.models import ( AtomicAttackIdentifier, AttackAnalyticsDimension, @@ -32,6 +34,7 @@ AttackResultRole, TargetIdentifier, ) +from pyrit.prompt_target import HTTPTarget from unit.memory.test_attack_analytics import make_result, predicate from unit.mocks import make_scenario_result @@ -518,6 +521,47 @@ async def test_mutated_models_are_revalidated_before_execution( run.assert_not_awaited() +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize("model_name", ["", " ", "\t", "\r\n", "\xa0", "\u2003", " \t\xa0"]) +@pytest.mark.parametrize("embedded", [False, True], ids=["normalized", "embedded"]) +async def test_blank_target_labels_keep_distinct_sdk_identities_async( + analytics: AttackResultAnalytics, sqlite_instance: SQLiteMemory, model_name: str, embedded: bool +) -> None: + targets = [ + HTTPTarget( + http_request=f"POST /chat HTTP/1.1\nHost: {host}.example\n\n{{PROMPT}}", + model_name=model_name, + ).get_identifier() + for host in ("bank", "hr") + ] + results = [make_result(index=index) for index in (1, 2)] + for result, target in zip(results, targets, strict=True): + result.atomic_attack_identifier = AtomicAttackIdentifier.build( + attack_identifier=AttackIdentifier(class_name="ProbeAttack", class_module="tests", objective_target=target) + ) + await sqlite_instance.add_attack_results_to_memory_async(attack_results=results) + if embedded: + async with await sqlite_instance.get_session_async() as session: + await session.execute(update(TargetIdentifierEntry).values(model_name=None, identifier_json=None)) + await session.commit() + axis = AttackAnalyticsDimension(name="objective_target") + expected = {target.hash: f"HTTPTarget ({target.hash[-8:]})" for target in targets} + report = await analytics.query_async(query=AttackAnalyticsQuery(group_by=axis)) + facets = await analytics.facets_async(query=AttackAnalyticsFacetQuery(dimension=axis, search="HTTPTarget")) + matrix = await analytics.query_async( + query=AttackAnalyticsQuery(group_by=axis, compare_by=AttackAnalyticsDimension(name="operation")) + ) + assert {group.key.value: group.label for group in report.groups} == expected + assert {item.key.value: item.label for item in facets.items} == expected + assert {row.key.value: row.label for row in matrix.rows} == expected + for group in report.groups: + page = await analytics.results_async( + query=AttackAnalyticsResultsQuery(filters=AttackAnalyticsFilters(dimensions=group.drilldown_filters)) + ) + assert len(page.items) == group.statistics.total_results == 1 + assert page.items[0].target_identifier_hash == group.key.value + + async def test_frozen_target_groups_preserve_content_hashes_and_every_result_id( analytics: AttackResultAnalytics, sqlite_instance: SQLiteMemory ) -> None: diff --git a/tests/unit/memory/test_analytics_migration.py b/tests/unit/memory/test_analytics_migration.py index 5f48352ab5..314e7b8e80 100644 --- a/tests/unit/memory/test_analytics_migration.py +++ b/tests/unit/memory/test_analytics_migration.py @@ -23,7 +23,7 @@ from pyrit.memory.alembic.versions.e5f7a9c1b3d2_add_identifiers_tables import IdentifierGraphInserter from pyrit.memory.analytics_identity_v1 import ObjectiveTargetAnalyticsIdentityV1 from pyrit.memory.memory_models import AttackResultEntry -from pyrit.models import AtomicAttackIdentifier, AttackIdentifier, TargetIdentifier +from pyrit.models import AtomicAttackIdentifier, AttackIdentifier, TargetIdentifier, compute_eval_hash if TYPE_CHECKING: from pytest import LogCaptureFixture @@ -31,11 +31,15 @@ MAIN_REVISION = "34a18645c7e9" ANALYTICS_REVISION = "901e6c7bf9d4" +TARGET_IDENTITY_REVISION = "3f6a9d2c8b1e" def test_analytics_migration_is_single_successor_of_main() -> None: scripts = ScriptDirectory(str(Path(__file__).resolve().parents[3] / "pyrit" / "memory" / "alembic")) - assert scripts.get_heads() == [ANALYTICS_REVISION] + assert scripts.get_heads() == [TARGET_IDENTITY_REVISION] + correction = scripts.get_revision(TARGET_IDENTITY_REVISION) + assert correction is not None + assert correction.down_revision == ANALYTICS_REVISION revision = scripts.get_revision(ANALYTICS_REVISION) assert revision is not None assert revision.down_revision == MAIN_REVISION @@ -80,7 +84,7 @@ def test_analytics_migration_upgrades_from_main_revisions(*, tmp_path: Path, sta head_revision = connection.execute( text("SELECT version_num FROM pyrit_memory_alembic_version") ).scalar_one() - assert head_revision == ANALYTICS_REVISION + assert head_revision == TARGET_IDENTITY_REVISION assert ( connection.execute( text('SELECT resolved_atomic_attack_identifier_hash FROM "AttackResultEntries" WHERE id = :id'), @@ -383,3 +387,67 @@ def test_mssql_evaluation_migration_uses_a_bounded_index_and_one_result_source() assert sql.count("FROM [AttackResultEntries]") == 1 assert "LEFT OUTER JOIN [TargetIdentifiers]" in sql assert "ScoreEntries" not in sql and "PromptMemoryEntries" not in sql + + +def test_target_identity_correction_recomputes_keys_written_by_the_original_revision(tmp_path: Path) -> None: + """A database that already applied the original revision keeps its merged keys until this revision runs.""" + engine = create_engine(f"sqlite:///{tmp_path / 'target-identity.sqlite'}") + bank = TargetIdentifier( + class_name="HTTPTarget", class_module="tests", model_name="", endpoint="https://bank.example" + ) + hr = TargetIdentifier(class_name="HTTPTarget", class_module="tests", model_name="", endpoint="https://hr.example") + model = TargetIdentifier( + class_name="MockTarget", class_module="tests", model_name="deployment", underlying_model_name="model-x" + ) + + def document(target: TargetIdentifier) -> dict[str, Any]: + attack = AttackIdentifier(class_name="ProbeAttack", class_module="tests", objective_target=target) + return AtomicAttackIdentifier.build(attack_identifier=attack).model_dump() + + def original_rule_hash(target: TargetIdentifier) -> str: + return compute_eval_hash( + target, + child_eval_rules=ObjectiveTargetAnalyticsIdentityV1._CHILD_RULES, + own_rule=ObjectiveTargetAnalyticsIdentityV1._OWN_RULE, + root_unwrap_child="targets", + ) + + try: + with engine.begin() as connection: + config = configuration(connection) + command.upgrade(config, ANALYTICS_REVISION) + ids = { + name: insert_result(connection, identifier=document(t)) + for name, t in (("bank", bank), ("hr", hr), ("model", model)) + } + # Simulate keys written before the correction: both HTTP targets share one merged key. + for name, target in (("bank", bank), ("hr", hr), ("model", model)): + connection.execute( + text('UPDATE "AttackResultEntries" SET objective_target_eval_hash_v1 = :key WHERE id = :id'), + {"key": original_rule_hash(target), "id": ids[name]}, + ) + assert original_rule_hash(bank) == original_rule_hash(hr) + + command.upgrade(config, "head") + + stored = dict( + connection.execute(text('SELECT id, objective_target_eval_hash_v1 FROM "AttackResultEntries"')).all() + ) + assert stored == { + ids["bank"]: bank.hash, + ids["hr"]: hr.hash, + ids["model"]: original_rule_hash(model), + } + assert stored[ids["model"]] == ObjectiveTargetAnalyticsIdentityV1.hash(identifier=model) + command.check(config) + command.downgrade(config, ANALYTICS_REVISION) + assert ( + dict( + connection.execute( + text('SELECT id, objective_target_eval_hash_v1 FROM "AttackResultEntries"') + ).all() + ) + == stored + ) + finally: + engine.dispose() diff --git a/tests/unit/memory/test_attack_analytics.py b/tests/unit/memory/test_attack_analytics.py index 8123a36dc3..220f3cf58f 100644 --- a/tests/unit/memory/test_attack_analytics.py +++ b/tests/unit/memory/test_attack_analytics.py @@ -35,6 +35,7 @@ JsonScalar, ResolvedAttackIdentifierHash, UnicodeLower, + UnicodeStrip, ) from pyrit.memory.attack_analytics import AttackAnalyticsReader from pyrit.memory.attack_analytics_query import AttackAnalyticsQueryCompiler @@ -1119,6 +1120,36 @@ def test_json_array_checks_distinguish_shape_from_missing_members( assert tuple(result) == (is_array, is_empty) +@pytest.mark.parametrize( + "value", + [ + None, + "", + *[chr(codepoint) for codepoint in range(0x3001) if chr(codepoint).isspace()], + " \t\r\n\xa0\u2003", + "\t model \xa0 name \r\n", + "\u200bmodel\u200b", + "\x00model\x00", + ], +) +def test_unicode_strip_matches_python_without_changing_internal_text( + sqlite_instance: SQLiteMemory, value: str | None +) -> None: + with sqlite_instance.engine.connect() as connection: + stripped = connection.execute(select(UnicodeStrip(literal(value, UnicodeText())))).scalar_one() + assert stripped == (value.strip() if value is not None else None) + + +def test_mssql_unicode_strip_uses_native_trim_with_bounded_unicode_characters() -> None: + statement = select(UnicodeStrip(column("model_name", UnicodeText()))) + sql = str(statement.compile(dialect=mssql.dialect())) + assert " FROM model_name COLLATE Latin1_General_100_BIN2)" in sql + whitespace = next(iter(statement.selected_columns)).clauses.clauses[1] + assert whitespace.type.length == 29 + assert set(whitespace.value) == {chr(codepoint) for codepoint in range(0x3001) if chr(codepoint).isspace()} + assert statement._generate_cache_key() is not None + + @pytest.mark.parametrize("raw", [1, '["Alpha", 1]', '{"class_name": "Alpha"}', "not json"]) def test_result_converter_projection_rejects_invalid_metadata(raw: Any) -> None: with pytest.raises(AnalyticsDataException): diff --git a/tests/unit/memory/test_attack_analytics_eval_identity.py b/tests/unit/memory/test_attack_analytics_eval_identity.py index 53a90943da..555981a515 100644 --- a/tests/unit/memory/test_attack_analytics_eval_identity.py +++ b/tests/unit/memory/test_attack_analytics_eval_identity.py @@ -42,6 +42,7 @@ ObjectiveTargetEvaluationIdentifier, TargetIdentifier, ) +from pyrit.prompt_target import GandalfLevel, GandalfTarget, HTTPTarget if TYPE_CHECKING: from pyrit.memory import SQLiteMemory @@ -103,6 +104,88 @@ def test_target_evaluation_v1_is_frozen_to_the_current_marker_rules() -> None: ) +def _model_less(*, class_name: str, endpoint: str, model_name: str = "") -> TargetIdentifier: + return TargetIdentifier(class_name=class_name, class_module="tests", model_name=model_name, endpoint=endpoint) + + +@pytest.mark.usefixtures("patch_central_database") +def test_targets_without_a_model_name_fall_back_to_their_content_hash() -> None: + bank = HTTPTarget(http_request="POST /chat HTTP/1.1\nHost: bank.example\n\n{PROMPT}").get_identifier() + hr = HTTPTarget(http_request="POST /ask HTTP/1.1\nHost: hr.example\n\n{PROMPT}").get_identifier() + level_one = GandalfTarget(level=GandalfLevel.LEVEL_1).get_identifier() + level_seven = GandalfTarget(level=GandalfLevel.LEVEL_7).get_identifier() + blank = _model_less(class_name="MockTarget", endpoint="https://blank.example", model_name=" ") + wrapped = TargetIdentifier(class_name="RoundRobinTarget", class_module="tests", targets=[bank]) + + keys = { + name: ObjectiveTargetAnalyticsIdentityV1.hash(identifier=identifier) + for name, identifier in { + "bank": bank, + "hr": hr, + "level_one": level_one, + "level_seven": level_seven, + "blank": blank, + }.items() + } + + assert len(set(keys.values())) == 5 + assert keys["bank"] == TargetIdentifier.from_component_identifier(bank).hash + assert keys["level_one"] == TargetIdentifier.from_component_identifier(level_one).hash + assert keys["blank"] == blank.hash + assert ObjectiveTargetAnalyticsIdentityV1.hash(identifier=wrapped) == keys["bank"] + # Targets with a model name keep the behavioral key, so deployments still merge. + assert ObjectiveTargetAnalyticsIdentityV1.hash( + identifier=_target(deployment="deploy-a", endpoint="https://one.example") + ) == ObjectiveTargetAnalyticsIdentityV1.hash( + identifier=_target(deployment="deploy-b", endpoint="https://two.example") + ) + + +async def test_model_less_targets_group_filter_facet_and_label_apart_async(sqlite_instance: SQLiteMemory) -> None: + bank = _model_less(class_name="HTTPTarget", endpoint="https://bank.example") + hr = _model_less(class_name="HTTPTarget", endpoint="https://hr.example") + model = _target(deployment="deploy-a", endpoint="https://one.example") + results = [ + _result(index=1, target=bank, outcome=AttackOutcome.SUCCESS, operation="night"), + _result(index=2, target=bank, outcome=AttackOutcome.SUCCESS, operation="day"), + _result(index=3, target=hr, outcome=AttackOutcome.FAILURE, operation="night"), + _result(index=4, target=model, outcome=AttackOutcome.FAILURE, operation="day"), + ] + await sqlite_instance.add_attack_results_to_memory_async(attack_results=results) + reader = AttackAnalyticsReader(memory=sqlite_instance) + axis = AttackAnalyticsDimension(name="objective_target") + bank_key, hr_key, model_key = (ObjectiveTargetAnalyticsIdentityV1.hash(identifier=t) for t in (bank, hr, model)) + + report = await reader.report_async(query=AttackAnalyticsQuery(group_by=axis), control=_control()) + groups = {group.option.key.value: group for group in report.groups} + bank_filter = AttackAnalyticsFilters( + dimensions=[AttackAnalyticsFilter(dimension=axis, values=[groups[bank_key].option.key])] + ) + drilled = await reader.report_async( + query=AttackAnalyticsQuery(filters=bank_filter, group_by=AttackAnalyticsDimension(name="operation")), + control=_control(), + ) + facets = await reader.facets_async(query=AttackAnalyticsFacetQuery(dimension=axis), control=_control()) + + assert {key: group.counts for key, group in groups.items()} == { + bank_key: {"success": 2}, + hr_key: {"failure": 1}, + model_key: {"failure": 1}, + } + assert groups[bank_key].option.label == groups[hr_key].option.label == "HTTPTarget" + assert groups[model_key].option.label == "deploy-a" + assert all("example" not in (group.option.label or "") for group in report.groups) + assert {group.option.key.value: group.counts for group in drilled.groups} == { + "night": {"success": 1}, + "day": {"success": 1}, + } + assert {item.key.value: item.label for item in facets.items} == { + bank_key: "HTTPTarget", + hr_key: "HTTPTarget", + model_key: "deploy-a", + } + + async def test_evaluation_groups_filters_facets_and_pages_share_result_identity(sqlite_instance: SQLiteMemory) -> None: first = _target(deployment="deploy-a", endpoint="https://one.example") second = _target(deployment="deploy-b", endpoint="https://two.example")