FIX: key model-less objective targets by content hash in analytics v1

doc/code/analytics/0_attack_results.md

@@ -253,7 +253,9 @@ is ASCII and native `lower` is equivalent. Unicode, embedded NULs, and non-text
 values retain the existing Python conversion and lowercasing.
 
 Objective-target keys are persisted frozen-v1 behavioral evaluation hashes.
-Deployment changes need not split behaviorally equivalent targets. The
+Deployment changes need not split behaviorally equivalent targets. Targets
+without a model name, such as HTTP or Gandalf targets, are keyed by their
+content hash instead, so different endpoints or levels stay separate. The
 `target_identifier_hash` in a result row is instead its exact content hash for
 inspection. Neither key is recomputed from current registry state. Scenario keys
 identify saved runs, not potentially repeated scenario names. The SDK adds short

doc/contributing/11_memory_models.md

@@ -138,6 +138,19 @@ their names. The frozen v1 evaluation rules match
 in place when evaluation rules evolve: add a new versioned column, backfill,
 and explicit selection policy instead of silently changing historical groups.
 
+v1 had one pre-release correction before it shipped in a release. Targets
+without a model name (HTTP, Gandalf and other non-LLM targets) carry no
+behavioral settings, so the original rules merged every such target of a
+class into one group. v1 now keys a target with no nonempty
+`underlying_model_name` or `model_name` by its content hash, after unwrapping
+a wrapper to its first inner target. Targets with a model name keep their
+original key. Revision `3f6a9d2c8b1e` recomputes stored keys for databases
+that already applied `901e6c7bf9d4`, so existing databases, fresh installs, and
+new writes agree. Its downgrade keeps the corrected keys. Objective-target
+labels skip blank model names and fall back to the class name, which the SDK
+suffixes with a short key. With this correction the v1 definition is frozen
+again.
+
 Filters OR values within each predicate (converter `ALL` is the exception) and AND
 separate predicates, including repeated dimensions. Missing metadata, recorded
 empty converter pipelines, and real empty strings retain different typed keys.

pyrit/memory/alembic/versions/3f6a9d2c8b1e_recompute_model_less_target_identity.py

@@ -0,0 +1,43 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Recompute frozen v1 objective-target keys after the model-less target correction.
+
+Revision ``901e6c7bf9d4`` grouped every target without a model name (HTTP, Gandalf and
+other non-LLM targets) of one class into a single key. The v1 rules now fall back to the
+target content hash for those targets. Databases that already applied ``901e6c7bf9d4``
+keep the old keys, because Alembic does not rerun an applied revision, so this revision
+recomputes them with the same bounded backfill. Keys for targets with a model name come
+out unchanged.
+
+Revision ID: 3f6a9d2c8b1e
+Revises: 901e6c7bf9d4
+Create Date: 2026-10-13 09:00:00.000000
+"""
+
+import importlib
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "3f6a9d2c8b1e"
+down_revision: str | None = "901e6c7bf9d4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_ANALYTICS_REVISION = importlib.import_module("pyrit.memory.alembic.versions.901e6c7bf9d4_index_attack_analytics")
+
+
+def upgrade() -> None:
+    """Recompute every stored v1 objective-target key with the corrected rules."""
+    _ANALYTICS_REVISION._backfill(connection=op.get_bind())
+
+
+def downgrade() -> None:
+    """
+    Keep the corrected keys.
+
+    The pre-correction keys merged distinct targets and can't be told apart again,
+    so downgrading leaves the stored values as they are.
+    """

pyrit/memory/analytics_identity_v1.py

@@ -14,6 +14,11 @@ class ObjectiveTargetAnalyticsIdentityV1:
     """
     The objective-target evaluation identity used for both writes and historical backfills.
 
+    Targets with a model name group by the behavioral evaluation rules (model, temperature,
+    top_p). Targets without one (HTTP, Gandalf and other non-LLM targets) group by their
+    content hash instead, so distinct endpoints or levels stay apart. That fallback is a
+    pre-release correction to v1, applied by revision ``3f6a9d2c8b1e``.
+
     Keep these rules unchanged when ``Evaluate`` markers evolve. A future rule
     change needs a new versioned column and migration so stored chart identities
     never acquire a new meaning merely because code was upgraded.
@@ -43,13 +48,45 @@ def hash(cls, *, identifier: ComponentIdentifier) -> str:
             str: The 64-character v1 evaluation hash.
         """
         target = TargetIdentifier.from_component_identifier(identifier)
+        evaluated = cls._evaluated_target(target=target)
+        if not cls._has_model_name(target=evaluated):
+            # The behavioral rules only see model settings, so they would merge every
+            # model-less target of a class into one group.
+            return evaluated.hash
         return compute_eval_hash(
             target,
             child_eval_rules=cls._CHILD_RULES,
             own_rule=cls._OWN_RULE,
             root_unwrap_child="targets",
         )
 
+    @staticmethod
+    def _evaluated_target(*, target: ComponentIdentifier) -> ComponentIdentifier:
+        """
+        Return the target the rules evaluate: a wrapper's first inner target, or the target itself.
+
+        Args:
+            target (ComponentIdentifier): The recorded objective target.
+
+        Returns:
+            ComponentIdentifier: The unwrapped target.
+        """
+        inner = target.get_child_list("targets")
+        return inner[0] if inner else target
+
+    @classmethod
+    def _has_model_name(cls, *, target: ComponentIdentifier) -> bool:
+        """
+        Check for a nonempty underlying model name or its ``model_name`` fallback.
+
+        Args:
+            target (ComponentIdentifier): The unwrapped target.
+
+        Returns:
+            bool: True if the target records a model name the rules can group by.
+        """
+        return any(str(target.params.get(name) or "").strip() for name in ("underlying_model_name", "model_name"))
+
     @classmethod
     def from_atomic_document(cls, *, document: dict[str, object] | None) -> str | None:
         """

pyrit/memory/attack_analytics_query.py

@@ -758,9 +758,11 @@ def _create_source(self, *, dimension: AttackAnalyticsDimension, embedded: bool
             return _Source(value=value, label=value, insensitive=True)
         if name is AttackAnalyticsDimensionName.OBJECTIVE_TARGET:
             value = self.root.c.objective_target_eval_hash_v1
+            # Model-less targets record a blank model name; fall through to the class name so
+            # the SDK's short-hash suffix makes the label readable without endpoint details.
             label = func.coalesce(
-                self.target.c.model_name,
-                self._target_property(path="params.model_name", embedded=embedded),
+                func.nullif(func.trim(self.target.c.model_name), ""),
+                func.nullif(func.trim(self._target_property(path="params.model_name", embedded=embedded)), ""),
                 self.target.c.class_name,
                 self._target_property(path="class_name", embedded=embedded),
                 value,

tests/unit/memory/test_analytics_migration.py

@@ -23,19 +23,23 @@
 from pyrit.memory.alembic.versions.e5f7a9c1b3d2_add_identifiers_tables import IdentifierGraphInserter
 from pyrit.memory.analytics_identity_v1 import ObjectiveTargetAnalyticsIdentityV1
 from pyrit.memory.memory_models import AttackResultEntry
-from pyrit.models import AtomicAttackIdentifier, AttackIdentifier, TargetIdentifier
+from pyrit.models import AtomicAttackIdentifier, AttackIdentifier, TargetIdentifier, compute_eval_hash
 
 if TYPE_CHECKING:
     from pytest import LogCaptureFixture
     from sqlalchemy.engine import Dialect
 
 MAIN_REVISION = "34a18645c7e9"
 ANALYTICS_REVISION = "901e6c7bf9d4"
+TARGET_IDENTITY_REVISION = "3f6a9d2c8b1e"
 
 
 def test_analytics_migration_is_single_successor_of_main() -> None:
     scripts = ScriptDirectory(str(Path(__file__).resolve().parents[3] / "pyrit" / "memory" / "alembic"))
-    assert scripts.get_heads() == [ANALYTICS_REVISION]
+    assert scripts.get_heads() == [TARGET_IDENTITY_REVISION]
+    correction = scripts.get_revision(TARGET_IDENTITY_REVISION)
+    assert correction is not None
+    assert correction.down_revision == ANALYTICS_REVISION
     revision = scripts.get_revision(ANALYTICS_REVISION)
     assert revision is not None
     assert revision.down_revision == MAIN_REVISION
@@ -80,7 +84,7 @@ def test_analytics_migration_upgrades_from_main_revisions(*, tmp_path: Path, sta
             head_revision = connection.execute(
                 text("SELECT version_num FROM pyrit_memory_alembic_version")
             ).scalar_one()
-            assert head_revision == ANALYTICS_REVISION
+            assert head_revision == TARGET_IDENTITY_REVISION
             assert (
                 connection.execute(
                     text('SELECT resolved_atomic_attack_identifier_hash FROM "AttackResultEntries" WHERE id = :id'),
@@ -383,3 +387,67 @@ def test_mssql_evaluation_migration_uses_a_bounded_index_and_one_result_source()
     assert sql.count("FROM [AttackResultEntries]") == 1
     assert "LEFT OUTER JOIN [TargetIdentifiers]" in sql
     assert "ScoreEntries" not in sql and "PromptMemoryEntries" not in sql
+
+
+def test_target_identity_correction_recomputes_keys_written_by_the_original_revision(tmp_path: Path) -> None:
+    """A database that already applied the original revision keeps its merged keys until this revision runs."""
+    engine = create_engine(f"sqlite:///{tmp_path / 'target-identity.sqlite'}")
+    bank = TargetIdentifier(
+        class_name="HTTPTarget", class_module="tests", model_name="", endpoint="https://bank.example"
+    )
+    hr = TargetIdentifier(class_name="HTTPTarget", class_module="tests", model_name="", endpoint="https://hr.example")
+    model = TargetIdentifier(
+        class_name="MockTarget", class_module="tests", model_name="deployment", underlying_model_name="model-x"
+    )
+
+    def document(target: TargetIdentifier) -> dict[str, Any]:
+        attack = AttackIdentifier(class_name="ProbeAttack", class_module="tests", objective_target=target)
+        return AtomicAttackIdentifier.build(attack_identifier=attack).model_dump()
+
+    def original_rule_hash(target: TargetIdentifier) -> str:
+        return compute_eval_hash(
+            target,
+            child_eval_rules=ObjectiveTargetAnalyticsIdentityV1._CHILD_RULES,
+            own_rule=ObjectiveTargetAnalyticsIdentityV1._OWN_RULE,
+            root_unwrap_child="targets",
+        )
+
+    try:
+        with engine.begin() as connection:
+            config = configuration(connection)
+            command.upgrade(config, ANALYTICS_REVISION)
+            ids = {
+                name: insert_result(connection, identifier=document(t))
+                for name, t in (("bank", bank), ("hr", hr), ("model", model))
+            }
+            # Simulate keys written before the correction: both HTTP targets share one merged key.
+            for name, target in (("bank", bank), ("hr", hr), ("model", model)):
+                connection.execute(
+                    text('UPDATE "AttackResultEntries" SET objective_target_eval_hash_v1 = :key WHERE id = :id'),
+                    {"key": original_rule_hash(target), "id": ids[name]},
+                )
+            assert original_rule_hash(bank) == original_rule_hash(hr)
+
+            command.upgrade(config, "head")
+
+            stored = dict(
+                connection.execute(text('SELECT id, objective_target_eval_hash_v1 FROM "AttackResultEntries"')).all()
+            )
+            assert stored == {
+                ids["bank"]: bank.hash,
+                ids["hr"]: hr.hash,
+                ids["model"]: original_rule_hash(model),
+            }
+            assert stored[ids["model"]] == ObjectiveTargetAnalyticsIdentityV1.hash(identifier=model)
+            command.check(config)
+            command.downgrade(config, ANALYTICS_REVISION)
+            assert (
+                dict(
+                    connection.execute(
+                        text('SELECT id, objective_target_eval_hash_v1 FROM "AttackResultEntries"')
+                    ).all()
+                )
+                == stored
+            )
+    finally:
+        engine.dispose()

tests/unit/memory/test_attack_analytics_eval_identity.py

@@ -42,6 +42,7 @@
     ObjectiveTargetEvaluationIdentifier,
     TargetIdentifier,
 )
+from pyrit.prompt_target import GandalfLevel, GandalfTarget, HTTPTarget
 
 if TYPE_CHECKING:
     from pyrit.memory import SQLiteMemory
@@ -103,6 +104,88 @@ def test_target_evaluation_v1_is_frozen_to_the_current_marker_rules() -> None:
     )
 
 
+def _model_less(*, class_name: str, endpoint: str, model_name: str = "") -> TargetIdentifier:
+    return TargetIdentifier(class_name=class_name, class_module="tests", model_name=model_name, endpoint=endpoint)
+
+
+@pytest.mark.usefixtures("patch_central_database")
+def test_targets_without_a_model_name_fall_back_to_their_content_hash() -> None:
+    bank = HTTPTarget(http_request="POST /chat HTTP/1.1\nHost: bank.example\n\n{PROMPT}").get_identifier()
+    hr = HTTPTarget(http_request="POST /ask HTTP/1.1\nHost: hr.example\n\n{PROMPT}").get_identifier()
+    level_one = GandalfTarget(level=GandalfLevel.LEVEL_1).get_identifier()
+    level_seven = GandalfTarget(level=GandalfLevel.LEVEL_7).get_identifier()
+    blank = _model_less(class_name="MockTarget", endpoint="https://blank.example", model_name="   ")
+    wrapped = TargetIdentifier(class_name="RoundRobinTarget", class_module="tests", targets=[bank])
+
+    keys = {
+        name: ObjectiveTargetAnalyticsIdentityV1.hash(identifier=identifier)
+        for name, identifier in {
+            "bank": bank,
+            "hr": hr,
+            "level_one": level_one,
+            "level_seven": level_seven,
+            "blank": blank,
+        }.items()
+    }
+
+    assert len(set(keys.values())) == 5
+    assert keys["bank"] == TargetIdentifier.from_component_identifier(bank).hash
+    assert keys["level_one"] == TargetIdentifier.from_component_identifier(level_one).hash
+    assert keys["blank"] == blank.hash
+    assert ObjectiveTargetAnalyticsIdentityV1.hash(identifier=wrapped) == keys["bank"]
+    # Targets with a model name keep the behavioral key, so deployments still merge.
+    assert ObjectiveTargetAnalyticsIdentityV1.hash(
+        identifier=_target(deployment="deploy-a", endpoint="https://one.example")
+    ) == ObjectiveTargetAnalyticsIdentityV1.hash(
+        identifier=_target(deployment="deploy-b", endpoint="https://two.example")
+    )
+
+
+async def test_model_less_targets_group_filter_facet_and_label_apart_async(sqlite_instance: SQLiteMemory) -> None:
+    bank = _model_less(class_name="HTTPTarget", endpoint="https://bank.example")
+    hr = _model_less(class_name="HTTPTarget", endpoint="https://hr.example")
+    model = _target(deployment="deploy-a", endpoint="https://one.example")
+    results = [
+        _result(index=1, target=bank, outcome=AttackOutcome.SUCCESS, operation="night"),
+        _result(index=2, target=bank, outcome=AttackOutcome.SUCCESS, operation="day"),
+        _result(index=3, target=hr, outcome=AttackOutcome.FAILURE, operation="night"),
+        _result(index=4, target=model, outcome=AttackOutcome.FAILURE, operation="day"),
+    ]
+    await sqlite_instance.add_attack_results_to_memory_async(attack_results=results)
+    reader = AttackAnalyticsReader(memory=sqlite_instance)
+    axis = AttackAnalyticsDimension(name="objective_target")
+    bank_key, hr_key, model_key = (ObjectiveTargetAnalyticsIdentityV1.hash(identifier=t) for t in (bank, hr, model))
+
+    report = await reader.report_async(query=AttackAnalyticsQuery(group_by=axis), control=_control())
+    groups = {group.option.key.value: group for group in report.groups}
+    bank_filter = AttackAnalyticsFilters(
+        dimensions=[AttackAnalyticsFilter(dimension=axis, values=[groups[bank_key].option.key])]
+    )
+    drilled = await reader.report_async(
+        query=AttackAnalyticsQuery(filters=bank_filter, group_by=AttackAnalyticsDimension(name="operation")),
+        control=_control(),
+    )
+    facets = await reader.facets_async(query=AttackAnalyticsFacetQuery(dimension=axis), control=_control())
+
+    assert {key: group.counts for key, group in groups.items()} == {
+        bank_key: {"success": 2},
+        hr_key: {"failure": 1},
+        model_key: {"failure": 1},
+    }
+    assert groups[bank_key].option.label == groups[hr_key].option.label == "HTTPTarget"
+    assert groups[model_key].option.label == "deploy-a"
+    assert all("example" not in (group.option.label or "") for group in report.groups)
+    assert {group.option.key.value: group.counts for group in drilled.groups} == {
+        "night": {"success": 1},
+        "day": {"success": 1},
+    }
+    assert {item.key.value: item.label for item in facets.items} == {
+        bank_key: "HTTPTarget",
+        hr_key: "HTTPTarget",
+        model_key: "deploy-a",
+    }
+
+
 async def test_evaluation_groups_filters_facets_and_pages_share_result_identity(sqlite_instance: SQLiteMemory) -> None:
     first = _target(deployment="deploy-a", endpoint="https://one.example")
     second = _target(deployment="deploy-b", endpoint="https://two.example")