From 4b00aba4d275cf1bc4f86988aca0e1d2753ee6a9 Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:52:23 -0700 Subject: [PATCH 1/4] [REFACTOR]: Add shared linear trace runner --- rampart/core/__init__.py | 10 ++ rampart/core/trace.py | 212 ++++++++++++++++++++++++ tests/unit/core/test_trace.py | 301 ++++++++++++++++++++++++++++++++++ 3 files changed, 523 insertions(+) create mode 100644 rampart/core/trace.py create mode 100644 tests/unit/core/test_trace.py diff --git a/rampart/core/__init__.py b/rampart/core/__init__.py index 6487d084..4ca46703 100644 --- a/rampart/core/__init__.py +++ b/rampart/core/__init__.py @@ -36,6 +36,12 @@ resolve_attack_verdict, resolve_probe_verdict, ) +from rampart.core.trace import ( + EvaluationRecord, + TraceRun, + evaluate_terminal_async, + run_trace_async, +) from rampart.core.types import ( EvalContext, EvalOutcome, @@ -63,6 +69,7 @@ "EvalOutcome", "EvalResult", "EvaluationPurpose", + "EvaluationRecord", "Evaluator", "ExecutionEvent", "ExecutionEventData", @@ -92,11 +99,14 @@ "ToolCall", "ToolDeclaration", "TraceEndReason", + "TraceRun", "Turn", + "evaluate_terminal_async", "evaluate_turn_async", "execute_trials_async", "resolve_as_attack", "resolve_as_probe", "resolve_attack_verdict", "resolve_probe_verdict", + "run_trace_async", ] diff --git a/rampart/core/trace.py b/rampart/core/trace.py new file mode 100644 index 00000000..3b54ac9d --- /dev/null +++ b/rampart/core/trace.py @@ -0,0 +1,212 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared linear trace execution and terminal evaluation helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING + +from rampart.core.types import ( + EvalContext, + EvalResult, + EvaluationPurpose, + ObservabilityLevel, + TraceEndReason, + Turn, +) + +if TYPE_CHECKING: + from rampart.core.adapter import Session + from rampart.core.evaluator import Evaluator + from rampart.core.manifest import AppManifest + from rampart.core.prompt_driver import PromptDriver + + +@dataclass(frozen=True, kw_only=True, eq=False) +class EvaluationRecord: + """One online evaluation and the exact context it judged. + + Args: + evaluator: Evaluator object that produced the result. Identity is the + reuse boundary. + context: Exact raw-trace context passed to the evaluator. + result: Evaluation returned for that context. + """ + + evaluator: Evaluator + context: EvalContext + result: EvalResult + + +@dataclass(kw_only=True) +class TraceRun: + """A completed linear trace and its latest online evaluation. + + ``turns`` is the driver/report view and may carry online evidence. + ``raw_turns`` is the evaluator view and never carries framework-produced + evaluation annotations. + + Args: + trace_end_reason: Why the trace stopped producing turns. + observability_level: What the adapter can observe. + manifest: Agent capabilities used to create evaluator contexts. + turns: Annotated history passed to prompt drivers and results. + raw_turns: Annotation-free history passed to evaluators. + latest_online_evaluation: Most recent stop-condition evaluation. + """ + + trace_end_reason: TraceEndReason + observability_level: ObservabilityLevel + manifest: AppManifest | None = None + turns: list[Turn] = field(default_factory=list[Turn]) + raw_turns: list[Turn] = field(default_factory=list[Turn]) + latest_online_evaluation: EvaluationRecord | None = None + + +def _evaluation_context( + *, + raw_turns: list[Turn], + observability_level: ObservabilityLevel, + manifest: AppManifest | None, +) -> EvalContext: + """Build an evaluator context from a snapshot of the raw trace. + + Returns: + EvalContext: Context holding a shallow snapshot of raw turns. + """ + return EvalContext( + turns=list(raw_turns), + observability_level=observability_level, + manifest=manifest, + ) + + +async def run_trace_async( + *, + session: Session, + driver: PromptDriver, + max_turns: int, + observability_level: ObservabilityLevel, + stop_when: Evaluator | None = None, + manifest: AppManifest | None = None, +) -> TraceRun: + """Drive a linear conversation with optional online stopping. + + The runner does not own session lifetime or exception conversion. Callers + keep the session context active around this function, and exceptions from + the driver, session, or evaluator propagate unchanged. + + Args: + session: Active agent session. + driver: Prompt source for the conversation. + max_turns: Maximum number of requests sent to the agent. + observability_level: What the adapter can observe. + stop_when: Optional evaluator checked after every response. A detected + outcome terminates the trace. + manifest: Agent capabilities exposed to evaluators. + + Returns: + TraceRun: Completed turns, termination reason, and online evidence. + + Raises: + ValueError: If ``max_turns`` is negative. + """ + if max_turns < 0: + msg = "max_turns must be non-negative." + raise ValueError(msg) + + run = TraceRun( + trace_end_reason=TraceEndReason.MAX_TURNS_REACHED, + observability_level=observability_level, + manifest=manifest, + ) + + for turn_index in range(max_turns): + decision = await driver.next_prompt_async(history=list(run.turns)) + if decision is None: + run.trace_end_reason = TraceEndReason.DRIVER_EXHAUSTED + return run + + response = await session.send_async(decision.request) + raw_turn = Turn( + request=decision.request, + response=response, + turn_number=turn_index, + driver_reasoning=decision.reasoning, + ) + run.raw_turns.append(raw_turn) + + if stop_when is None: + run.turns.append(raw_turn) + continue + + context = _evaluation_context( + raw_turns=run.raw_turns, + observability_level=observability_level, + manifest=manifest, + ) + evaluation = await stop_when.evaluate_async(context=context) + run.latest_online_evaluation = EvaluationRecord( + evaluator=stop_when, + context=context, + result=evaluation, + ) + run.turns.append( + replace( + raw_turn, + eval_result=evaluation, + eval_purpose=EvaluationPurpose.STOP_CHECK, + ), + ) + if evaluation.detected: + run.trace_end_reason = TraceEndReason.STOP_CONDITION_MET + return run + + return run + + +async def evaluate_terminal_async( + *, + evaluator: Evaluator, + run: TraceRun, +) -> EvalResult | None: + """Evaluate the terminal raw trace, reusing an identical online judgment. + + Args: + evaluator: Evaluator responsible for the final verdict. + run: Completed trace from :func:`run_trace_async`. + + Returns: + EvalResult | None: Final evaluation, or None when no turns exist. + + Call this before leaving any active session or injection context required + by the evaluator. Requests, responses, and their nested values are treated + as immutable after the runner appends them. + """ + if not run.raw_turns: + return None + + record = run.latest_online_evaluation + if ( + record is not None + and record.evaluator is evaluator + and len(record.context.turns) == len(run.raw_turns) + and all( + evaluated is terminal + for evaluated, terminal in zip( + record.context.turns, + run.raw_turns, + strict=True, + ) + ) + ): + return replace(record.result, evidence=list(record.result.evidence)) + + context = _evaluation_context( + raw_turns=run.raw_turns, + observability_level=run.observability_level, + manifest=run.manifest, + ) + return await evaluator.evaluate_async(context=context) diff --git a/tests/unit/core/test_trace.py b/tests/unit/core/test_trace.py new file mode 100644 index 00000000..924099de --- /dev/null +++ b/tests/unit/core/test_trace.py @@ -0,0 +1,301 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Tests for the shared linear trace runner.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from rampart.core.evaluator import Evaluator +from rampart.core.manifest import AppManifest +from rampart.core.prompt_driver import PromptDecision +from rampart.core.trace import evaluate_terminal_async, run_trace_async +from rampart.core.types import ( + EvalOutcome, + EvalResult, + EvaluationPurpose, + ObservabilityLevel, + Request, + Response, + TraceEndReason, + Turn, +) +from rampart.drivers.static import StaticDriver +from tests.fixtures import MockSession + + +def _session(*responses: str) -> MockSession: + """Build a session returning the supplied response texts.""" + return MockSession(responses=[Response(text=text) for text in responses]) + + +def _evaluator(*outcomes: EvalOutcome) -> AsyncMock: + """Build an evaluator mock returning outcomes in order.""" + evaluator = AsyncMock(spec=Evaluator) + evaluator.evaluate_async.side_effect = [ + EvalResult(outcome=outcome, rationale=f"call {index}") + for index, outcome in enumerate(outcomes) + ] + return evaluator + + +class TestRunTraceAsync: + async def test_driver_exhaustion_returns_raw_turns_async(self) -> None: + run = await run_trace_async( + session=_session("r1", "r2"), + driver=StaticDriver(prompts=["p1", "p2"]), + max_turns=3, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) + + assert run.trace_end_reason is TraceEndReason.DRIVER_EXHAUSTED + assert [turn.response.text for turn in run.turns] == ["r1", "r2"] + assert run.turns == run.raw_turns + assert run.latest_online_evaluation is None + + async def test_turn_budget_is_a_normal_termination_async(self) -> None: + run = await run_trace_async( + session=_session("r1", "r2", "r3"), + driver=StaticDriver(prompts=["p1", "p2", "p3"]), + max_turns=2, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) + + assert run.trace_end_reason is TraceEndReason.MAX_TURNS_REACHED + assert len(run.turns) == 2 + + async def test_zero_budget_does_not_call_driver_async(self) -> None: + driver = AsyncMock() + + run = await run_trace_async( + session=_session("unused"), + driver=driver, + max_turns=0, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) + + assert run.trace_end_reason is TraceEndReason.MAX_TURNS_REACHED + assert run.turns == [] + driver.next_prompt_async.assert_not_awaited() + + async def test_stop_condition_annotates_only_public_history_async(self) -> None: + evaluator = _evaluator(EvalOutcome.NOT_DETECTED, EvalOutcome.DETECTED) + manifest = AppManifest(name="agent") + + run = await run_trace_async( + session=_session("r1", "r2", "r3"), + driver=StaticDriver(prompts=["p1", "p2", "p3"]), + max_turns=3, + observability_level=ObservabilityLevel.TOOL_ONLY, + stop_when=evaluator, + manifest=manifest, + ) + + assert run.trace_end_reason is TraceEndReason.STOP_CONDITION_MET + assert len(run.turns) == 2 + assert all( + turn.eval_purpose is EvaluationPurpose.STOP_CHECK for turn in run.turns + ) + assert all(turn.eval_result is not None for turn in run.turns) + assert all(turn.eval_result is None for turn in run.raw_turns) + contexts = [ + call.kwargs["context"] for call in evaluator.evaluate_async.await_args_list + ] + assert [len(context.turns) for context in contexts] == [1, 2] + assert all( + turn.eval_result is None for context in contexts for turn in context.turns + ) + assert all( + context.observability_level is ObservabilityLevel.TOOL_ONLY + for context in contexts + ) + assert contexts[-1].manifest is manifest + + async def test_driver_cannot_mutate_owned_history_list_async(self) -> None: + class MutatingDriver: + def __init__(self) -> None: + self.calls = 0 + + async def next_prompt_async( + self, + *, + history: list[Turn], + ) -> PromptDecision | None: + history.append( + Turn( + request=Request(prompt="injected"), + response=Response(text="injected"), + ), + ) + if self.calls: + return None + self.calls += 1 + return PromptDecision(request=Request(prompt="p")) + + run = await run_trace_async( + session=_session("r"), + driver=MutatingDriver(), + max_turns=2, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) + + assert len(run.turns) == 1 + assert run.turns[0].request.prompt == "p" + + async def test_evaluator_exception_propagates_async(self) -> None: + evaluator = AsyncMock(spec=Evaluator) + evaluator.evaluate_async.side_effect = RuntimeError("judge failed") + + with pytest.raises(RuntimeError, match="judge failed"): + await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + + +class TestEvaluateTerminalAsync: + async def test_empty_trace_skips_evaluator_async(self) -> None: + evaluator = _evaluator(EvalOutcome.DETECTED) + run = await run_trace_async( + session=_session("unused"), + driver=StaticDriver(prompts=[]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) + + result = await evaluate_terminal_async(evaluator=evaluator, run=run) + + assert result is None + assert run.trace_end_reason is TraceEndReason.DRIVER_EXHAUSTED + evaluator.evaluate_async.assert_not_awaited() + + @pytest.mark.parametrize( + "outcomes", + [ + (EvalOutcome.DETECTED,), + (EvalOutcome.NOT_DETECTED,), + ], + ) + async def test_reuses_identical_latest_online_evaluation_async( + self, + outcomes: tuple[EvalOutcome, ...], + ) -> None: + evaluator = _evaluator(*outcomes) + run = await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + online_result = run.latest_online_evaluation + assert online_result is not None + + result = await evaluate_terminal_async(evaluator=evaluator, run=run) + + assert result == online_result.result + assert result is not online_result.result + assert result.evidence is not online_result.result.evidence + assert evaluator.evaluate_async.await_count == 1 + + async def test_non_firing_stop_reuses_terminal_prefix_without_extra_call_async( + self, + ) -> None: + evaluator = _evaluator( + EvalOutcome.NOT_DETECTED, + EvalOutcome.NOT_DETECTED, + EvalOutcome.NOT_DETECTED, + ) + run = await run_trace_async( + session=_session("r1", "r2", "r3"), + driver=StaticDriver(prompts=["p1", "p2", "p3"]), + max_turns=3, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + + result = await evaluate_terminal_async(evaluator=evaluator, run=run) + + assert result is not None and result.outcome is EvalOutcome.NOT_DETECTED + assert evaluator.evaluate_async.await_count == 3 + + async def test_distinct_evaluator_runs_once_on_terminal_trace_async(self) -> None: + stop = _evaluator(EvalOutcome.NOT_DETECTED) + verdict = _evaluator(EvalOutcome.DETECTED) + manifest = AppManifest(name="agent") + run = await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.RESPONSE_ONLY, + stop_when=stop, + manifest=manifest, + ) + + result = await evaluate_terminal_async( + evaluator=verdict, + run=run, + ) + + assert result is not None and result.outcome is EvalOutcome.DETECTED + verdict.evaluate_async.assert_awaited_once() + context = verdict.evaluate_async.await_args.kwargs["context"] + assert context.turns == run.raw_turns + assert context.observability_level is ObservabilityLevel.RESPONSE_ONLY + assert all(turn.eval_result is None for turn in context.turns) + + async def test_post_run_trace_mutation_prevents_reuse_async(self) -> None: + evaluator = _evaluator(EvalOutcome.NOT_DETECTED, EvalOutcome.DETECTED) + run = await run_trace_async( + session=_session("r", "later"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + run.raw_turns.append( + Turn( + request=Request(prompt="later"), + response=Response(text="later"), + turn_number=1, + ), + ) + + result = await evaluate_terminal_async(evaluator=evaluator, run=run) + + assert result is not None and result.outcome is EvalOutcome.DETECTED + assert evaluator.evaluate_async.await_count == 2 + + async def test_undetermined_stop_does_not_terminate_async(self) -> None: + evaluator = _evaluator( + EvalOutcome.UNDETERMINED, + EvalOutcome.NOT_DETECTED, + ) + run = await run_trace_async( + session=_session("r1", "r2"), + driver=StaticDriver(prompts=["p1", "p2"]), + max_turns=2, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + + assert run.trace_end_reason is TraceEndReason.MAX_TURNS_REACHED + assert len(run.turns) == 2 + assert run.turns[0].eval_purpose is EvaluationPurpose.STOP_CHECK + + +async def test_negative_turn_budget_raises_async() -> None: + """Negative budgets are rejected rather than treated as zero.""" + with pytest.raises(ValueError, match="non-negative"): + await run_trace_async( + session=_session("unused"), + driver=StaticDriver(prompts=[]), + max_turns=-1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + ) From 2bb235af3e109a4faf9016849ae588b4b60c81f4 Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:42:22 -0700 Subject: [PATCH 2/4] [FIX]: Guard terminal reuse by complete evaluation context Require the same observability and manifest before reusing an online judgment. Copy optional evidence through the shared tolerant renderer so malformed supporting text cannot discard an established verdict. Keep terminal evidence and operand lists independent from online records. --- rampart/core/trace.py | 41 ++++++++++----- tests/unit/core/test_trace.py | 97 +++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/rampart/core/trace.py b/rampart/core/trace.py index 3b54ac9d..5f1d96e6 100644 --- a/rampart/core/trace.py +++ b/rampart/core/trace.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING +from rampart.common.text import safe_str_list from rampart.core.types import ( EvalContext, EvalResult, @@ -83,6 +84,23 @@ def _evaluation_context( ) +def _matches_terminal_context(*, context: EvalContext, run: TraceRun) -> bool: + """Check that the raw trace and adapter context are unchanged. + + Returns: + bool: Whether this context can supply the terminal judgment. + """ + return ( + context.observability_level is run.observability_level + and context.manifest is run.manifest + and len(context.turns) == len(run.raw_turns) + and all( + evaluated is terminal + for evaluated, terminal in zip(context.turns, run.raw_turns, strict=True) + ) + ) + + async def run_trace_async( *, session: Session, @@ -182,8 +200,9 @@ async def evaluate_terminal_async( EvalResult | None: Final evaluation, or None when no turns exist. Call this before leaving any active session or injection context required - by the evaluator. Requests, responses, and their nested values are treated - as immutable after the runner appends them. + by the evaluator. Reuse requires matching evaluator, raw-turn, and manifest + identities and the same observability level. Requests, responses, manifests, + and their nested values are treated as immutable once evaluated. """ if not run.raw_turns: return None @@ -192,17 +211,15 @@ async def evaluate_terminal_async( if ( record is not None and record.evaluator is evaluator - and len(record.context.turns) == len(run.raw_turns) - and all( - evaluated is terminal - for evaluated, terminal in zip( - record.context.turns, - run.raw_turns, - strict=True, - ) - ) + and _matches_terminal_context(context=record.context, run=run) ): - return replace(record.result, evidence=list(record.result.evidence)) + return replace( + record.result, + evidence=safe_str_list(value=record.result.evidence), + undetermined_operands=safe_str_list( + value=record.result.undetermined_operands, + ), + ) context = _evaluation_context( raw_turns=run.raw_turns, diff --git a/tests/unit/core/test_trace.py b/tests/unit/core/test_trace.py index 924099de..db6754d8 100644 --- a/tests/unit/core/test_trace.py +++ b/tests/unit/core/test_trace.py @@ -5,6 +5,7 @@ from __future__ import annotations +from dataclasses import replace from unittest.mock import AsyncMock import pytest @@ -24,9 +25,15 @@ Turn, ) from rampart.drivers.static import StaticDriver +from rampart.evaluators import ToolCalled from tests.fixtures import MockSession +class _FailingIterable: + def __iter__(self) -> object: + raise RuntimeError("evidence unavailable") + + def _session(*responses: str) -> MockSession: """Build a session returning the supplied response texts.""" return MockSession(responses=[Response(text=text) for text in responses]) @@ -160,6 +167,95 @@ async def test_evaluator_exception_propagates_async(self) -> None: class TestEvaluateTerminalAsync: + async def test_changed_observability_requires_new_judgment_async(self) -> None: + evaluator = ToolCalled("send_email") + run = await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + updated_run = replace( + run, + observability_level=ObservabilityLevel.RESPONSE_ONLY, + ) + + result = await evaluate_terminal_async(evaluator=evaluator, run=updated_run) + + assert run.latest_online_evaluation is not None + assert run.latest_online_evaluation.result.outcome is EvalOutcome.NOT_DETECTED + assert result is not None + assert result.outcome is EvalOutcome.UNDETERMINED + + @pytest.mark.parametrize("manifest", [None, AppManifest(name="agent")]) + async def test_changed_manifest_requires_new_judgment_async( + self, + manifest: AppManifest | None, + ) -> None: + evaluator = _evaluator(EvalOutcome.NOT_DETECTED, EvalOutcome.DETECTED) + run = await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + manifest=AppManifest(name="agent"), + ) + + result = await evaluate_terminal_async( + evaluator=evaluator, + run=replace(run, manifest=manifest), + ) + + assert result is not None and result.outcome is EvalOutcome.DETECTED + assert evaluator.evaluate_async.await_count == 2 + assert ( + evaluator.evaluate_async.await_args.kwargs["context"].manifest is manifest + ) + + @pytest.mark.parametrize( + ("evidence", "expected"), + [ + (None, []), + (42, []), + (_FailingIterable(), []), + ("confirmed", ["confirmed"]), + (["confirmed"], ["confirmed"]), + ], + ) + async def test_reuse_preserves_verdict_with_optional_evidence_async( + self, + evidence: object, + expected: list[str], + ) -> None: + online = EvalResult( + outcome=EvalOutcome.DETECTED, + evidence=evidence, # ty: ignore[invalid-argument-type] + rationale="condition confirmed", + undetermined_operands=["missing side effects"], + ) + evaluator = AsyncMock(spec=Evaluator) + evaluator.evaluate_async.return_value = online + run = await run_trace_async( + session=_session("r"), + driver=StaticDriver(prompts=["p"]), + max_turns=1, + observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + stop_when=evaluator, + ) + + result = await evaluate_terminal_async(evaluator=evaluator, run=run) + + assert result is not None + assert result.outcome is EvalOutcome.DETECTED + assert result.rationale == online.rationale + assert result.evidence == expected + assert result.evidence is not online.evidence + assert result.undetermined_operands == online.undetermined_operands + assert result.undetermined_operands is not online.undetermined_operands + evaluator.evaluate_async.assert_awaited_once() + async def test_empty_trace_skips_evaluator_async(self) -> None: evaluator = _evaluator(EvalOutcome.DETECTED) run = await run_trace_async( @@ -193,6 +289,7 @@ async def test_reuses_identical_latest_online_evaluation_async( max_turns=1, observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, stop_when=evaluator, + manifest=AppManifest(name="agent"), ) online_result = run.latest_online_evaluation assert online_result is not None From ab186d444bc5f496de3a66aadef57c0e2038d44d Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:44:49 -0700 Subject: [PATCH 3/4] [REFACTOR]: Name the final-trace evaluation helper Rename evaluate_terminal_async to evaluate_final_trace_async so the public runner helper matches Result.final_trace_evaluation. Document the trace execution helpers where they are introduced. --- docs/api/core-protocols.md | 10 ++++++++++ rampart/core/__init__.py | 4 ++-- rampart/core/trace.py | 16 ++++++++-------- tests/unit/core/test_trace.py | 24 ++++++++++++------------ 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/api/core-protocols.md b/docs/api/core-protocols.md index c4571992..dc9c3d5c 100644 --- a/docs/api/core-protocols.md +++ b/docs/api/core-protocols.md @@ -55,6 +55,16 @@ Protocols and ABCs that define RAMPART's extension points. Implement these to co - register_default_handler_factory - clear_default_handler_factory +## Trace Execution + +::: rampart.core.trace + options: + members: + - EvaluationRecord + - TraceRun + - run_trace_async + - evaluate_final_trace_async + ## Errors ::: rampart.core.errors diff --git a/rampart/core/__init__.py b/rampart/core/__init__.py index 4ca46703..89a50326 100644 --- a/rampart/core/__init__.py +++ b/rampart/core/__init__.py @@ -39,7 +39,7 @@ from rampart.core.trace import ( EvaluationRecord, TraceRun, - evaluate_terminal_async, + evaluate_final_trace_async, run_trace_async, ) from rampart.core.types import ( @@ -101,7 +101,7 @@ "TraceEndReason", "TraceRun", "Turn", - "evaluate_terminal_async", + "evaluate_final_trace_async", "evaluate_turn_async", "execute_trials_async", "resolve_as_attack", diff --git a/rampart/core/trace.py b/rampart/core/trace.py index 5f1d96e6..0741889c 100644 --- a/rampart/core/trace.py +++ b/rampart/core/trace.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Shared linear trace execution and terminal evaluation helpers.""" +"""Shared linear trace execution and final-trace evaluation helpers.""" from __future__ import annotations @@ -84,19 +84,19 @@ def _evaluation_context( ) -def _matches_terminal_context(*, context: EvalContext, run: TraceRun) -> bool: +def _matches_final_trace_context(*, context: EvalContext, run: TraceRun) -> bool: """Check that the raw trace and adapter context are unchanged. Returns: - bool: Whether this context can supply the terminal judgment. + bool: Whether this context can supply the final-trace judgment. """ return ( context.observability_level is run.observability_level and context.manifest is run.manifest and len(context.turns) == len(run.raw_turns) and all( - evaluated is terminal - for evaluated, terminal in zip(context.turns, run.raw_turns, strict=True) + evaluated is final + for evaluated, final in zip(context.turns, run.raw_turns, strict=True) ) ) @@ -185,12 +185,12 @@ async def run_trace_async( return run -async def evaluate_terminal_async( +async def evaluate_final_trace_async( *, evaluator: Evaluator, run: TraceRun, ) -> EvalResult | None: - """Evaluate the terminal raw trace, reusing an identical online judgment. + """Evaluate the final raw trace, reusing an identical online judgment. Args: evaluator: Evaluator responsible for the final verdict. @@ -211,7 +211,7 @@ async def evaluate_terminal_async( if ( record is not None and record.evaluator is evaluator - and _matches_terminal_context(context=record.context, run=run) + and _matches_final_trace_context(context=record.context, run=run) ): return replace( record.result, diff --git a/tests/unit/core/test_trace.py b/tests/unit/core/test_trace.py index db6754d8..4e0b858b 100644 --- a/tests/unit/core/test_trace.py +++ b/tests/unit/core/test_trace.py @@ -13,7 +13,7 @@ from rampart.core.evaluator import Evaluator from rampart.core.manifest import AppManifest from rampart.core.prompt_driver import PromptDecision -from rampart.core.trace import evaluate_terminal_async, run_trace_async +from rampart.core.trace import evaluate_final_trace_async, run_trace_async from rampart.core.types import ( EvalOutcome, EvalResult, @@ -166,7 +166,7 @@ async def test_evaluator_exception_propagates_async(self) -> None: ) -class TestEvaluateTerminalAsync: +class TestEvaluateFinalTraceAsync: async def test_changed_observability_requires_new_judgment_async(self) -> None: evaluator = ToolCalled("send_email") run = await run_trace_async( @@ -181,7 +181,7 @@ async def test_changed_observability_requires_new_judgment_async(self) -> None: observability_level=ObservabilityLevel.RESPONSE_ONLY, ) - result = await evaluate_terminal_async(evaluator=evaluator, run=updated_run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=updated_run) assert run.latest_online_evaluation is not None assert run.latest_online_evaluation.result.outcome is EvalOutcome.NOT_DETECTED @@ -203,7 +203,7 @@ async def test_changed_manifest_requires_new_judgment_async( manifest=AppManifest(name="agent"), ) - result = await evaluate_terminal_async( + result = await evaluate_final_trace_async( evaluator=evaluator, run=replace(run, manifest=manifest), ) @@ -245,7 +245,7 @@ async def test_reuse_preserves_verdict_with_optional_evidence_async( stop_when=evaluator, ) - result = await evaluate_terminal_async(evaluator=evaluator, run=run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=run) assert result is not None assert result.outcome is EvalOutcome.DETECTED @@ -265,7 +265,7 @@ async def test_empty_trace_skips_evaluator_async(self) -> None: observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, ) - result = await evaluate_terminal_async(evaluator=evaluator, run=run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=run) assert result is None assert run.trace_end_reason is TraceEndReason.DRIVER_EXHAUSTED @@ -294,14 +294,14 @@ async def test_reuses_identical_latest_online_evaluation_async( online_result = run.latest_online_evaluation assert online_result is not None - result = await evaluate_terminal_async(evaluator=evaluator, run=run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=run) assert result == online_result.result assert result is not online_result.result assert result.evidence is not online_result.result.evidence assert evaluator.evaluate_async.await_count == 1 - async def test_non_firing_stop_reuses_terminal_prefix_without_extra_call_async( + async def test_non_firing_stop_reuses_final_prefix_without_extra_call_async( self, ) -> None: evaluator = _evaluator( @@ -317,12 +317,12 @@ async def test_non_firing_stop_reuses_terminal_prefix_without_extra_call_async( stop_when=evaluator, ) - result = await evaluate_terminal_async(evaluator=evaluator, run=run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=run) assert result is not None and result.outcome is EvalOutcome.NOT_DETECTED assert evaluator.evaluate_async.await_count == 3 - async def test_distinct_evaluator_runs_once_on_terminal_trace_async(self) -> None: + async def test_distinct_evaluator_runs_once_on_final_trace_async(self) -> None: stop = _evaluator(EvalOutcome.NOT_DETECTED) verdict = _evaluator(EvalOutcome.DETECTED) manifest = AppManifest(name="agent") @@ -335,7 +335,7 @@ async def test_distinct_evaluator_runs_once_on_terminal_trace_async(self) -> Non manifest=manifest, ) - result = await evaluate_terminal_async( + result = await evaluate_final_trace_async( evaluator=verdict, run=run, ) @@ -364,7 +364,7 @@ async def test_post_run_trace_mutation_prevents_reuse_async(self) -> None: ), ) - result = await evaluate_terminal_async(evaluator=evaluator, run=run) + result = await evaluate_final_trace_async(evaluator=evaluator, run=run) assert result is not None and result.outcome is EvalOutcome.DETECTED assert evaluator.evaluate_async.await_count == 2 From 45fb3bb44e538ea24476ac63d4ce021dfb6a9b5a Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:09:00 -0700 Subject: [PATCH 4/4] [REFACTOR]: Clarify stop-check naming in the trace runner Rename TraceRun.latest_online_evaluation to latest_stop_check, matching EvaluationPurpose.STOP_CHECK, and describe the final-trace evaluator as producing evidence rather than the safety status. --- rampart/core/trace.py | 17 +++++++++-------- tests/unit/core/test_trace.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/rampart/core/trace.py b/rampart/core/trace.py index 0741889c..1d357ac7 100644 --- a/rampart/core/trace.py +++ b/rampart/core/trace.py @@ -27,7 +27,7 @@ @dataclass(frozen=True, kw_only=True, eq=False) class EvaluationRecord: - """One online evaluation and the exact context it judged. + """One stop-condition check and the exact context it judged. Args: evaluator: Evaluator object that produced the result. Identity is the @@ -43,7 +43,7 @@ class EvaluationRecord: @dataclass(kw_only=True) class TraceRun: - """A completed linear trace and its latest online evaluation. + """A completed linear trace and its latest stop-condition check. ``turns`` is the driver/report view and may carry online evidence. ``raw_turns`` is the evaluator view and never carries framework-produced @@ -55,7 +55,8 @@ class TraceRun: manifest: Agent capabilities used to create evaluator contexts. turns: Annotated history passed to prompt drivers and results. raw_turns: Annotation-free history passed to evaluators. - latest_online_evaluation: Most recent stop-condition evaluation. + latest_stop_check: Most recent ``stop_when`` evaluation, or None when + no stop condition was configured. """ trace_end_reason: TraceEndReason @@ -63,7 +64,7 @@ class TraceRun: manifest: AppManifest | None = None turns: list[Turn] = field(default_factory=list[Turn]) raw_turns: list[Turn] = field(default_factory=list[Turn]) - latest_online_evaluation: EvaluationRecord | None = None + latest_stop_check: EvaluationRecord | None = None def _evaluation_context( @@ -166,7 +167,7 @@ async def run_trace_async( manifest=manifest, ) evaluation = await stop_when.evaluate_async(context=context) - run.latest_online_evaluation = EvaluationRecord( + run.latest_stop_check = EvaluationRecord( evaluator=stop_when, context=context, result=evaluation, @@ -190,10 +191,10 @@ async def evaluate_final_trace_async( evaluator: Evaluator, run: TraceRun, ) -> EvalResult | None: - """Evaluate the final raw trace, reusing an identical online judgment. + """Evaluate the final raw trace, reusing an identical stop-condition check. Args: - evaluator: Evaluator responsible for the final verdict. + evaluator: Evaluator applied to the final trace. run: Completed trace from :func:`run_trace_async`. Returns: @@ -207,7 +208,7 @@ async def evaluate_final_trace_async( if not run.raw_turns: return None - record = run.latest_online_evaluation + record = run.latest_stop_check if ( record is not None and record.evaluator is evaluator diff --git a/tests/unit/core/test_trace.py b/tests/unit/core/test_trace.py index 4e0b858b..e91f2994 100644 --- a/tests/unit/core/test_trace.py +++ b/tests/unit/core/test_trace.py @@ -61,7 +61,7 @@ async def test_driver_exhaustion_returns_raw_turns_async(self) -> None: assert run.trace_end_reason is TraceEndReason.DRIVER_EXHAUSTED assert [turn.response.text for turn in run.turns] == ["r1", "r2"] assert run.turns == run.raw_turns - assert run.latest_online_evaluation is None + assert run.latest_stop_check is None async def test_turn_budget_is_a_normal_termination_async(self) -> None: run = await run_trace_async( @@ -183,8 +183,8 @@ async def test_changed_observability_requires_new_judgment_async(self) -> None: result = await evaluate_final_trace_async(evaluator=evaluator, run=updated_run) - assert run.latest_online_evaluation is not None - assert run.latest_online_evaluation.result.outcome is EvalOutcome.NOT_DETECTED + assert run.latest_stop_check is not None + assert run.latest_stop_check.result.outcome is EvalOutcome.NOT_DETECTED assert result is not None assert result.outcome is EvalOutcome.UNDETERMINED @@ -278,7 +278,7 @@ async def test_empty_trace_skips_evaluator_async(self) -> None: (EvalOutcome.NOT_DETECTED,), ], ) - async def test_reuses_identical_latest_online_evaluation_async( + async def test_reuses_identical_latest_stop_check_async( self, outcomes: tuple[EvalOutcome, ...], ) -> None: @@ -291,14 +291,14 @@ async def test_reuses_identical_latest_online_evaluation_async( stop_when=evaluator, manifest=AppManifest(name="agent"), ) - online_result = run.latest_online_evaluation - assert online_result is not None + stop_check = run.latest_stop_check + assert stop_check is not None result = await evaluate_final_trace_async(evaluator=evaluator, run=run) - assert result == online_result.result - assert result is not online_result.result - assert result.evidence is not online_result.result.evidence + assert result == stop_check.result + assert result is not stop_check.result + assert result.evidence is not stop_check.result.evidence assert evaluator.evaluate_async.await_count == 1 async def test_non_firing_stop_reuses_final_prefix_without_extra_call_async(