From 890674fe782fae42743e179420ddf3d6c4b935bf Mon Sep 17 00:00:00 2001 From: Rsakhuja Date: Tue, 20 Oct 2026 08:26:15 -0400 Subject: [PATCH] fix(engine): give the model a final answer turn when max_iterations is hit Hitting max_iterations broke out of the tool loop and returned the last model response, which at the cutoff is always a tool request with no text. Runs ended as completed with an empty answer, and child agents handed their orchestrator an empty result. Answer the pending tool calls with the existing blocked_message() without executing them and give the model one final, checkpointed turn to answer with what it has. If that turn still has no text, return a fixed fallback answer so a capped run never ends empty. Co-Authored-By: Claude Opus 5.5 --- docs/EXECUTION_LIMITS.md | 8 +- docs/execution-limits.mdx | 2 +- .../engine/langgraph/execution/model_loop.py | 32 +++++- src/agent_engine/runtime/execution_limiter.py | 4 + tests/engine/test_execution_limits.py | 104 ++++++++++++++++++ 5 files changed, 146 insertions(+), 4 deletions(-) diff --git a/docs/EXECUTION_LIMITS.md b/docs/EXECUTION_LIMITS.md index bb65734c..442d02ba 100644 --- a/docs/EXECUTION_LIMITS.md +++ b/docs/EXECUTION_LIMITS.md @@ -65,8 +65,12 @@ Limits degrade **gracefully** — a run never crashes with an obscure exception: The model receives a short controlled message (e.g. *"Tool call blocked: the 'max_tool_calls' execution limit (10) was reached… finish with the information you have."*) and naturally wraps up. -- **`max_iterations`:** the node's tool loop **stops** and returns the latest - model response. No further tools are called. +- **`max_iterations`:** the node's tool loop **stops**. The tool calls the model + requested last are **not executed**; each receives the same controlled message, + and the model gets one final turn to answer with the information it has. No + further tools are called. If that turn still produces no text, the node answers + *"I couldn't finish this request within the allowed number of steps."* — a + capped run never ends with an empty answer. When a limit is hit the engine logs **safe metadata only** at `WARNING`: `run_id`, `node_id`/`agent_id` (when available), the limit name, the current diff --git a/docs/execution-limits.mdx b/docs/execution-limits.mdx index a684c765..3dbb6eec 100644 --- a/docs/execution-limits.mdx +++ b/docs/execution-limits.mdx @@ -49,6 +49,6 @@ Limits are validated at parse time — invalid values (non-integers, wrong types The run **degrades gracefully** — no exception, no crash. - **Tool/child-agent limits and duplicates** — the call is blocked. The model receives a short message like *"Tool call limit reached, finish with what you have."* and naturally wraps up. -- **`max_iterations`** — the node's loop stops and returns the latest model response. +- **`max_iterations`** — the node's loop stops. Its pending tool calls are blocked, and the model gets one final turn to answer with what it has. If that turn has no text, the node answers *"I couldn't finish this request within the allowed number of steps."* instead of returning an empty answer. The engine logs a warning with the run id, limit name, and current count. It never logs prompts, tool arguments, or secrets. diff --git a/src/agent_engine/engine/langgraph/execution/model_loop.py b/src/agent_engine/engine/langgraph/execution/model_loop.py index 7b5b35bb..f08631ab 100644 --- a/src/agent_engine/engine/langgraph/execution/model_loop.py +++ b/src/agent_engine/engine/langgraph/execution/model_loop.py @@ -15,7 +15,9 @@ ) from agent_engine.engine.langgraph.execution.model_context import ModelContext from agent_engine.runtime.execution_limiter import ( + ITERATION_LIMIT_ANSWER, ExecutionLimitExceeded, + blocked_message, current_execution, log_limit, ) @@ -102,7 +104,7 @@ async def next_response() -> Any: limiter.register_iteration(node_path) except ExecutionLimitExceeded as exc: log_limit(exc) - break + return await _wrap_up(response, exc, context, next_response) context.append(response) for tool_call in response.tool_calls: logger.debug( @@ -136,6 +138,34 @@ async def next_response() -> Any: return response +async def _wrap_up( + response: Any, + exc: ExecutionLimitExceeded, + context: ModelContext, + next_response: Callable[[], Awaitable[Any]], +) -> Any: + """Give the model one turn to answer after ``max_iterations`` is reached. + + The pending tool calls are answered with the same controlled message the + other limits use, without executing them, so the model can finish with what + it has. A wrap-up turn that still produces no text yields a fixed answer + instead of an empty one. + """ + context.append(response) + for tool_call in response.tool_calls: + context.append( + ToolMessage( + content=blocked_message(exc), + tool_call_id=tool_call["id"], + name=tool_call["name"], + ) + ) + final = await next_response() + if as_text(final.content).strip(): + return final + return AIMessage(content=ITERATION_LIMIT_ANSWER) + + async def _refresh( context: ModelContext, refresh_execution_context: ExecutionContextRefresher | None, diff --git a/src/agent_engine/runtime/execution_limiter.py b/src/agent_engine/runtime/execution_limiter.py index 3bdcb45b..a275323f 100644 --- a/src/agent_engine/runtime/execution_limiter.py +++ b/src/agent_engine/runtime/execution_limiter.py @@ -14,6 +14,10 @@ logger = logging.getLogger(__name__) +#: Answer returned when a node exhausts ``max_iterations`` and its wrap-up model +#: turn still produces no text, so a capped run never ends with a silent answer. +ITERATION_LIMIT_ANSWER = "I couldn't finish this request within the allowed number of steps." + class ExecutionLimiter: """Enforces an :class:`ExecutionPolicy` for the duration of one run.""" diff --git a/tests/engine/test_execution_limits.py b/tests/engine/test_execution_limits.py index 1c2d357e..901ca031 100644 --- a/tests/engine/test_execution_limits.py +++ b/tests/engine/test_execution_limits.py @@ -28,6 +28,7 @@ ToolSpec, ) from agent_engine.engine.langgraph.engine import LangGraphEngine +from agent_engine.runtime.execution_limiter import ITERATION_LIMIT_ANSWER _MODEL = ModelConfig(provider="fake", name="fake", temperature=None) @@ -185,6 +186,109 @@ async def test_max_iterations_enforced(tmp_path: Path) -> None: model = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"}) result = await _run_single_agent(tmp_path, policy, model) assert len(result.used_tools) == 3 # 3 iterations executed, then the loop stopped + # The wrap-up turn still asks for a tool, so the run ends on the controlled + # answer instead of the tool request's empty text. + assert result.answer == ITERATION_LIMIT_ANSWER + + +class WrapUpModel(ToolThenAnswerModel): + """Calls tools until a call is blocked, then answers from what it has. + + Records the messages of its last turn so tests can inspect the wrap-up input.""" + + def __init__(self) -> None: + super().__init__(1000, lambda i: {"message": f"m{i}"}, ["echo"]) + self.last_messages: list[Any] = [] + + def bind_tools(self, tools: list[Any]) -> WrapUpModel: + return self + + def _respond(self, messages: list[Any]) -> AIMessage: + self.last_messages = list(messages) + last = messages[-1] + if isinstance(last, ToolMessage) and str(last.content).startswith("Call blocked"): + return AIMessage(content="partial summary") + return super()._respond(messages) + + +_ITERATION_POLICY = ExecutionPolicy( + max_iterations=3, + max_tool_calls=999, + max_tool_calls_per_agent=999, + max_child_agent_calls=8, + allow_duplicate_tool_calls=True, +) + + +async def test_max_iterations_lets_the_model_wrap_up(tmp_path: Path) -> None: + model = WrapUpModel() + result = await _run_single_agent(tmp_path, _ITERATION_POLICY, model) + + assert len(result.used_tools) == 3 # the blocked call never executed + assert result.answer == "partial summary" + blocked = [m for m in model.last_messages if isinstance(m, ToolMessage)][-1] + assert blocked.tool_call_id == "c3" # the unexecuted call still gets a result + assert "max_iterations" in str(blocked.content) + + +async def test_max_iterations_answer_is_streamed_as_final(tmp_path: Path) -> None: + _write_tool(tmp_path, "echo") + spec = _system( + GraphNode(node=_agent_spec("agent", tools=(ToolSpec("echo", "echo a message"),))), + _ITERATION_POLICY, + ) + model = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"}) + async with LangGraphEngine(tmp_path, model_factory=_factory(cast(BaseChatModel, model))) as eng: + await eng.build(spec) + events = [event async for event in eng.stream("go")] + + final = [event for event in events if event.type == "final"] + assert len(final) == 1 + assert final[0].content == ITERATION_LIMIT_ANSWER + + +class ReportChildModel: + """Orchestrator that delegates once, then repeats exactly what the child said.""" + + def bind_tools(self, tools: list[Any]) -> ReportChildModel: + return self + + async def ainvoke(self, messages: list[Any]) -> AIMessage: + results = [m for m in messages if isinstance(m, ToolMessage)] + if not results: + return AIMessage( + content="", + tool_calls=[ToolCall(name="child", args={"message": "work"}, id="o1")], + ) + return AIMessage(content=f"child said: {results[-1].content}") + + +async def test_child_hitting_max_iterations_reports_to_orchestrator(tmp_path: Path) -> None: + _write_tool(tmp_path, "echo") + orch_model = ModelConfig(provider="fake", name="orch", temperature=None) + graph = GraphNode( + node=OrchestratorSpec( + id="root", + name="root", + description="router", + model=orch_model, + prompts=OrchestratorPromptSet(orchestrator="route"), + ), + children=( + GraphNode(node=_agent_spec("child", tools=(ToolSpec("echo", "echo a message"),))), + ), + ) + orchestrator = ReportChildModel() + child = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"}) + + def factory(provider: str, name: str, temperature: float | None) -> BaseChatModel: + return cast(BaseChatModel, orchestrator if name == "orch" else child) + + async with LangGraphEngine(tmp_path, model_factory=factory) as eng: + await eng.build(_system(graph, _ITERATION_POLICY)) + result = await eng.run("go") + + assert result.answer == f"child said: {ITERATION_LIMIT_ANSWER}" async def test_max_child_agent_calls_enforced(tmp_path: Path) -> None: