Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions docs/EXECUTION_LIMITS.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,12 @@ Limits degrade **gracefully** — a run never crashes with an obscure exception:
The model receives a short controlled message (e.g. *"Tool call blocked: the
'max_tool_calls' execution limit (10) was reached… finish with the information
you have."*) and naturally wraps up.
- **`max_iterations`:** the node's tool loop **stops** and returns the latest
model response. No further tools are called.
- **`max_iterations`:** the node's tool loop **stops**. The tool calls the model
requested last are **not executed**; each receives the same controlled message,
and the model gets one final turn to answer with the information it has. No
further tools are called. If that turn still produces no text, the node answers
*"I couldn't finish this request within the allowed number of steps."* — a
capped run never ends with an empty answer.

When a limit is hit the engine logs **safe metadata only** at `WARNING`:
`run_id`, `node_id`/`agent_id` (when available), the limit name, the current
Expand Down
2 changes: 1 addition & 1 deletion docs/execution-limits.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,6 @@ Limits are validated at parse time — invalid values (non-integers, wrong types
The run **degrades gracefully** — no exception, no crash.

- **Tool/child-agent limits and duplicates** — the call is blocked. The model receives a short message like *"Tool call limit reached, finish with what you have."* and naturally wraps up.
- **`max_iterations`** — the node's loop stops and returns the latest model response.
- **`max_iterations`** — the node's loop stops. Its pending tool calls are blocked, and the model gets one final turn to answer with what it has. If that turn has no text, the node answers *"I couldn't finish this request within the allowed number of steps."* instead of returning an empty answer.

The engine logs a warning with the run id, limit name, and current count. It never logs prompts, tool arguments, or secrets.
32 changes: 31 additions & 1 deletion src/agent_engine/engine/langgraph/execution/model_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
)
from agent_engine.engine.langgraph.execution.model_context import ModelContext
from agent_engine.runtime.execution_limiter import (
ITERATION_LIMIT_ANSWER,
ExecutionLimitExceeded,
blocked_message,
current_execution,
log_limit,
)
Expand Down Expand Up @@ -102,7 +104,7 @@ async def next_response() -> Any:
limiter.register_iteration(node_path)
except ExecutionLimitExceeded as exc:
log_limit(exc)
break
return await _wrap_up(response, exc, context, next_response)
context.append(response)
for tool_call in response.tool_calls:
logger.debug(
Expand Down Expand Up @@ -136,6 +138,34 @@ async def next_response() -> Any:
return response


async def _wrap_up(
response: Any,
exc: ExecutionLimitExceeded,
context: ModelContext,
next_response: Callable[[], Awaitable[Any]],
) -> Any:
"""Give the model one turn to answer after ``max_iterations`` is reached.

The pending tool calls are answered with the same controlled message the
other limits use, without executing them, so the model can finish with what
it has. A wrap-up turn that still produces no text yields a fixed answer
instead of an empty one.
"""
context.append(response)
for tool_call in response.tool_calls:
context.append(
ToolMessage(
content=blocked_message(exc),
tool_call_id=tool_call["id"],
name=tool_call["name"],
)
)
final = await next_response()
if as_text(final.content).strip():
return final
return AIMessage(content=ITERATION_LIMIT_ANSWER)


async def _refresh(
context: ModelContext,
refresh_execution_context: ExecutionContextRefresher | None,
Expand Down
4 changes: 4 additions & 0 deletions src/agent_engine/runtime/execution_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@

logger = logging.getLogger(__name__)

#: Answer returned when a node exhausts ``max_iterations`` and its wrap-up model
#: turn still produces no text, so a capped run never ends with a silent answer.
ITERATION_LIMIT_ANSWER = "I couldn't finish this request within the allowed number of steps."


class ExecutionLimiter:
"""Enforces an :class:`ExecutionPolicy` for the duration of one run."""
Expand Down
104 changes: 104 additions & 0 deletions tests/engine/test_execution_limits.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
ToolSpec,
)
from agent_engine.engine.langgraph.engine import LangGraphEngine
from agent_engine.runtime.execution_limiter import ITERATION_LIMIT_ANSWER

_MODEL = ModelConfig(provider="fake", name="fake", temperature=None)

Expand Down Expand Up @@ -185,6 +186,109 @@ async def test_max_iterations_enforced(tmp_path: Path) -> None:
model = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"})
result = await _run_single_agent(tmp_path, policy, model)
assert len(result.used_tools) == 3 # 3 iterations executed, then the loop stopped
# The wrap-up turn still asks for a tool, so the run ends on the controlled
# answer instead of the tool request's empty text.
assert result.answer == ITERATION_LIMIT_ANSWER


class WrapUpModel(ToolThenAnswerModel):
"""Calls tools until a call is blocked, then answers from what it has.
Records the messages of its last turn so tests can inspect the wrap-up input."""

def __init__(self) -> None:
super().__init__(1000, lambda i: {"message": f"m{i}"}, ["echo"])
self.last_messages: list[Any] = []

def bind_tools(self, tools: list[Any]) -> WrapUpModel:
return self

def _respond(self, messages: list[Any]) -> AIMessage:
self.last_messages = list(messages)
last = messages[-1]
if isinstance(last, ToolMessage) and str(last.content).startswith("Call blocked"):
return AIMessage(content="partial summary")
return super()._respond(messages)


_ITERATION_POLICY = ExecutionPolicy(
max_iterations=3,
max_tool_calls=999,
max_tool_calls_per_agent=999,
max_child_agent_calls=8,
allow_duplicate_tool_calls=True,
)


async def test_max_iterations_lets_the_model_wrap_up(tmp_path: Path) -> None:
model = WrapUpModel()
result = await _run_single_agent(tmp_path, _ITERATION_POLICY, model)

assert len(result.used_tools) == 3 # the blocked call never executed
assert result.answer == "partial summary"
blocked = [m for m in model.last_messages if isinstance(m, ToolMessage)][-1]
assert blocked.tool_call_id == "c3" # the unexecuted call still gets a result
assert "max_iterations" in str(blocked.content)


async def test_max_iterations_answer_is_streamed_as_final(tmp_path: Path) -> None:
_write_tool(tmp_path, "echo")
spec = _system(
GraphNode(node=_agent_spec("agent", tools=(ToolSpec("echo", "echo a message"),))),
_ITERATION_POLICY,
)
model = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"})
async with LangGraphEngine(tmp_path, model_factory=_factory(cast(BaseChatModel, model))) as eng:
await eng.build(spec)
events = [event async for event in eng.stream("go")]

final = [event for event in events if event.type == "final"]
assert len(final) == 1
assert final[0].content == ITERATION_LIMIT_ANSWER


class ReportChildModel:
"""Orchestrator that delegates once, then repeats exactly what the child said."""

def bind_tools(self, tools: list[Any]) -> ReportChildModel:
return self

async def ainvoke(self, messages: list[Any]) -> AIMessage:
results = [m for m in messages if isinstance(m, ToolMessage)]
if not results:
return AIMessage(
content="",
tool_calls=[ToolCall(name="child", args={"message": "work"}, id="o1")],
)
return AIMessage(content=f"child said: {results[-1].content}")


async def test_child_hitting_max_iterations_reports_to_orchestrator(tmp_path: Path) -> None:
_write_tool(tmp_path, "echo")
orch_model = ModelConfig(provider="fake", name="orch", temperature=None)
graph = GraphNode(
node=OrchestratorSpec(
id="root",
name="root",
description="router",
model=orch_model,
prompts=OrchestratorPromptSet(orchestrator="route"),
),
children=(
GraphNode(node=_agent_spec("child", tools=(ToolSpec("echo", "echo a message"),))),
),
)
orchestrator = ReportChildModel()
child = ToolThenAnswerModel(1000, lambda i: {"message": f"m{i}"})

def factory(provider: str, name: str, temperature: float | None) -> BaseChatModel:
return cast(BaseChatModel, orchestrator if name == "orch" else child)

async with LangGraphEngine(tmp_path, model_factory=factory) as eng:
await eng.build(_system(graph, _ITERATION_POLICY))
result = await eng.run("go")

assert result.answer == f"child said: {ITERATION_LIMIT_ANSWER}"


async def test_max_child_agent_calls_enforced(tmp_path: Path) -> None:
Expand Down
Loading