From 93d5abf26afb92b4ff3bd1b5b53560194091da59 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 14:07:19 -0700 Subject: [PATCH 01/29] Add fixtures for detecting if langchain, langgraph and deepagents are installed Signed-off-by: David Gardner --- python/tests/conftest.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index cc4b9794..92250a16 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -7,6 +7,7 @@ import typing from collections.abc import Iterator +import types from uuid import uuid4 import pytest @@ -28,3 +29,37 @@ def event_recorder(event: nemo_flow.Event) -> None: nemo_flow.subscribers.register(subscriber_name, event_recorder) yield events nemo_flow.subscribers.deregister(subscriber_name) + +@pytest.fixture(name="integration_langchain", scope='session') +def integration_langchain_fixture() -> types.ModuleType: + """ + Use for integration tests that require LangChain to be installed. + """ + try: + import langchain + return langchain + except ImportError: + pytest.skip(reason="langchain must be installed to run LangChain based tests") + +@pytest.fixture(name="integration_langgraph", scope='session') +def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: + """ + Use for integration tests that require LangGraph to be installed. + """ + try: + import langgraph + return langgraph + except ImportError: + pytest.skip(reason="langgraph must be installed to run LangGraph based tests") + + +@pytest.fixture(name="integration_deepagents", scope='session') +def integration_deepagents_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: + """ + Use for integration tests that require Deep Agents to be installed. + """ + try: + import deepagents + return deepagents + except ImportError: + pytest.skip(reason="deepagents must be installed to run Deep Agents based tests") From fcb88801d4c99f9ce11af40ca7e06a2d2b079a70 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 14:09:32 -0700 Subject: [PATCH 02/29] Move fixtures Signed-off-by: David Gardner --- python/tests/conftest.py | 34 ----------------------- python/tests/integrations/conftest.py | 40 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 python/tests/integrations/conftest.py diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 92250a16..922a5a6f 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -29,37 +29,3 @@ def event_recorder(event: nemo_flow.Event) -> None: nemo_flow.subscribers.register(subscriber_name, event_recorder) yield events nemo_flow.subscribers.deregister(subscriber_name) - -@pytest.fixture(name="integration_langchain", scope='session') -def integration_langchain_fixture() -> types.ModuleType: - """ - Use for integration tests that require LangChain to be installed. - """ - try: - import langchain - return langchain - except ImportError: - pytest.skip(reason="langchain must be installed to run LangChain based tests") - -@pytest.fixture(name="integration_langgraph", scope='session') -def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: - """ - Use for integration tests that require LangGraph to be installed. - """ - try: - import langgraph - return langgraph - except ImportError: - pytest.skip(reason="langgraph must be installed to run LangGraph based tests") - - -@pytest.fixture(name="integration_deepagents", scope='session') -def integration_deepagents_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: - """ - Use for integration tests that require Deep Agents to be installed. - """ - try: - import deepagents - return deepagents - except ImportError: - pytest.skip(reason="deepagents must be installed to run Deep Agents based tests") diff --git a/python/tests/integrations/conftest.py b/python/tests/integrations/conftest.py new file mode 100644 index 00000000..2094126d --- /dev/null +++ b/python/tests/integrations/conftest.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import types + +import pytest + +@pytest.fixture(name="integration_langchain", scope='session') +def integration_langchain_fixture() -> types.ModuleType: + """ + Use for integration tests that require LangChain to be installed. + """ + try: + import langchain + return langchain + except ImportError: + pytest.skip(reason="langchain must be installed to run LangChain based tests") + +@pytest.fixture(name="integration_langgraph", scope='session') +def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: + """ + Use for integration tests that require LangGraph to be installed. + """ + try: + import langgraph + return langgraph + except ImportError: + pytest.skip(reason="langgraph must be installed to run LangGraph based tests") + + +@pytest.fixture(name="integration_deepagents", scope='session') +def integration_deepagents_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: + """ + Use for integration tests that require Deep Agents to be installed. + """ + try: + import deepagents + return deepagents + except ImportError: + pytest.skip(reason="deepagents must be installed to run Deep Agents based tests") From 8a2021bf557949ac53ab2ad539a0a55d20174545 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 15:24:02 -0700 Subject: [PATCH 03/29] Rename 'langchain' dir to prevent 'import langchain' from accidentally suceeding Signed-off-by: David Gardner --- python/tests/integrations/conftest.py | 8 ++-- .../tests/integrations/deepagents/conftest.py | 13 +++++++ .../integrations/langchain_tests/conftest.py | 13 +++++++ .../test_callbacks.py | 38 ++++++++++++------- .../test_middleware.py | 0 .../tests/integrations/langgraph/conftest.py | 13 +++++++ 6 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 python/tests/integrations/deepagents/conftest.py create mode 100644 python/tests/integrations/langchain_tests/conftest.py rename python/tests/integrations/{langchain => langchain_tests}/test_callbacks.py (84%) rename python/tests/integrations/{langchain => langchain_tests}/test_middleware.py (100%) create mode 100644 python/tests/integrations/langgraph/conftest.py diff --git a/python/tests/integrations/conftest.py b/python/tests/integrations/conftest.py index 2094126d..e14da885 100644 --- a/python/tests/integrations/conftest.py +++ b/python/tests/integrations/conftest.py @@ -5,6 +5,7 @@ import pytest + @pytest.fixture(name="integration_langchain", scope='session') def integration_langchain_fixture() -> types.ModuleType: """ @@ -13,9 +14,10 @@ def integration_langchain_fixture() -> types.ModuleType: try: import langchain return langchain - except ImportError: + except Exception: pytest.skip(reason="langchain must be installed to run LangChain based tests") + @pytest.fixture(name="integration_langgraph", scope='session') def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: """ @@ -24,7 +26,7 @@ def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> ty try: import langgraph return langgraph - except ImportError: + except Exception: pytest.skip(reason="langgraph must be installed to run LangGraph based tests") @@ -36,5 +38,5 @@ def integration_deepagents_fixture(integration_langgraph: types.ModuleType) -> t try: import deepagents return deepagents - except ImportError: + except Exception: pytest.skip(reason="deepagents must be installed to run Deep Agents based tests") diff --git a/python/tests/integrations/deepagents/conftest.py b/python/tests/integrations/deepagents/conftest.py new file mode 100644 index 00000000..a50e6e5d --- /dev/null +++ b/python/tests/integrations/deepagents/conftest.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import types + +import pytest + +@pytest.fixture(name="integration_deepagents", scope='session', autouse=True) +def integration_deepagents_fixture(integration_deepagents: types.ModuleType) -> types.ModuleType: + """ + Override the integration_deepagents fixture to make it autouse + """ + yield integration_deepagents diff --git a/python/tests/integrations/langchain_tests/conftest.py b/python/tests/integrations/langchain_tests/conftest.py new file mode 100644 index 00000000..eb4f7d35 --- /dev/null +++ b/python/tests/integrations/langchain_tests/conftest.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import types + +import pytest + +@pytest.fixture(name="integration_langchain", scope='session', autouse=True) +def integration_langchain_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: + """ + Override the integration_langchain fixture to make it autouse + """ + yield integration_langchain diff --git a/python/tests/integrations/langchain/test_callbacks.py b/python/tests/integrations/langchain_tests/test_callbacks.py similarity index 84% rename from python/tests/integrations/langchain/test_callbacks.py rename to python/tests/integrations/langchain_tests/test_callbacks.py index 60700e1e..eebf04bf 100644 --- a/python/tests/integrations/langchain/test_callbacks.py +++ b/python/tests/integrations/langchain_tests/test_callbacks.py @@ -5,26 +5,26 @@ from __future__ import annotations -from types import SimpleNamespace +import types +import typing from unittest.mock import MagicMock from uuid import uuid4 import pytest -from langchain_core.messages import ToolMessage -from langgraph.types import Command -from nemo_flow.integrations.langchain import callbacks as callbacks_module -from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler + +if typing.TYPE_CHECKING: + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler def _make_mock_nemo_flow() -> MagicMock: """Build a minimal mock of the ``nemo_flow`` module.""" mock_nemo_flow = MagicMock(name="nemo_flow") - mock_nemo_flow.ScopeType = SimpleNamespace(Agent="Agent") + mock_nemo_flow.ScopeType = types.SimpleNamespace(Agent="Agent") - scope = SimpleNamespace() + scope = types.SimpleNamespace() scope.push = MagicMock( - side_effect=lambda name, scope_type, **kwargs: SimpleNamespace( + side_effect=lambda name, scope_type, **kwargs: types.SimpleNamespace( uuid=str(uuid4()), name=name, scope_type=scope_type, @@ -35,9 +35,14 @@ def _make_mock_nemo_flow() -> MagicMock: mock_nemo_flow.scope = scope return mock_nemo_flow +@pytest.fixture(name="callbacks_module", scope="session") +def callbacks_module_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: + """Fixture to provide the callbacks module.""" + import nemo_flow.integrations.langchain.callbacks as callbacks_module + return callbacks_module @pytest.fixture() -def mock_nemo_flow(monkeypatch: pytest.MonkeyPatch) -> MagicMock: +def mock_nemo_flow(monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType) -> MagicMock: mock_nemo_flow = _make_mock_nemo_flow() monkeypatch.setattr(callbacks_module, "nemo_flow", mock_nemo_flow) return mock_nemo_flow @@ -45,6 +50,7 @@ def mock_nemo_flow(monkeypatch: pytest.MonkeyPatch) -> MagicMock: @pytest.fixture() def handler(mock_nemo_flow: MagicMock) -> NemoFlowCallbackHandler: + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler return NemoFlowCallbackHandler() @@ -121,6 +127,8 @@ def test_on_chain_error_pops_scope(self, handler: NemoFlowCallbackHandler, mock_ assert run_id not in handler._scope_handles def test_on_chain_end_prepares_command_outputs(self, handler: NemoFlowCallbackHandler, mock_nemo_flow: MagicMock): + from langgraph.types import Command + from langchain_core.messages import ToolMessage run_id = uuid4() handler.on_chain_start( {"name": "MyChain"}, @@ -216,20 +224,23 @@ def test_name_fallback_to_id(self, handler: NemoFlowCallbackHandler, mock_nemo_f class TestGracefulNoOp: """Verify callbacks are silent if the module-level runtime is unavailable.""" - def test_no_nemo_flow_on_chain_start(self, monkeypatch: pytest.MonkeyPatch): + def test_no_nemo_flow_on_chain_start(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler handler = NemoFlowCallbackHandler() handler.on_chain_start({"name": "x"}, {}, run_id=uuid4()) - def test_no_nemo_flow_on_chain_end(self, monkeypatch: pytest.MonkeyPatch): + def test_no_nemo_flow_on_chain_end(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler handler = NemoFlowCallbackHandler() handler.on_chain_end({}, run_id=uuid4()) - def test_no_nemo_flow_on_chain_error(self, monkeypatch: pytest.MonkeyPatch): + def test_no_nemo_flow_on_chain_error(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler handler = NemoFlowCallbackHandler() handler.on_chain_error(RuntimeError("e"), run_id=uuid4()) @@ -238,9 +249,8 @@ def test_no_nemo_flow_on_chain_error(self, monkeypatch: pytest.MonkeyPatch): class TestErrorSwallowing: """Ensure NeMo Flow errors never propagate.""" - def test_scope_push_error_swallowed(self, mock_nemo_flow: MagicMock): + def test_scope_push_error_swallowed(self, handler: NemoFlowCallbackHandler, mock_nemo_flow: MagicMock): mock_nemo_flow.scope.push.side_effect = RuntimeError("nemo flow failure") - handler = NemoFlowCallbackHandler() handler.on_chain_start({"name": "x"}, {}, run_id=uuid4()) diff --git a/python/tests/integrations/langchain/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py similarity index 100% rename from python/tests/integrations/langchain/test_middleware.py rename to python/tests/integrations/langchain_tests/test_middleware.py diff --git a/python/tests/integrations/langgraph/conftest.py b/python/tests/integrations/langgraph/conftest.py new file mode 100644 index 00000000..678734fa --- /dev/null +++ b/python/tests/integrations/langgraph/conftest.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import types + +import pytest + +@pytest.fixture(name="integration_langgraph", scope='session', autouse=True) +def integration_langgraph_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: + """ + Override the integration_langgraph fixture to make it autouse + """ + yield integration_langgraph From 231d1508bd0dec45f87b047dd1ded1d5f7a97157 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 15:25:10 -0700 Subject: [PATCH 04/29] Remove unneeded fixture dependency Signed-off-by: David Gardner --- python/tests/integrations/langchain_tests/test_callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integrations/langchain_tests/test_callbacks.py b/python/tests/integrations/langchain_tests/test_callbacks.py index eebf04bf..168215df 100644 --- a/python/tests/integrations/langchain_tests/test_callbacks.py +++ b/python/tests/integrations/langchain_tests/test_callbacks.py @@ -36,7 +36,7 @@ def _make_mock_nemo_flow() -> MagicMock: return mock_nemo_flow @pytest.fixture(name="callbacks_module", scope="session") -def callbacks_module_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: +def callbacks_module_fixture() -> types.ModuleType: """Fixture to provide the callbacks module.""" import nemo_flow.integrations.langchain.callbacks as callbacks_module return callbacks_module From d2ceaedf1ec4ce0018ad5866d71e1a5b0515cd8b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 16:24:35 -0700 Subject: [PATCH 05/29] Make langchain tests skippable when not installed Signed-off-by: David Gardner --- .../langchain_tests/test_middleware.py | 236 ++++++++++-------- 1 file changed, 134 insertions(+), 102 deletions(-) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index b5b1a18f..a52d6b98 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -6,25 +6,77 @@ from __future__ import annotations import asyncio -from typing import Any +from collections.abc import Awaitable, Callable +from typing import Any, TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock import pytest -from langchain.agents import create_agent -from langchain.agents.middleware import ModelRequest, ModelResponse, ToolCallRequest -from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from langchain_core.tools import tool import nemo_flow from nemo_flow.codecs import AnthropicMessagesCodec, OpenAIChatCodec, OpenAIResponsesCodec -from nemo_flow.integrations.langchain import _serialization -from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware + +if TYPE_CHECKING: + from langchain_core.messages import AIMessage + from langchain_core.messages import ToolMessage + from langchain.agents.middleware import ToolCallRequest + from langchain.agents.middleware import ModelRequest, ModelResponse + + from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware _DEFAULT_MOCK_RESPONSE_MSG = "nemo_flow unittest result" +@pytest.fixture(name="model_request_handler") +def model_request_handler_fixture() -> tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]]: + from langchain_core.messages import AIMessage + from langchain.agents.middleware import ModelResponse + + seen_request: dict[str, ModelRequest[Any]] = {} + + def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: + seen_request["request"] = request + return ModelResponse(result=[AIMessage(content="done")]) + + return handler, seen_request + +@pytest.fixture(name="async_model_request_handler") +def async_model_request_handler_fixture(model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]]) -> tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]]: + (sync_handler, seen_request) = model_request_handler + async def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: + return sync_handler(request) + + return handler, seen_request + +@pytest.fixture(name="tool_request_handler") +def tool_request_handler_fixture() -> tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]: + from langchain_core.messages import ToolMessage + seen_request: dict[str, ToolCallRequest] = {} + + def handler(request: ToolCallRequest) -> ToolMessage: + seen_request["request"] = ToolCallRequest + return ToolMessage(content="done", tool_call_id=request.tool_call["id"]) + + return handler, seen_request + +@pytest.fixture(name="async_tool_request_handler") +def async_tool_request_handler_fixture(tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]) -> tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]: + (sync_handler, seen_request) = tool_request_handler + async def handler(request: ToolCallRequest) -> ToolMessage: + return sync_handler(request) + + return handler, seen_request + +@pytest.fixture(name="mock_tool_execute") +def mock_tool_execute_fixture() -> AsyncMock: + async def execute_side_effect(*, func: Any, **kwargs: Any) -> ToolMessage: + return func({"query": "intercepted"}) + + return AsyncMock(side_effect=execute_side_effect) + def _mk_mock_model(returned_message: str | list[AIMessage] = _DEFAULT_MOCK_RESPONSE_MSG) -> MagicMock: + from langchain_core.language_models import BaseChatModel + from langchain_core.messages import AIMessage + mock_model = MagicMock(spec=BaseChatModel) mock_model.bind.return_value = mock_model mock_model.bind_tools.return_value = mock_model @@ -40,39 +92,50 @@ def _mk_mock_model(returned_message: str | list[AIMessage] = _DEFAULT_MOCK_RESPO return mock_model +@pytest.fixture(name="nemo_flow_middleware") +def nemo_flow_middleware_fixture() -> NemoFlowMiddleware: + from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware + return NemoFlowMiddleware() + +@pytest.fixture(name="recording_middleware") +def recording_middleware_fixture() -> NemoFlowMiddleware: + from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware + class RecordingMiddleware(NemoFlowMiddleware): + def __init__(self) -> None: + super().__init__() + self.calls: list[dict[str, Any]] = [] + + async def _llm_execute( + self, + model_name: str, + request: nemo_flow.LLMRequest, + codec: Any, + response_codec: Any, + func: Any, + ) -> Any: + self.calls.append( + { + "model_name": model_name, + "request": request, + "codec": codec, + "response_codec": response_codec, + } + ) + intercepted = nemo_flow.LLMRequest( + request.headers, + { + **request.content, + "model_settings": {"temperature": 0.25}, + }, + ) + return await func(intercepted) -class RecordingMiddleware(NemoFlowMiddleware): - def __init__(self) -> None: - super().__init__() - self.calls: list[dict[str, Any]] = [] - - async def _llm_execute( - self, - model_name: str, - request: nemo_flow.LLMRequest, - codec: Any, - response_codec: Any, - func: Any, - ) -> Any: - self.calls.append( - { - "model_name": model_name, - "request": request, - "codec": codec, - "response_codec": response_codec, - } - ) - intercepted = nemo_flow.LLMRequest( - request.headers, - { - **request.content, - "model_settings": {"temperature": 0.25}, - }, - ) - return await func(intercepted) - + return RecordingMiddleware() def _model_request() -> ModelRequest[Any]: + from langchain.agents.middleware import ModelRequest + from langchain_core.messages import HumanMessage + mock_model = _mk_mock_model() return ModelRequest( @@ -83,6 +146,7 @@ def _model_request() -> ModelRequest[Any]: def _tool_call_request() -> ToolCallRequest: + from langchain.agents.middleware import ToolCallRequest return ToolCallRequest( tool_call={"name": "lookup", "args": {"query": "original"}, "id": "call-1"}, tool=None, @@ -91,69 +155,44 @@ def _tool_call_request() -> ToolCallRequest: ) -def test_wrap_model_call_routes_through_llm_execute() -> None: - middleware = RecordingMiddleware() - seen_request: ModelRequest[Any] | None = None +def test_wrap_model_call_routes_through_llm_execute(model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): + + (handler, seen_request) = model_request_handler - def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: - nonlocal seen_request - seen_request = request - return ModelResponse(result=[AIMessage(content="done")]) - - response = middleware.wrap_model_call(_model_request(), handler) + response = recording_middleware.wrap_model_call(_model_request(), handler) assert response.result[0].content == "done" - assert seen_request is not None - assert seen_request.model_settings == {"temperature": 0.25} - assert middleware.calls[0]["model_name"] == "mock-model" - assert middleware.calls[0]["request"].content["model"] == "mock-model" - assert middleware.calls[0]["codec"] is None - assert middleware.calls[0]["response_codec"] is None - + assert seen_request["request"].model_settings == {"temperature": 0.25} + assert recording_middleware.calls[0]["model_name"] == "mock-model" + assert recording_middleware.calls[0]["request"].content["model"] == "mock-model" + assert recording_middleware.calls[0]["codec"] is None + assert recording_middleware.calls[0]["response_codec"] is None -def test_awrap_model_call_routes_through_llm_execute() -> None: - middleware = RecordingMiddleware() - seen_request: ModelRequest[Any] | None = None - async def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: - nonlocal seen_request - seen_request = request - return ModelResponse(result=[AIMessage(content="done")]) +def test_awrap_model_call_routes_through_llm_execute(async_model_request_handler: tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): + (handler, seen_request) = async_model_request_handler - response = asyncio.run(middleware.awrap_model_call(_model_request(), handler)) + response = asyncio.run(recording_middleware.awrap_model_call(_model_request(), handler)) assert response.result[0].content == "done" - assert seen_request is not None - assert seen_request.model_settings == {"temperature": 0.25} - assert middleware.calls[0]["model_name"] == "mock-model" - assert middleware.calls[0]["request"].content["model"] == "mock-model" - assert middleware.calls[0]["codec"] is None - assert middleware.calls[0]["response_codec"] is None + assert seen_request["request"].model_settings == {"temperature": 0.25} + assert recording_middleware.calls[0]["model_name"] == "mock-model" + assert recording_middleware.calls[0]["request"].content["model"] == "mock-model" + assert recording_middleware.calls[0]["codec"] is None + assert recording_middleware.calls[0]["response_codec"] is None -def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch) -> None: - middleware = NemoFlowMiddleware() +def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): + (handler, seen_request) = tool_request_handler parent_handle = MagicMock() - seen_request: ToolCallRequest | None = None - - async def execute_side_effect(*, func: Any, **kwargs: Any) -> ToolMessage: - return func({"query": "intercepted"}) - - mock_tool_execute = AsyncMock(side_effect=execute_side_effect) - - def handler(request: ToolCallRequest) -> ToolMessage: - nonlocal seen_request - seen_request = request - return ToolMessage(content="done", tool_call_id=request.tool_call["id"]) monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = middleware.wrap_tool_call(_tool_call_request(), handler) + response = nemo_flow_middleware.wrap_tool_call(_tool_call_request(), handler) assert response.content == "done" - assert seen_request is not None - assert seen_request.tool_call["args"] == {"query": "intercepted"} + assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} mock_tool_execute.assert_awaited_once() kwargs = mock_tool_execute.await_args.kwargs assert kwargs["name"] == "lookup" @@ -163,29 +202,17 @@ def handler(request: ToolCallRequest) -> ToolMessage: assert isinstance(kwargs["result_codec"], nemo_flow.typed.BestEffortAnyCodec) -def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch) -> None: - middleware = NemoFlowMiddleware() +def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): parent_handle = MagicMock() - seen_request: ToolCallRequest | None = None - - async def execute_side_effect(*, func: Any, **kwargs: Any) -> ToolMessage: - return await func({"query": "intercepted"}) - - mock_tool_execute = AsyncMock(side_effect=execute_side_effect) - - async def handler(request: ToolCallRequest) -> ToolMessage: - nonlocal seen_request - seen_request = request - return ToolMessage(content="done", tool_call_id=request.tool_call["id"]) + (handler, seen_request) = async_tool_request_handler monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = asyncio.run(middleware.awrap_tool_call(_tool_call_request(), handler)) + response = asyncio.run(nemo_flow_middleware.awrap_tool_call(_tool_call_request(), handler)) assert response.content == "done" - assert seen_request is not None - assert seen_request.tool_call["args"] == {"query": "intercepted"} + assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} mock_tool_execute.assert_awaited_once() kwargs = mock_tool_execute.await_args.kwargs assert kwargs["name"] == "lookup" @@ -196,6 +223,7 @@ async def handler(request: ToolCallRequest) -> ToolMessage: def test_infer_codec_from_supported_model_classes(monkeypatch: pytest.MonkeyPatch) -> None: + from nemo_flow.integrations.langchain import _serialization class FakeChatAnthropic: pass @@ -224,8 +252,12 @@ class FakeChatNVIDIA: @pytest.mark.parametrize("use_async", [False, True]) -def test_agent_integration(use_async: bool) -> None: +def test_agent_integration(use_async: bool, nemo_flow_middleware: NemoFlowMiddleware) -> None: """An integration test to verify that the middleware correctly wraps a model call end-to-end.""" + from langchain.agents import create_agent + from langchain_core.messages import AIMessage + from langchain_core.tools import tool + model_responses = [ AIMessage( content="", @@ -247,7 +279,7 @@ def get_weather(location: str) -> str: """Get the current weather for a location.""" return f"The weather in {location} is sunny and 72 degrees." - agent = create_agent(model=mock_model, tools=[get_weather], middleware=[NemoFlowMiddleware()]) + agent = create_agent(model=mock_model, tools=[get_weather], middleware=[nemo_flow_middleware]) input_payload = { "messages": [ From 1aa90871bd9560ad3784fd2fc5ed58fd9785e074 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 16:34:10 -0700 Subject: [PATCH 06/29] Instruct the agent how to write fixtures Signed-off-by: David Gardner --- .agents/skills/test-python-binding/SKILL.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.agents/skills/test-python-binding/SKILL.md b/.agents/skills/test-python-binding/SKILL.md index 330eda60..b3081130 100644 --- a/.agents/skills/test-python-binding/SKILL.md +++ b/.agents/skills/test-python-binding/SKILL.md @@ -38,6 +38,13 @@ Use this skill when the change is primarily in `python/nemo_flow`, - The name of the mocked class should be prefixed with `mock`, not `fake`. - Prefer pytest fixtures over helper methods. - Do not repeat fixtures, if a fixture is needed in multiple test files, place it in a `conftest.py` file. +- When creating a fixture follow this pattern: + ```python + @pytest.fixture(name=""[, scope=""]) + def _fixture() -> : + ... + ``` + Only specify the scope argument when the value is something other than "function". - Prefer `pytest.mark.parametrize` over creating individual tests for different input types. From b119ece89933e741beb378a9390b7cff9214a080 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 16:34:42 -0700 Subject: [PATCH 07/29] Refactor _model_request and _tool_call_request as fixtures Signed-off-by: David Gardner --- .../langchain_tests/test_middleware.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index a52d6b98..92987e56 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -101,7 +101,7 @@ def nemo_flow_middleware_fixture() -> NemoFlowMiddleware: def recording_middleware_fixture() -> NemoFlowMiddleware: from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware class RecordingMiddleware(NemoFlowMiddleware): - def __init__(self) -> None: + def __init__(self): super().__init__() self.calls: list[dict[str, Any]] = [] @@ -132,7 +132,8 @@ async def _llm_execute( return RecordingMiddleware() -def _model_request() -> ModelRequest[Any]: +@pytest.fixture(name="model_request") +def model_request_fixture() -> ModelRequest[Any]: from langchain.agents.middleware import ModelRequest from langchain_core.messages import HumanMessage @@ -145,7 +146,8 @@ def _model_request() -> ModelRequest[Any]: ) -def _tool_call_request() -> ToolCallRequest: +@pytest.fixture(name="tool_request") +def tool_request_fixture() -> ToolCallRequest: from langchain.agents.middleware import ToolCallRequest return ToolCallRequest( tool_call={"name": "lookup", "args": {"query": "original"}, "id": "call-1"}, @@ -155,11 +157,11 @@ def _tool_call_request() -> ToolCallRequest: ) -def test_wrap_model_call_routes_through_llm_execute(model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): +def test_wrap_model_call_routes_through_llm_execute(model_request: ModelRequest[Any], model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): (handler, seen_request) = model_request_handler - response = recording_middleware.wrap_model_call(_model_request(), handler) + response = recording_middleware.wrap_model_call(model_request, handler) assert response.result[0].content == "done" assert seen_request["request"].model_settings == {"temperature": 0.25} @@ -169,10 +171,10 @@ def test_wrap_model_call_routes_through_llm_execute(model_request_handler: tuple assert recording_middleware.calls[0]["response_codec"] is None -def test_awrap_model_call_routes_through_llm_execute(async_model_request_handler: tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): +def test_awrap_model_call_routes_through_llm_execute(model_request: ModelRequest[Any], async_model_request_handler: tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): (handler, seen_request) = async_model_request_handler - response = asyncio.run(recording_middleware.awrap_model_call(_model_request(), handler)) + response = asyncio.run(recording_middleware.awrap_model_call(model_request, handler)) assert response.result[0].content == "done" assert seen_request["request"].model_settings == {"temperature": 0.25} @@ -182,14 +184,14 @@ def test_awrap_model_call_routes_through_llm_execute(async_model_request_handler assert recording_middleware.calls[0]["response_codec"] is None -def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): +def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request: ToolCallRequest, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): (handler, seen_request) = tool_request_handler parent_handle = MagicMock() monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = nemo_flow_middleware.wrap_tool_call(_tool_call_request(), handler) + response = nemo_flow_middleware.wrap_tool_call(tool_request, handler) assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} @@ -202,14 +204,14 @@ def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPa assert isinstance(kwargs["result_codec"], nemo_flow.typed.BestEffortAnyCodec) -def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): +def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request: ToolCallRequest, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): parent_handle = MagicMock() (handler, seen_request) = async_tool_request_handler monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = asyncio.run(nemo_flow_middleware.awrap_tool_call(_tool_call_request(), handler)) + response = asyncio.run(nemo_flow_middleware.awrap_tool_call(tool_request, handler)) assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} @@ -222,13 +224,13 @@ def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyP assert isinstance(kwargs["result_codec"], nemo_flow.typed.BestEffortAnyCodec) -def test_infer_codec_from_supported_model_classes(monkeypatch: pytest.MonkeyPatch) -> None: +def test_infer_codec_from_supported_model_classes(monkeypatch: pytest.MonkeyPatch): from nemo_flow.integrations.langchain import _serialization class FakeChatAnthropic: pass class FakeChatOpenAI: - def __init__(self, *, use_responses_api: bool = False) -> None: + def __init__(self, *, use_responses_api: bool = False): self.use_responses_api = use_responses_api class FakeChatNVIDIA: @@ -252,7 +254,7 @@ class FakeChatNVIDIA: @pytest.mark.parametrize("use_async", [False, True]) -def test_agent_integration(use_async: bool, nemo_flow_middleware: NemoFlowMiddleware) -> None: +def test_agent_integration(use_async: bool, nemo_flow_middleware: NemoFlowMiddleware): """An integration test to verify that the middleware correctly wraps a model call end-to-end.""" from langchain.agents import create_agent from langchain_core.messages import AIMessage @@ -302,7 +304,7 @@ def get_weather(location: str) -> str: "scope.end.langchain-request", ] - def event_recorder(event) -> None: + def event_recorder(event): events.append(f"{event.kind}.{event.scope_category}.{event.name}") nemo_flow.subscribers.register("event_recorder", event_recorder) From f7a7b32f51c05b6962ebf9e8d95c8f0cce612f93 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 16:41:54 -0700 Subject: [PATCH 08/29] WIP Signed-off-by: David Gardner --- .../langchain_tests/test_middleware.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index 92987e56..69bfddaf 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -52,7 +52,7 @@ def tool_request_handler_fixture() -> tuple[Callable[[ToolCallRequest], ToolMess seen_request: dict[str, ToolCallRequest] = {} def handler(request: ToolCallRequest) -> ToolMessage: - seen_request["request"] = ToolCallRequest + seen_request["request"] = request return ToolMessage(content="done", tool_call_id=request.tool_call["id"]) return handler, seen_request @@ -226,30 +226,27 @@ def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyP def test_infer_codec_from_supported_model_classes(monkeypatch: pytest.MonkeyPatch): from nemo_flow.integrations.langchain import _serialization - class FakeChatAnthropic: - pass - - class FakeChatOpenAI: - def __init__(self, *, use_responses_api: bool = False): - self.use_responses_api = use_responses_api - class FakeChatNVIDIA: - pass + MockChatAnthropic = MagicMock(spec=type("MockChatAnthropic", (), {})) + MockChatOpenAI = MagicMock(spec=type("MockChatOpenAI", (), {})) + MockChatOpenAIResponses = MagicMock(spec=MockChatOpenAI.__class__) + MockChatOpenAIResponses.use_responses_api = True + MockChatNVIDIA = MagicMock(spec=type("MockChatNVIDIA", (), {})) - monkeypatch.setattr(_serialization, "ChatAnthropic", FakeChatAnthropic, raising=False) - monkeypatch.setattr(_serialization, "ChatOpenAI", FakeChatOpenAI, raising=False) - monkeypatch.setattr(_serialization, "ChatNVIDIA", FakeChatNVIDIA, raising=False) + monkeypatch.setattr(_serialization, "ChatAnthropic", MockChatAnthropic.__class__, raising=False) + monkeypatch.setattr(_serialization, "ChatOpenAI", MockChatOpenAI.__class__, raising=False) + monkeypatch.setattr(_serialization, "ChatNVIDIA", MockChatNVIDIA.__class__, raising=False) monkeypatch.setattr(_serialization, "_HAS_ANTHROPIC", True) monkeypatch.setattr(_serialization, "_HAS_OPENAI", True) monkeypatch.setattr(_serialization, "_HAS_NVIDIA", True) - assert isinstance(_serialization.infer_codec_from_model(FakeChatAnthropic()), AnthropicMessagesCodec) - assert isinstance(_serialization.infer_codec_from_model(FakeChatOpenAI()), OpenAIChatCodec) + assert isinstance(_serialization.infer_codec_from_model(MockChatAnthropic), AnthropicMessagesCodec) + assert isinstance(_serialization.infer_codec_from_model(MockChatOpenAI), OpenAIChatCodec) assert isinstance( - _serialization.infer_codec_from_model(FakeChatOpenAI(use_responses_api=True)), + _serialization.infer_codec_from_model(MockChatOpenAIResponses), OpenAIResponsesCodec, ) - assert isinstance(_serialization.infer_codec_from_model(FakeChatNVIDIA()), OpenAIChatCodec) + assert isinstance(_serialization.infer_codec_from_model(MockChatNVIDIA), OpenAIChatCodec) assert _serialization.infer_codec_from_model(object()) is None From 535040cfb529f10aeaa1f8c694abb31995b498e7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 May 2026 16:59:22 -0700 Subject: [PATCH 09/29] Rename tool_request fixture to tool_call_request [skip ci] Signed-off-by: David Gardner --- .../langchain_tests/test_middleware.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index 69bfddaf..98eb4c6d 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import inspect from collections.abc import Awaitable, Callable from typing import Any, TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock @@ -68,7 +69,10 @@ async def handler(request: ToolCallRequest) -> ToolMessage: @pytest.fixture(name="mock_tool_execute") def mock_tool_execute_fixture() -> AsyncMock: async def execute_side_effect(*, func: Any, **kwargs: Any) -> ToolMessage: - return func({"query": "intercepted"}) + result = func({"query": "intercepted"}) + if inspect.isawaitable(result): + return await result + return result return AsyncMock(side_effect=execute_side_effect) @@ -146,8 +150,8 @@ def model_request_fixture() -> ModelRequest[Any]: ) -@pytest.fixture(name="tool_request") -def tool_request_fixture() -> ToolCallRequest: +@pytest.fixture(name="tool_call_request") +def tool_call_request_fixture() -> ToolCallRequest: from langchain.agents.middleware import ToolCallRequest return ToolCallRequest( tool_call={"name": "lookup", "args": {"query": "original"}, "id": "call-1"}, @@ -184,14 +188,14 @@ def test_awrap_model_call_routes_through_llm_execute(model_request: ModelRequest assert recording_middleware.calls[0]["response_codec"] is None -def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request: ToolCallRequest, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): +def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_call_request: ToolCallRequest, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): (handler, seen_request) = tool_request_handler parent_handle = MagicMock() monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = nemo_flow_middleware.wrap_tool_call(tool_request, handler) + response = nemo_flow_middleware.wrap_tool_call(tool_call_request, handler) assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} @@ -204,14 +208,14 @@ def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPa assert isinstance(kwargs["result_codec"], nemo_flow.typed.BestEffortAnyCodec) -def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_request: ToolCallRequest, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): +def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_call_request: ToolCallRequest, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): parent_handle = MagicMock() (handler, seen_request) = async_tool_request_handler monkeypatch.setattr(nemo_flow.scope, "get_handle", lambda: parent_handle) monkeypatch.setattr(nemo_flow.typed, "tool_execute", mock_tool_execute) - response = asyncio.run(nemo_flow_middleware.awrap_tool_call(tool_request, handler)) + response = asyncio.run(nemo_flow_middleware.awrap_tool_call(tool_call_request, handler)) assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} From 2ffa7139436793b577361f1a429f3bfeadd65787 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 08:31:51 -0700 Subject: [PATCH 10/29] Rename langgraph test dir to not conflict with the langgraph package Signed-off-by: David Gardner --- .../tests/integrations/{langgraph => langgraph_tests}/conftest.py | 0 .../{langgraph => langgraph_tests}/test_langgraph_integration.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/tests/integrations/{langgraph => langgraph_tests}/conftest.py (100%) rename python/tests/integrations/{langgraph => langgraph_tests}/test_langgraph_integration.py (100%) diff --git a/python/tests/integrations/langgraph/conftest.py b/python/tests/integrations/langgraph_tests/conftest.py similarity index 100% rename from python/tests/integrations/langgraph/conftest.py rename to python/tests/integrations/langgraph_tests/conftest.py diff --git a/python/tests/integrations/langgraph/test_langgraph_integration.py b/python/tests/integrations/langgraph_tests/test_langgraph_integration.py similarity index 100% rename from python/tests/integrations/langgraph/test_langgraph_integration.py rename to python/tests/integrations/langgraph_tests/test_langgraph_integration.py From 9c6350cfdaafd14c648ce0219d1c1befc98df99e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 08:32:33 -0700 Subject: [PATCH 11/29] Rename deepagents test dir to not conflict with the deepagents package Signed-off-by: David Gardner --- .../integrations/{deepagents => deepagents_tests}/conftest.py | 0 .../test_deepagents_integration.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/tests/integrations/{deepagents => deepagents_tests}/conftest.py (100%) rename python/tests/integrations/{deepagents => deepagents_tests}/test_deepagents_integration.py (100%) diff --git a/python/tests/integrations/deepagents/conftest.py b/python/tests/integrations/deepagents_tests/conftest.py similarity index 100% rename from python/tests/integrations/deepagents/conftest.py rename to python/tests/integrations/deepagents_tests/conftest.py diff --git a/python/tests/integrations/deepagents/test_deepagents_integration.py b/python/tests/integrations/deepagents_tests/test_deepagents_integration.py similarity index 100% rename from python/tests/integrations/deepagents/test_deepagents_integration.py rename to python/tests/integrations/deepagents_tests/test_deepagents_integration.py From df7f35621727937035e20a38c12bba8c39c12cb5 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 08:48:54 -0700 Subject: [PATCH 12/29] Perform langgraph imports lazily [skip ci] Signed-off-by: David Gardner --- .../test_langgraph_integration.py | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/python/tests/integrations/langgraph_tests/test_langgraph_integration.py b/python/tests/integrations/langgraph_tests/test_langgraph_integration.py index 02e9443b..61817082 100644 --- a/python/tests/integrations/langgraph_tests/test_langgraph_integration.py +++ b/python/tests/integrations/langgraph_tests/test_langgraph_integration.py @@ -10,18 +10,15 @@ from uuid import uuid4 import pytest -from langgraph.callbacks import GraphCallbackHandler, GraphInterruptEvent, GraphResumeEvent -from langgraph.graph import END, START, StateGraph -from langgraph.types import Interrupt from typing_extensions import TypedDict import nemo_flow -from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler as LangChainCallbackHandler -from nemo_flow.integrations.langgraph import NemoFlowCallbackHandler if TYPE_CHECKING: from langgraph.graph import CompiledStateGraph + from nemo_flow.integrations.langgraph import NemoFlowCallbackHandler + class State(TypedDict): value: int @@ -37,6 +34,8 @@ async def aincrement(state: State) -> State: def _build_graph(use_async: bool = False) -> CompiledStateGraph: + from langgraph.graph import END, START, StateGraph + # The cast here avoids a ty linting error builder = StateGraph(cast(Any, State)) if use_async: @@ -58,7 +57,14 @@ def async_graph_fixture() -> CompiledStateGraph: return _build_graph(use_async=True) -def events_to_strings(events: list[nemo_flow.Event]) -> list[str]: +@pytest.fixture(name="callback_handler") +def callback_handler_fixture() -> NemoFlowCallbackHandler: + from nemo_flow.integrations.langgraph import NemoFlowCallbackHandler + + return NemoFlowCallbackHandler() + + +def _events_to_strings(events: list[nemo_flow.Event]) -> list[str]: event_strings: list[str] = [] for event in events: @@ -70,10 +76,13 @@ def events_to_strings(events: list[nemo_flow.Event]) -> list[str]: return event_strings -def test_handler_type(): - handler = NemoFlowCallbackHandler() - assert isinstance(handler, LangChainCallbackHandler) - assert isinstance(handler, GraphCallbackHandler) +def test_handler_type(callback_handler: NemoFlowCallbackHandler): + from langgraph.callbacks import GraphCallbackHandler + + from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler as LangChainCallbackHandler + + assert isinstance(callback_handler, LangChainCallbackHandler) + assert isinstance(callback_handler, GraphCallbackHandler) class TestGraphCallbacks: @@ -91,27 +100,34 @@ def test_sync( self, sync_graph: CompiledStateGraph, subscribed_events: list[nemo_flow.Event], + callback_handler: NemoFlowCallbackHandler, ): with nemo_flow.scope.scope("request", nemo_flow.ScopeType.Agent): - result = sync_graph.invoke({"value": 1}, config={"callbacks": [NemoFlowCallbackHandler()]}) + result = sync_graph.invoke({"value": 1}, config={"callbacks": [callback_handler]}) assert result == {"value": 2} - assert events_to_strings(subscribed_events) == self._expected_events + assert _events_to_strings(subscribed_events) == self._expected_events async def test_async( self, async_graph: CompiledStateGraph, subscribed_events: list[nemo_flow.Event], + callback_handler: NemoFlowCallbackHandler, ): with nemo_flow.scope.scope("request", nemo_flow.ScopeType.Agent): - result = await async_graph.ainvoke({"value": 1}, config={"callbacks": [NemoFlowCallbackHandler()]}) + result = await async_graph.ainvoke({"value": 1}, config={"callbacks": [callback_handler]}) assert result == {"value": 2} - assert events_to_strings(subscribed_events) == self._expected_events + assert _events_to_strings(subscribed_events) == self._expected_events + +def test_graph_lifecycle_callbacks_emit_marks( + subscribed_events: list[nemo_flow.Event], + callback_handler: NemoFlowCallbackHandler, +): + from langgraph.callbacks import GraphInterruptEvent, GraphResumeEvent + from langgraph.types import Interrupt -def test_graph_lifecycle_callbacks_emit_marks(subscribed_events: list[nemo_flow.Event]): - handler = NemoFlowCallbackHandler() run_id = uuid4() expected_event_strings = [ @@ -122,7 +138,7 @@ def test_graph_lifecycle_callbacks_emit_marks(subscribed_events: list[nemo_flow. ] with nemo_flow.scope.scope("request", nemo_flow.ScopeType.Agent): - handler.on_interrupt( + callback_handler.on_interrupt( GraphInterruptEvent( run_id=run_id, status="interrupt_after", @@ -132,7 +148,7 @@ def test_graph_lifecycle_callbacks_emit_marks(subscribed_events: list[nemo_flow. ) ) - handler.on_resume( + callback_handler.on_resume( GraphResumeEvent( run_id=run_id, status="pending", @@ -141,7 +157,7 @@ def test_graph_lifecycle_callbacks_emit_marks(subscribed_events: list[nemo_flow. ) ) - assert events_to_strings(subscribed_events) == expected_event_strings + assert _events_to_strings(subscribed_events) == expected_event_strings interrupt_event = subscribed_events[1] assert isinstance(interrupt_event, nemo_flow.MarkEvent) From b915df0e2b82715f6ec0da81ae1af15e9a56959b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 09:08:28 -0700 Subject: [PATCH 13/29] Remove constructor, avoids warning about not being able to collect the test Signed-off-by: David Gardner --- .../test_langgraph_integration.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/python/tests/integrations/langgraph_tests/test_langgraph_integration.py b/python/tests/integrations/langgraph_tests/test_langgraph_integration.py index 61817082..5b09bb6b 100644 --- a/python/tests/integrations/langgraph_tests/test_langgraph_integration.py +++ b/python/tests/integrations/langgraph_tests/test_langgraph_integration.py @@ -86,15 +86,14 @@ def test_handler_type(callback_handler: NemoFlowCallbackHandler): class TestGraphCallbacks: - def __init__(self): - self._expected_events = [ - "scope.start.request", - "scope.start.LangGraph", - "scope.start.increment", - "scope.end.increment", - "scope.end.LangGraph", - "scope.end.request", - ] + _expected_events = [ + "scope.start.request", + "scope.start.LangGraph", + "scope.start.increment", + "scope.end.increment", + "scope.end.LangGraph", + "scope.end.request", + ] def test_sync( self, From 59332a3a715852caba8679953905099d9d201d09 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 09:08:51 -0700 Subject: [PATCH 14/29] Perform lazy imports for deepagents tests [skip ci] Signed-off-by: David Gardner --- .../test_deepagents_integration.py | 104 ++++++++++++------ 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/python/tests/integrations/deepagents_tests/test_deepagents_integration.py b/python/tests/integrations/deepagents_tests/test_deepagents_integration.py index ea70cb01..ac34267f 100644 --- a/python/tests/integrations/deepagents_tests/test_deepagents_integration.py +++ b/python/tests/integrations/deepagents_tests/test_deepagents_integration.py @@ -5,31 +5,46 @@ from __future__ import annotations +import types from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from unittest.mock import MagicMock from uuid import uuid4 -from deepagents import create_deep_agent -from deepagents.backends import LocalShellBackend -from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel -from langchain_core.messages import AIMessage, ToolMessage -from langgraph.callbacks import GraphInterruptEvent, GraphResumeEvent -from langgraph.types import Interrupt +import pytest import nemo_flow -from nemo_flow.integrations.deepagents import ( - NemoFlowDeepAgentsCallbackHandler, - NemoFlowDeepAgentsMiddleware, - add_nemo_flow_integration, -) +if TYPE_CHECKING: + from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel -class _MockDeepAgentsChatModel(FakeMessagesListChatModel): - model: str = "mock-model" + import nemo_flow.integrations.deepagents as deepagents_integration - def bind_tools(self, _tools: Any, *_args: Any, **_kwargs: Any) -> _MockDeepAgentsChatModel: - return self + +@pytest.fixture(name="deepagents_integration_module", scope="session") +def deepagents_integration_module_fixture() -> types.ModuleType: + import nemo_flow.integrations.deepagents as deepagents_integration + + return deepagents_integration + + +@pytest.fixture(name="callback_handler") +def callback_handler_fixture( + deepagents_integration_module: types.ModuleType, +) -> deepagents_integration.NemoFlowDeepAgentsCallbackHandler: + return deepagents_integration_module.NemoFlowDeepAgentsCallbackHandler() + + +def _mock_deepagents_chat_model(responses: list[Any]) -> FakeMessagesListChatModel: + from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel + + class _MockDeepAgentsChatModel(FakeMessagesListChatModel): + model: str = "mock-model" + + def bind_tools(self, _tools: Any, *_args: Any, **_kwargs: Any) -> _MockDeepAgentsChatModel: + return self + + return _MockDeepAgentsChatModel(responses=responses) def _filter_mark_events(events: list[nemo_flow.Event]) -> list[nemo_flow.MarkEvent]: @@ -46,8 +61,11 @@ def _mark_metadata(mark: nemo_flow.MarkEvent) -> dict[str, Any]: return cast(dict[str, Any], mark.metadata) -def test_before_agent_emits_configuration_mark(subscribed_events: list[nemo_flow.Event]): - middleware = NemoFlowDeepAgentsMiddleware( +def test_before_agent_emits_configuration_mark( + subscribed_events: list[nemo_flow.Event], + deepagents_integration_module: types.ModuleType, +): + middleware = deepagents_integration_module.NemoFlowDeepAgentsMiddleware( agent_name="main-agent", skills=["/skills/research/"], subagents=[{"name": "researcher"}], @@ -65,8 +83,13 @@ def test_before_agent_emits_configuration_mark(subscribed_events: list[nemo_flow assert _mark_data(marks[0])["backend"] == "StateBackend" -def test_callback_handler_emits_human_in_the_loop_marks(subscribed_events: list[nemo_flow.Event]): - handler = NemoFlowDeepAgentsCallbackHandler() +def test_callback_handler_emits_human_in_the_loop_marks( + subscribed_events: list[nemo_flow.Event], + callback_handler: deepagents_integration.NemoFlowDeepAgentsCallbackHandler, +): + from langgraph.callbacks import GraphInterruptEvent, GraphResumeEvent + from langgraph.types import Interrupt + run_id = uuid4() hitl_request = { "action_requests": [ @@ -80,7 +103,7 @@ def test_callback_handler_emits_human_in_the_loop_marks(subscribed_events: list[ } with nemo_flow.scope.scope("request", nemo_flow.ScopeType.Agent): - handler.on_interrupt( + callback_handler.on_interrupt( GraphInterruptEvent( run_id=run_id, status="interrupt_after", @@ -89,7 +112,7 @@ def test_callback_handler_emits_human_in_the_loop_marks(subscribed_events: list[ interrupts=(Interrupt(hitl_request, id="interrupt-1"),), ) ) - handler.on_resume( + callback_handler.on_resume( GraphResumeEvent( run_id=run_id, status="pending", @@ -108,12 +131,17 @@ def test_callback_handler_emits_human_in_the_loop_marks(subscribed_events: list[ assert _mark_metadata(marks[1])["phase"] == "resume" -def test_callback_handler_falls_back_for_non_hitl_interrupt(subscribed_events: list[nemo_flow.Event]): - handler = NemoFlowDeepAgentsCallbackHandler() +def test_callback_handler_falls_back_for_non_hitl_interrupt( + subscribed_events: list[nemo_flow.Event], + callback_handler: deepagents_integration.NemoFlowDeepAgentsCallbackHandler, +): + from langgraph.callbacks import GraphInterruptEvent, GraphResumeEvent + from langgraph.types import Interrupt + run_id = uuid4() with nemo_flow.scope.scope("request", nemo_flow.ScopeType.Agent): - handler.on_interrupt( + callback_handler.on_interrupt( GraphInterruptEvent( run_id=run_id, status="interrupt_after", @@ -122,7 +150,7 @@ def test_callback_handler_falls_back_for_non_hitl_interrupt(subscribed_events: l interrupts=(Interrupt("custom pause", id="interrupt-1"),), ) ) - handler.on_resume( + callback_handler.on_resume( GraphResumeEvent( run_id=run_id, status="pending", @@ -137,10 +165,10 @@ def test_callback_handler_falls_back_for_non_hitl_interrupt(subscribed_events: l assert "deepagents_kind" not in _mark_metadata(marks[0]) -def test_add_nemo_flow_integration_preserves_backend(): +def test_add_nemo_flow_integration_preserves_backend(deepagents_integration_module: types.ModuleType): mock_backend = MagicMock(name="mock_backend") mock_compiled_subagent = MagicMock(name="mock_compiled_subagent") - kwargs = add_nemo_flow_integration( + kwargs = deepagents_integration_module.add_nemo_flow_integration( model="mock-model", name="main-agent", skills=["/skills/main/"], @@ -153,22 +181,32 @@ def test_add_nemo_flow_integration_preserves_backend(): ) assert kwargs["backend"] is mock_backend - assert any(isinstance(item, NemoFlowDeepAgentsMiddleware) for item in kwargs["middleware"]) - assert any(isinstance(item, NemoFlowDeepAgentsMiddleware) for item in kwargs["subagents"][0]["middleware"]) + assert any( + isinstance(item, deepagents_integration_module.NemoFlowDeepAgentsMiddleware) for item in kwargs["middleware"] + ) + assert any( + isinstance(item, deepagents_integration_module.NemoFlowDeepAgentsMiddleware) + for item in kwargs["subagents"][0]["middleware"] + ) assert kwargs["subagents"][1] is mock_compiled_subagent def test_e2e_agent( tmp_path: Path, subscribed_events: list[nemo_flow.Event], + deepagents_integration_module: types.ModuleType, ): + from deepagents import create_deep_agent + from deepagents.backends import LocalShellBackend + from langchain_core.messages import AIMessage, ToolMessage + reviewer_description = "Reviews filesystem work performed by the main agent." - reviewer_model = _MockDeepAgentsChatModel( + reviewer_model = _mock_deepagents_chat_model( responses=[ AIMessage(content="reviewer verified turtle"), ] ) - model = _MockDeepAgentsChatModel( + model = _mock_deepagents_chat_model( responses=[ AIMessage( content="", @@ -196,7 +234,7 @@ def test_e2e_agent( AIMessage(content="created turtle after reviewer verified turtle"), ] ) - kwargs = add_nemo_flow_integration( + kwargs = deepagents_integration_module.add_nemo_flow_integration( model=model, tools=[], name="main-agent", From 317c902ccad4a3785658345ee63eed6abc7f7122 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 09:18:29 -0700 Subject: [PATCH 15/29] WIP Signed-off-by: David Gardner --- .github/ci-path-filters.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index e6dcf43a..ec6a2767 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -67,6 +67,20 @@ python_package: - 'rust-toolchain.toml' - 'uv.lock' +python_integration_langchain: + # Includes LangGraph and DeepAgents integrations as well + - 'justfile' + - 'pyproject.toml' + - 'python/nemo_flow/integrations/deepagents/**' + - 'python/nemo_flow/integrations/langchain/**' + - 'python/nemo_flow/integrations/langgraph/**' + - 'python/tests/integrations/conftest.py' + - 'python/tests/integrations/deepagents_tests/**' + - 'python/tests/integrations/langchain_tests/**' + - 'python/tests/integrations/langgraph_tests/**' + - 'rust-toolchain.toml' + - 'uv.lock' + wasm_package: - 'Cargo.lock' - 'Cargo.toml' From 39e77fa7c499103740d9c9af5d4b6f593e7928c1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 09:28:31 -0700 Subject: [PATCH 16/29] Exlude integration tests from coverage reports Signed-off-by: David Gardner --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f9167b20..95b140ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,10 @@ package = true testpaths = ["python/tests", "third_party/langgraph_tests"] asyncio_mode = "auto" +[tool.coverage.run] +# Exclude integration tests from coverage, since we don't run these by default +omit = ["python/nemo_flow/integrations/*"] + [tool.ty.analysis] # nemo_flow._native is a compiled Rust extension (built by maturin) that only # exists after `uv sync` / `pip install -e .`. Suppress unresolved-import for it. From 93670c924219e287bfa26d2f1a6c3ce2e464b19b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 09:37:03 -0700 Subject: [PATCH 17/29] Seperate the just recipes for python [skip ci] Signed-off-by: David Gardner --- justfile | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 18dcbae6..4ebd4a91 100644 --- a/justfile +++ b/justfile @@ -854,12 +854,12 @@ test-python: fi cargo test -p nemo-flow-python --lib fi - uv sync --inexact --no-install-project --no-install-package nemo-flow --extra langchain --extra langgraph --extra deepagents + uv sync --inexact --no-install-project --no-install-package nemo-flow activate_project_venv python_executable="$(project_python_executable)" use_project_python_source "$python_executable" "$python_executable" -m maturin develop --skip-install - "$python_executable" -m "${pytest_cmd[@]}" + "$python_executable" -m "${pytest_cmd[@]}" --ignore=python/tests/integrations if is_true "{{ ci }}" && [[ -n "$rust_coverage_out" ]]; then cargo llvm-cov report \ -p nemo-flow-python \ @@ -868,6 +868,21 @@ test-python: --output-path "$rust_coverage_out" fi +test-python-langchain: + #!/usr/bin/env bash + {{ bash_helpers }} + pytest_cmd=(pytest) + cd "$NEMO_FLOW_REPO_ROOT" + uv sync --inexact --no-install-project --no-install-package nemo-flow --extra langchain --extra langgraph --extra deepagents + activate_project_venv + python_executable="$(project_python_executable)" + use_project_python_source "$python_executable" + "$python_executable" -m maturin develop --skip-install + "$python_executable" -m "${pytest_cmd[@]}" \ + python/tests/integrations/deepagents_tests \ + python/tests/integrations/langchain_tests \ + python/tests/integrations/langgraph_tests + # --set [output_dir=] [ci=true|false] test-go: #!/usr/bin/env bash @@ -1012,7 +1027,7 @@ test-wasm: fi # --set [output_dir=] [ci=true|false] -test-all: test-rust test-python test-go test-node test-openclaw test-wasm +test-all: test-rust test-python test-python-langchain test-go test-node test-openclaw test-wasm # [version] or --set ref_name= set-version version="": From 661d50557aa3f61bc94091aab1e628d855780cb7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 10:34:59 -0700 Subject: [PATCH 18/29] Optionally run Python LC integration tests Signed-off-by: David Gardner --- .github/workflows/ci.yaml | 5 +++-- .github/workflows/ci_changes.yml | 4 ++++ .github/workflows/ci_python.yml | 10 ++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 29890de1..6a99df3a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -203,7 +203,7 @@ jobs: name: Python needs: [prepare, ci_changes, ci_check] uses: ./.github/workflows/ci_python.yml - if: ${{ needs.ci_check.result == 'success' && needs.ci_changes.outputs.run_python == 'true' }} + if: ${{ needs.ci_check.result == 'success' && ( needs.ci_changes.outputs.run_python == 'true' || needs.ci_changes.outputs.run_python_integration_langchain == 'true' ) }} permissions: contents: read secrets: @@ -211,7 +211,8 @@ jobs: with: ref_type: ${{ github.ref_type }} ref_name: ${{ github.ref_name }} - run_package: ${{ needs.ci_changes.outputs.run_python_package == 'true' }} + run_package: ${{ ( needs.ci_changes.outputs.run_python == 'true' || needs.ci_changes.outputs.run_python_integration_langchain == 'true' ) }} + run_integration_langchain: ${{ needs.ci_changes.outputs.run_python_integration_langchain == 'true' }} ci_wasm: name: WebAssembly diff --git a/.github/workflows/ci_changes.yml b/.github/workflows/ci_changes.yml index 6785992a..ddf02859 100644 --- a/.github/workflows/ci_changes.yml +++ b/.github/workflows/ci_changes.yml @@ -48,6 +48,9 @@ on: run_python: description: 'Whether Python jobs should run' value: ${{ jobs.changes.outputs.run_python }} + run_python_integration_langchain: + description: 'Whether LangChain, LangGraph, and DeepAgents Python integration jobs should run' + value: ${{ jobs.changes.outputs.run_python_integration_langchain }} run_python_package: description: 'Whether Python packaging jobs should run' value: ${{ jobs.changes.outputs.run_python_package }} @@ -80,6 +83,7 @@ jobs: run_node_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.node_package == 'true' }} run_openclaw: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' }} run_python: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python == 'true' }} + run_python_integration_langchain: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python_integration_langchain == 'true' }} run_python_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.python_package == 'true' }} run_rust: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.rust == 'true' }} run_rust_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.rust_package == 'true' }} diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index ebbc4fd0..cc17fa13 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -19,6 +19,11 @@ on: required: false default: true type: boolean + run_integration_langchain: + description: 'Whether to run LangChain, LangGraph, and DeepAgents Python integration tests' + required: false + default: false + type: boolean secrets: CODECOV_TOKEN: required: false @@ -131,6 +136,11 @@ jobs: working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} run: just --set ci true --set output_dir "${{ github.workspace }}" test-python + - name: Run Python LangChain integration tests + if: ${{ inputs.run_integration_langchain == 'true' }} + working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} + run: just --set ci true --set output_dir "${{ github.workspace }}" test-python-integration-langchain + - name: Upload Python coverage to Codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 with: From 58decb548725431328a94c2f64f76f36a963011b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 11:25:24 -0700 Subject: [PATCH 19/29] Formatting/linting Signed-off-by: David Gardner --- python/tests/conftest.py | 1 - python/tests/integrations/conftest.py | 9 ++- .../integrations/deepagents_tests/conftest.py | 3 +- .../integrations/langchain_tests/conftest.py | 3 +- .../langchain_tests/test_callbacks.py | 11 ++- .../langchain_tests/test_middleware.py | 69 +++++++++++++++---- .../integrations/langgraph_tests/conftest.py | 3 +- 7 files changed, 75 insertions(+), 24 deletions(-) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 922a5a6f..cc4b9794 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -7,7 +7,6 @@ import typing from collections.abc import Iterator -import types from uuid import uuid4 import pytest diff --git a/python/tests/integrations/conftest.py b/python/tests/integrations/conftest.py index e14da885..a83bf56e 100644 --- a/python/tests/integrations/conftest.py +++ b/python/tests/integrations/conftest.py @@ -6,37 +6,40 @@ import pytest -@pytest.fixture(name="integration_langchain", scope='session') +@pytest.fixture(name="integration_langchain", scope="session") def integration_langchain_fixture() -> types.ModuleType: """ Use for integration tests that require LangChain to be installed. """ try: import langchain + return langchain except Exception: pytest.skip(reason="langchain must be installed to run LangChain based tests") -@pytest.fixture(name="integration_langgraph", scope='session') +@pytest.fixture(name="integration_langgraph", scope="session") def integration_langgraph_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: """ Use for integration tests that require LangGraph to be installed. """ try: import langgraph + return langgraph except Exception: pytest.skip(reason="langgraph must be installed to run LangGraph based tests") -@pytest.fixture(name="integration_deepagents", scope='session') +@pytest.fixture(name="integration_deepagents", scope="session") def integration_deepagents_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: """ Use for integration tests that require Deep Agents to be installed. """ try: import deepagents + return deepagents except Exception: pytest.skip(reason="deepagents must be installed to run Deep Agents based tests") diff --git a/python/tests/integrations/deepagents_tests/conftest.py b/python/tests/integrations/deepagents_tests/conftest.py index a50e6e5d..0c385601 100644 --- a/python/tests/integrations/deepagents_tests/conftest.py +++ b/python/tests/integrations/deepagents_tests/conftest.py @@ -5,7 +5,8 @@ import pytest -@pytest.fixture(name="integration_deepagents", scope='session', autouse=True) + +@pytest.fixture(name="integration_deepagents", scope="session", autouse=True) def integration_deepagents_fixture(integration_deepagents: types.ModuleType) -> types.ModuleType: """ Override the integration_deepagents fixture to make it autouse diff --git a/python/tests/integrations/langchain_tests/conftest.py b/python/tests/integrations/langchain_tests/conftest.py index eb4f7d35..e5306a8f 100644 --- a/python/tests/integrations/langchain_tests/conftest.py +++ b/python/tests/integrations/langchain_tests/conftest.py @@ -5,7 +5,8 @@ import pytest -@pytest.fixture(name="integration_langchain", scope='session', autouse=True) + +@pytest.fixture(name="integration_langchain", scope="session", autouse=True) def integration_langchain_fixture(integration_langchain: types.ModuleType) -> types.ModuleType: """ Override the integration_langchain fixture to make it autouse diff --git a/python/tests/integrations/langchain_tests/test_callbacks.py b/python/tests/integrations/langchain_tests/test_callbacks.py index 168215df..31fb4d23 100644 --- a/python/tests/integrations/langchain_tests/test_callbacks.py +++ b/python/tests/integrations/langchain_tests/test_callbacks.py @@ -12,7 +12,6 @@ import pytest - if typing.TYPE_CHECKING: from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler @@ -35,12 +34,15 @@ def _make_mock_nemo_flow() -> MagicMock: mock_nemo_flow.scope = scope return mock_nemo_flow + @pytest.fixture(name="callbacks_module", scope="session") def callbacks_module_fixture() -> types.ModuleType: """Fixture to provide the callbacks module.""" import nemo_flow.integrations.langchain.callbacks as callbacks_module + return callbacks_module + @pytest.fixture() def mock_nemo_flow(monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType) -> MagicMock: mock_nemo_flow = _make_mock_nemo_flow() @@ -51,6 +53,7 @@ def mock_nemo_flow(monkeypatch: pytest.MonkeyPatch, callbacks_module: types.Modu @pytest.fixture() def handler(mock_nemo_flow: MagicMock) -> NemoFlowCallbackHandler: from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler + return NemoFlowCallbackHandler() @@ -127,8 +130,9 @@ def test_on_chain_error_pops_scope(self, handler: NemoFlowCallbackHandler, mock_ assert run_id not in handler._scope_handles def test_on_chain_end_prepares_command_outputs(self, handler: NemoFlowCallbackHandler, mock_nemo_flow: MagicMock): - from langgraph.types import Command from langchain_core.messages import ToolMessage + from langgraph.types import Command + run_id = uuid4() handler.on_chain_start( {"name": "MyChain"}, @@ -227,6 +231,7 @@ class TestGracefulNoOp: def test_no_nemo_flow_on_chain_start(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler + handler = NemoFlowCallbackHandler() handler.on_chain_start({"name": "x"}, {}, run_id=uuid4()) @@ -234,6 +239,7 @@ def test_no_nemo_flow_on_chain_start(self, monkeypatch: pytest.MonkeyPatch, call def test_no_nemo_flow_on_chain_end(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler + handler = NemoFlowCallbackHandler() handler.on_chain_end({}, run_id=uuid4()) @@ -241,6 +247,7 @@ def test_no_nemo_flow_on_chain_end(self, monkeypatch: pytest.MonkeyPatch, callba def test_no_nemo_flow_on_chain_error(self, monkeypatch: pytest.MonkeyPatch, callbacks_module: types.ModuleType): monkeypatch.setattr(callbacks_module, "nemo_flow", None) from nemo_flow.integrations.langchain.callbacks import NemoFlowCallbackHandler + handler = NemoFlowCallbackHandler() handler.on_chain_error(RuntimeError("e"), run_id=uuid4()) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index 98eb4c6d..c3e0df5f 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -8,7 +8,7 @@ import asyncio import inspect from collections.abc import Awaitable, Callable -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, MagicMock import pytest @@ -17,19 +17,20 @@ from nemo_flow.codecs import AnthropicMessagesCodec, OpenAIChatCodec, OpenAIResponsesCodec if TYPE_CHECKING: - from langchain_core.messages import AIMessage - from langchain_core.messages import ToolMessage - from langchain.agents.middleware import ToolCallRequest - from langchain.agents.middleware import ModelRequest, ModelResponse + from langchain.agents.middleware import ModelRequest, ModelResponse, ToolCallRequest + from langchain_core.messages import AIMessage, ToolMessage from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware _DEFAULT_MOCK_RESPONSE_MSG = "nemo_flow unittest result" + @pytest.fixture(name="model_request_handler") -def model_request_handler_fixture() -> tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]]: - from langchain_core.messages import AIMessage +def model_request_handler_fixture() -> tuple[ + Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]] +]: from langchain.agents.middleware import ModelResponse + from langchain_core.messages import AIMessage seen_request: dict[str, ModelRequest[Any]] = {} @@ -39,17 +40,23 @@ def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: return handler, seen_request + @pytest.fixture(name="async_model_request_handler") -def async_model_request_handler_fixture(model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]]) -> tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]]: +def async_model_request_handler_fixture( + model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], +) -> tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]]: (sync_handler, seen_request) = model_request_handler + async def handler(request: ModelRequest[Any]) -> ModelResponse[Any]: return sync_handler(request) return handler, seen_request + @pytest.fixture(name="tool_request_handler") def tool_request_handler_fixture() -> tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]: from langchain_core.messages import ToolMessage + seen_request: dict[str, ToolCallRequest] = {} def handler(request: ToolCallRequest) -> ToolMessage: @@ -58,14 +65,19 @@ def handler(request: ToolCallRequest) -> ToolMessage: return handler, seen_request + @pytest.fixture(name="async_tool_request_handler") -def async_tool_request_handler_fixture(tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]) -> tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]: +def async_tool_request_handler_fixture( + tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]], +) -> tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]: (sync_handler, seen_request) = tool_request_handler + async def handler(request: ToolCallRequest) -> ToolMessage: return sync_handler(request) return handler, seen_request + @pytest.fixture(name="mock_tool_execute") def mock_tool_execute_fixture() -> AsyncMock: async def execute_side_effect(*, func: Any, **kwargs: Any) -> ToolMessage: @@ -96,14 +108,18 @@ def _mk_mock_model(returned_message: str | list[AIMessage] = _DEFAULT_MOCK_RESPO return mock_model + @pytest.fixture(name="nemo_flow_middleware") def nemo_flow_middleware_fixture() -> NemoFlowMiddleware: from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware + return NemoFlowMiddleware() + @pytest.fixture(name="recording_middleware") def recording_middleware_fixture() -> NemoFlowMiddleware: from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware + class RecordingMiddleware(NemoFlowMiddleware): def __init__(self): super().__init__() @@ -136,6 +152,7 @@ async def _llm_execute( return RecordingMiddleware() + @pytest.fixture(name="model_request") def model_request_fixture() -> ModelRequest[Any]: from langchain.agents.middleware import ModelRequest @@ -153,6 +170,7 @@ def model_request_fixture() -> ModelRequest[Any]: @pytest.fixture(name="tool_call_request") def tool_call_request_fixture() -> ToolCallRequest: from langchain.agents.middleware import ToolCallRequest + return ToolCallRequest( tool_call={"name": "lookup", "args": {"query": "original"}, "id": "call-1"}, tool=None, @@ -161,8 +179,11 @@ def tool_call_request_fixture() -> ToolCallRequest: ) -def test_wrap_model_call_routes_through_llm_execute(model_request: ModelRequest[Any], model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): - +def test_wrap_model_call_routes_through_llm_execute( + model_request: ModelRequest[Any], + model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], + recording_middleware: NemoFlowMiddleware, +): (handler, seen_request) = model_request_handler response = recording_middleware.wrap_model_call(model_request, handler) @@ -175,7 +196,13 @@ def test_wrap_model_call_routes_through_llm_execute(model_request: ModelRequest[ assert recording_middleware.calls[0]["response_codec"] is None -def test_awrap_model_call_routes_through_llm_execute(model_request: ModelRequest[Any], async_model_request_handler: tuple[Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]]], recording_middleware: NemoFlowMiddleware): +def test_awrap_model_call_routes_through_llm_execute( + model_request: ModelRequest[Any], + async_model_request_handler: tuple[ + Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]] + ], + recording_middleware: NemoFlowMiddleware, +): (handler, seen_request) = async_model_request_handler response = asyncio.run(recording_middleware.awrap_model_call(model_request, handler)) @@ -188,7 +215,13 @@ def test_awrap_model_call_routes_through_llm_execute(model_request: ModelRequest assert recording_middleware.calls[0]["response_codec"] is None -def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_call_request: ToolCallRequest, tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]]): +def test_wrap_tool_call_routes_through_tool_execute( + monkeypatch: pytest.MonkeyPatch, + nemo_flow_middleware: NemoFlowMiddleware, + mock_tool_execute: AsyncMock, + tool_call_request: ToolCallRequest, + tool_request_handler: tuple[Callable[[ToolCallRequest], ToolMessage], dict[str, ToolCallRequest]], +): (handler, seen_request) = tool_request_handler parent_handle = MagicMock() @@ -208,7 +241,13 @@ def test_wrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPa assert isinstance(kwargs["result_codec"], nemo_flow.typed.BestEffortAnyCodec) -def test_awrap_tool_call_routes_through_tool_execute(monkeypatch: pytest.MonkeyPatch, nemo_flow_middleware: NemoFlowMiddleware, mock_tool_execute: AsyncMock, tool_call_request: ToolCallRequest, async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]]): +def test_awrap_tool_call_routes_through_tool_execute( + monkeypatch: pytest.MonkeyPatch, + nemo_flow_middleware: NemoFlowMiddleware, + mock_tool_execute: AsyncMock, + tool_call_request: ToolCallRequest, + async_tool_request_handler: tuple[Callable[[ToolCallRequest], Awaitable[ToolMessage]], dict[str, ToolCallRequest]], +): parent_handle = MagicMock() (handler, seen_request) = async_tool_request_handler @@ -260,7 +299,7 @@ def test_agent_integration(use_async: bool, nemo_flow_middleware: NemoFlowMiddle from langchain.agents import create_agent from langchain_core.messages import AIMessage from langchain_core.tools import tool - + model_responses = [ AIMessage( content="", diff --git a/python/tests/integrations/langgraph_tests/conftest.py b/python/tests/integrations/langgraph_tests/conftest.py index 678734fa..82701647 100644 --- a/python/tests/integrations/langgraph_tests/conftest.py +++ b/python/tests/integrations/langgraph_tests/conftest.py @@ -5,7 +5,8 @@ import pytest -@pytest.fixture(name="integration_langgraph", scope='session', autouse=True) + +@pytest.fixture(name="integration_langgraph", scope="session", autouse=True) def integration_langgraph_fixture(integration_langgraph: types.ModuleType) -> types.ModuleType: """ Override the integration_langgraph fixture to make it autouse From 6b22798dd0cbdec6b317bacc4e76555af6bc2b6d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 11:27:32 -0700 Subject: [PATCH 20/29] Use return to match return type hint Signed-off-by: David Gardner --- python/tests/integrations/deepagents_tests/conftest.py | 2 +- python/tests/integrations/langchain_tests/conftest.py | 2 +- python/tests/integrations/langgraph_tests/conftest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/tests/integrations/deepagents_tests/conftest.py b/python/tests/integrations/deepagents_tests/conftest.py index 0c385601..f5f69fb5 100644 --- a/python/tests/integrations/deepagents_tests/conftest.py +++ b/python/tests/integrations/deepagents_tests/conftest.py @@ -11,4 +11,4 @@ def integration_deepagents_fixture(integration_deepagents: types.ModuleType) -> """ Override the integration_deepagents fixture to make it autouse """ - yield integration_deepagents + return integration_deepagents diff --git a/python/tests/integrations/langchain_tests/conftest.py b/python/tests/integrations/langchain_tests/conftest.py index e5306a8f..80511211 100644 --- a/python/tests/integrations/langchain_tests/conftest.py +++ b/python/tests/integrations/langchain_tests/conftest.py @@ -11,4 +11,4 @@ def integration_langchain_fixture(integration_langchain: types.ModuleType) -> ty """ Override the integration_langchain fixture to make it autouse """ - yield integration_langchain + return integration_langchain diff --git a/python/tests/integrations/langgraph_tests/conftest.py b/python/tests/integrations/langgraph_tests/conftest.py index 82701647..9a7c1ade 100644 --- a/python/tests/integrations/langgraph_tests/conftest.py +++ b/python/tests/integrations/langgraph_tests/conftest.py @@ -11,4 +11,4 @@ def integration_langgraph_fixture(integration_langgraph: types.ModuleType) -> ty """ Override the integration_langgraph fixture to make it autouse """ - yield integration_langgraph + return integration_langgraph From 0341d2c14f8b90f9506f9a4a99c0321fdb420ea2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 11:38:10 -0700 Subject: [PATCH 21/29] Linting fixes Signed-off-by: David Gardner --- .../langchain_tests/test_middleware.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index c3e0df5f..c2dc0175 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -8,7 +8,7 @@ import asyncio import inspect from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol from unittest.mock import AsyncMock, MagicMock import pytest @@ -115,12 +115,17 @@ def nemo_flow_middleware_fixture() -> NemoFlowMiddleware: return NemoFlowMiddleware() +class RecordingMiddleware(Protocol): + calls: list[dict[str, Any]] + wrap_model_call: Callable + awrap_model_call: Callable + @pytest.fixture(name="recording_middleware") -def recording_middleware_fixture() -> NemoFlowMiddleware: +def recording_middleware_fixture() -> RecordingMiddleware: from nemo_flow.integrations.langchain.middleware import NemoFlowMiddleware - class RecordingMiddleware(NemoFlowMiddleware): + class _RecordingMiddleware(NemoFlowMiddleware, RecordingMiddleware): def __init__(self): super().__init__() self.calls: list[dict[str, Any]] = [] @@ -150,7 +155,7 @@ async def _llm_execute( ) return await func(intercepted) - return RecordingMiddleware() + return _RecordingMiddleware() @pytest.fixture(name="model_request") @@ -182,7 +187,7 @@ def tool_call_request_fixture() -> ToolCallRequest: def test_wrap_model_call_routes_through_llm_execute( model_request: ModelRequest[Any], model_request_handler: tuple[Callable[[ModelRequest[Any]], ModelResponse[Any]], dict[str, ModelRequest[Any]]], - recording_middleware: NemoFlowMiddleware, + recording_middleware: RecordingMiddleware, ): (handler, seen_request) = model_request_handler @@ -201,7 +206,7 @@ def test_awrap_model_call_routes_through_llm_execute( async_model_request_handler: tuple[ Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], dict[str, ModelRequest[Any]] ], - recording_middleware: NemoFlowMiddleware, + recording_middleware: RecordingMiddleware, ): (handler, seen_request) = async_model_request_handler @@ -233,6 +238,7 @@ def test_wrap_tool_call_routes_through_tool_execute( assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} mock_tool_execute.assert_awaited_once() + assert mock_tool_execute.await_args is not None kwargs = mock_tool_execute.await_args.kwargs assert kwargs["name"] == "lookup" assert kwargs["args"] == {"query": "original"} @@ -259,6 +265,7 @@ def test_awrap_tool_call_routes_through_tool_execute( assert response.content == "done" assert seen_request["request"].tool_call["args"] == {"query": "intercepted"} mock_tool_execute.assert_awaited_once() + assert mock_tool_execute.await_args is not None kwargs = mock_tool_execute.await_args.kwargs assert kwargs["name"] == "lookup" assert kwargs["args"] == {"query": "original"} From 9add9b5dfb564419ab1ae2776b27be9f1a53c58d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 11:40:59 -0700 Subject: [PATCH 22/29] Formatting Signed-off-by: David Gardner --- python/tests/integrations/langchain_tests/test_middleware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tests/integrations/langchain_tests/test_middleware.py b/python/tests/integrations/langchain_tests/test_middleware.py index c2dc0175..72fe9642 100644 --- a/python/tests/integrations/langchain_tests/test_middleware.py +++ b/python/tests/integrations/langchain_tests/test_middleware.py @@ -115,6 +115,7 @@ def nemo_flow_middleware_fixture() -> NemoFlowMiddleware: return NemoFlowMiddleware() + class RecordingMiddleware(Protocol): calls: list[dict[str, Any]] wrap_model_call: Callable From c044f9d9bbf5371192099e3c1f16225641ad967b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 12:34:46 -0700 Subject: [PATCH 23/29] Avoid installing LC deps if we don't need to Signed-off-by: David Gardner --- .github/workflows/ci.yaml | 1 + .github/workflows/ci_check.yml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6a99df3a..66608177 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,7 @@ jobs: with: full_ci: ${{ needs.prepare.outputs.full_ci == 'true' }} base: ${{ needs.ci_changes.outputs.base }} + run_python_integration_langchain: ${{ needs.ci_changes.outputs.run_python_integration_langchain == 'true' }} ci_license_diff: name: License Diff diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 131c5e6e..f110f09c 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -11,6 +11,11 @@ on: required: false default: false type: boolean + run_python_integration_langchain: + description: 'Whether LangChain, LangGraph, and DeepAgents Python integration checks may need optional extras' + required: false + default: false + type: boolean base: description: 'The comparison base used for filtered pre-commit checks' required: false @@ -104,12 +109,18 @@ jobs: FULL_CI: ${{ inputs.full_ci }} PRE_COMMIT_BASE: ${{ inputs.base }} PRE_COMMIT_HOME: ${{ runner.temp }}/.cache/pre-commit + PYTHON_INTEGRATION_LANGCHAIN: ${{ inputs.run_python_integration_langchain }} # The attribution hook syncs docs deps; do not apply repo warning policy to third-party Rust builds. RUSTFLAGS: "" run: | set -e uv tool install pre-commit==${{ steps.ci-config.outputs.pre_commit_version }} - uv sync --inexact --no-install-project --no-install-package nemo-flow --extra langchain --extra langgraph --extra deepagents + if [[ "$PYTHON_INTEGRATION_LANGCHAIN" == "true" ]]; then + FLOW_CI_UV_SYNC_EXTRA_ARGS="--extra langchain --extra langgraph --extra deepagents" + else + FLOW_CI_UV_SYNC_EXTRA_ARGS="" + fi + uv sync --inexact --no-install-project --no-install-package nemo-flow ${FLOW_CI_UV_SYNC_EXTRA_ARGS} if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure else From 4e48df5f55fe5b45633b4318e0280f1998924e85 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 12:49:27 -0700 Subject: [PATCH 24/29] Linting fix Signed-off-by: David Gardner --- .github/workflows/ci_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index f110f09c..56cbbee7 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -120,7 +120,7 @@ jobs: else FLOW_CI_UV_SYNC_EXTRA_ARGS="" fi - uv sync --inexact --no-install-project --no-install-package nemo-flow ${FLOW_CI_UV_SYNC_EXTRA_ARGS} + uv sync --inexact --no-install-project --no-install-package nemo-flow "${FLOW_CI_UV_SYNC_EXTRA_ARGS}" if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure else From 8d6c1423684d8f800e12980aaa8cbb74d2310a79 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 12:52:34 -0700 Subject: [PATCH 25/29] Linting fix Signed-off-by: David Gardner --- .github/workflows/ci_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 56cbbee7..4dfe65f8 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -120,7 +120,7 @@ jobs: else FLOW_CI_UV_SYNC_EXTRA_ARGS="" fi - uv sync --inexact --no-install-project --no-install-package nemo-flow "${FLOW_CI_UV_SYNC_EXTRA_ARGS}" + uv sync "${FLOW_CI_UV_SYNC_EXTRA_ARGS}" --inexact --no-install-project --no-install-package nemo-flow if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure else From d4737b605552662f8f54b8c3c8204d8c566ed932 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 12:56:43 -0700 Subject: [PATCH 26/29] Linting Signed-off-by: David Gardner --- .github/workflows/ci_check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 4dfe65f8..13d150b7 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -116,11 +116,11 @@ jobs: set -e uv tool install pre-commit==${{ steps.ci-config.outputs.pre_commit_version }} if [[ "$PYTHON_INTEGRATION_LANGCHAIN" == "true" ]]; then - FLOW_CI_UV_SYNC_EXTRA_ARGS="--extra langchain --extra langgraph --extra deepagents" + uv sync --inexact --no-install-project --no-install-package nemo-flow --extra langchain --extra langgraph --extra deepagents else - FLOW_CI_UV_SYNC_EXTRA_ARGS="" + uv sync --inexact --no-install-project --no-install-package nemo-flow fi - uv sync "${FLOW_CI_UV_SYNC_EXTRA_ARGS}" --inexact --no-install-project --no-install-package nemo-flow + if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure else From 12793eedd90e1c53cd03b53a40ee7cbb32a21cb6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 12:58:51 -0700 Subject: [PATCH 27/29] Fix handling of uv sync flags Signed-off-by: David Gardner --- .github/workflows/ci_check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 13d150b7..43afc987 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -115,12 +115,12 @@ jobs: run: | set -e uv tool install pre-commit==${{ steps.ci-config.outputs.pre_commit_version }} + FLOW_CI_UV_SYNC_EXTRA_ARGS=() if [[ "$PYTHON_INTEGRATION_LANGCHAIN" == "true" ]]; then - uv sync --inexact --no-install-project --no-install-package nemo-flow --extra langchain --extra langgraph --extra deepagents - else - uv sync --inexact --no-install-project --no-install-package nemo-flow + FLOW_CI_UV_SYNC_EXTRA_ARGS+=(--extra langchain --extra langgraph --extra deepagents) fi + uv sync --inexact --no-install-project --no-install-package nemo-flow "${FLOW_CI_UV_SYNC_EXTRA_ARGS[@]}" if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure else From 005b7673e3d6f7de26b74db582ac7c0e841bb4a8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 13:01:01 -0700 Subject: [PATCH 28/29] Fix recipe name Signed-off-by: David Gardner --- .github/workflows/ci_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index cc17fa13..7242f1cb 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -139,7 +139,7 @@ jobs: - name: Run Python LangChain integration tests if: ${{ inputs.run_integration_langchain == 'true' }} working-directory: ${{ env.NEMO_FLOW_CI_WORKSPACE }} - run: just --set ci true --set output_dir "${{ github.workspace }}" test-python-integration-langchain + run: just --set ci true --set output_dir "${{ github.workspace }}" test-python-langchain - name: Upload Python coverage to Codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 From cc72e20c7080b91c76c85bf5f3225802fc302106 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 May 2026 13:04:41 -0700 Subject: [PATCH 29/29] Formatting Signed-off-by: David Gardner --- .github/workflows/ci_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 43afc987..e11e78af 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -119,7 +119,7 @@ jobs: if [[ "$PYTHON_INTEGRATION_LANGCHAIN" == "true" ]]; then FLOW_CI_UV_SYNC_EXTRA_ARGS+=(--extra langchain --extra langgraph --extra deepagents) fi - + uv sync --inexact --no-install-project --no-install-package nemo-flow "${FLOW_CI_UV_SYNC_EXTRA_ARGS[@]}" if [[ "$FULL_CI" == "true" || -z "$PRE_COMMIT_BASE" ]]; then pre-commit run --all-files --show-diff-on-failure