From 073cebaee9adeb5defa78dace9a00fd14431ead3 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Tue, 25 Nov 2025 13:36:01 +0800 Subject: [PATCH 1/2] Fix asyncio.run() error in novelty checking Update novelty LLM check to handle calls from both async and sync contexts, using ThreadPoolExecutor when an event loop is running. Add tests to verify correct behavior in both contexts and address issue #313. --- openevolve/database.py | 38 +++++-- tests/test_novelty_asyncio_issue.py | 169 ++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 tests/test_novelty_asyncio_issue.py diff --git a/openevolve/database.py b/openevolve/database.py index 1c7f73937..f7b10bdb2 100644 --- a/openevolve/database.py +++ b/openevolve/database.py @@ -975,20 +975,36 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool """ import asyncio from openevolve.novelty_judge import NOVELTY_SYSTEM_MSG, NOVELTY_USER_MSG - + user_msg = NOVELTY_USER_MSG.format( language=program.language, existing_code=similar_program.code, proposed_code=program.code, ) - + try: - content: str = asyncio.run( - self.novelty_llm.generate_with_context( - system_message=NOVELTY_SYSTEM_MSG, - messages=[{"role": "user", "content": user_msg}], + # Check if we're already in an event loop + try: + loop = asyncio.get_running_loop() + # We're in an async context, need to run in a new thread + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + asyncio.run, + self.novelty_llm.generate_with_context( + system_message=NOVELTY_SYSTEM_MSG, + messages=[{"role": "user", "content": user_msg}], + ) + ) + content: str = future.result() + except RuntimeError: + # No event loop running, safe to use asyncio.run() + content: str = asyncio.run( + self.novelty_llm.generate_with_context( + system_message=NOVELTY_SYSTEM_MSG, + messages=[{"role": "user", "content": user_msg}], + ) ) - ) if content is None or content is None: logger.warning("Novelty LLM returned empty response") @@ -999,11 +1015,11 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool # Parse the response NOVEL_i = content.upper().find("NOVEL") NOT_NOVEL_i = content.upper().find("NOT NOVEL") - + if NOVEL_i == -1 and NOT_NOVEL_i == -1: logger.warning(f"Unexpected novelty LLM response: {content}") return True # Assume novel if we can't parse - + if NOVEL_i != -1 and NOT_NOVEL_i != -1: # Both found, take the one that appears first is_novel = NOVEL_i < NOT_NOVEL_i @@ -1011,12 +1027,12 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool is_novel = True else: is_novel = False - + return is_novel except Exception as e: logger.error(f"Error in novelty LLM check: {e}") - + return True def _is_novel(self, program_id: int, island_idx: int) -> bool: diff --git a/tests/test_novelty_asyncio_issue.py b/tests/test_novelty_asyncio_issue.py new file mode 100644 index 000000000..46fb03475 --- /dev/null +++ b/tests/test_novelty_asyncio_issue.py @@ -0,0 +1,169 @@ +""" +Test for issue #313: asyncio.run() error in novelty checking +https://github.com/algorithmicsuperintelligence/openevolve/issues/313 + +This test reproduces the bug where calling database.add() from within an async context +triggers a novelty check that uses asyncio.run(), which fails because it's already +running in an event loop. +""" + +import unittest +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch, Mock +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase + + +class MockLLM: + """Mock LLM that implements the async interface""" + + async def generate_with_context(self, system_message: str, messages: list): + """Mock async generate method that returns NOVEL""" + return "NOVEL" + + +class TestNoveltyAsyncioIssue(unittest.TestCase): + """Test for asyncio.run() error in novelty checking (issue #313)""" + + @patch('openevolve.embedding.EmbeddingClient') + def setUp(self, mock_embedding_client_class): + """Set up test database with novelty checking enabled""" + # Mock the embedding client + mock_instance = MagicMock() + mock_instance.get_embedding.return_value = [0.1] * 1536 # Mock embedding vector + mock_embedding_client_class.return_value = mock_instance + + config = Config() + config.database.in_memory = True + config.database.embedding_model = "text-embedding-3-small" + config.database.similarity_threshold = 0.99 + config.database.novelty_llm = MockLLM() + + self.db = ProgramDatabase(config.database) + self.mock_embedding_client_class = mock_embedding_client_class + + def test_novelty_check_from_async_context_works(self): + """ + Test that novelty checking works correctly when called from within + an async context (this was the bug in issue #313). + + Expected behavior: Should successfully run the novelty check without + any asyncio.run() errors, properly using ThreadPoolExecutor to handle + the async LLM call from within a running event loop. + """ + import logging + + # Create two programs with similar embeddings to trigger LLM novelty check + program1 = Program( + id="prog1", + code="def test(): return 1", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog2", + code="def test(): return 2", + language="python", + metrics={"score": 0.6}, + parent_id="prog1", + ) + + async def async_add_programs(): + """Add programs from async context - this simulates controller.run()""" + # Add first program (no novelty check, no similar programs yet) + prog1_id = self.db.add(program1) + self.assertIsNotNone(prog1_id) + + # Add second program - this triggers novelty check + # Since embeddings are similar (both [0.1] * 1536), it will call + # _llm_judge_novelty which should now work correctly + prog2_id = self.db.add(program2) + + # The novelty check should succeed without errors + # The program should be added (MockLLM returns "NOVEL") + self.assertIsNotNone(prog2_id) + + return True + + # This should work without any errors now + result = asyncio.run(async_add_programs()) + self.assertTrue(result) + + # Verify both programs were added + self.assertIn("prog1", self.db.programs) + self.assertIn("prog2", self.db.programs) + + def test_novelty_check_from_sync_context_works(self): + """ + Test that novelty checking also works correctly when called from + a synchronous (non-async) context. + + Expected behavior: Should successfully run the novelty check using + asyncio.run() since there's no running event loop. + """ + # Create two programs with similar embeddings to trigger LLM novelty check + program1 = Program( + id="prog3", + code="def test(): return 3", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog4", + code="def test(): return 4", + language="python", + metrics={"score": 0.6}, + parent_id="prog3", + ) + + # Add programs from synchronous context (no event loop running) + prog1_id = self.db.add(program1) + self.assertIsNotNone(prog1_id) + + prog2_id = self.db.add(program2) + self.assertIsNotNone(prog2_id) + + # Verify both programs were added + self.assertIn("prog3", self.db.programs) + self.assertIn("prog4", self.db.programs) + + def test_novelty_check_disabled_works_fine(self): + """ + Test that when novelty checking is disabled, adding programs + from async context works fine (this is the workaround from issue #313). + """ + # Create a new database with novelty checking disabled + config = Config() + config.database.in_memory = True + config.database.similarity_threshold = 0.0 # Disable novelty checking + db_no_novelty = ProgramDatabase(config.database) + + program1 = Program( + id="prog1", + code="def test(): return 1", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog2", + code="def test(): return 2", + language="python", + metrics={"score": 0.6}, + ) + + async def async_add_programs(): + """Add programs from async context""" + db_no_novelty.add(program1) + db_no_novelty.add(program2) + return True + + # This should work fine without novelty checking + result = asyncio.run(async_add_programs()) + self.assertTrue(result) + + +if __name__ == "__main__": + unittest.main() From 24b8eca9aecacb76be90232a568758f7e4423db6 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Tue, 25 Nov 2025 13:40:49 +0800 Subject: [PATCH 2/2] Update _version.py --- openevolve/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openevolve/_version.py b/openevolve/_version.py index 57b6d4eb9..ddad2c119 100644 --- a/openevolve/_version.py +++ b/openevolve/_version.py @@ -1,3 +1,3 @@ """Version information for openevolve package.""" -__version__ = "0.2.20" +__version__ = "0.2.21"