Support 1.37 features

.github/workflows/main.yaml

@@ -26,9 +26,9 @@ env:
   WEAVIATE_132: 1.32.27
   WEAVIATE_133: 1.33.18
   WEAVIATE_134: 1.34.19
-  WEAVIATE_135: 1.35.15
-  WEAVIATE_136: 1.36.6-8edcf08.amd64
-  WEAVIATE_137: 1.37.0-dev-29d5c87.amd64
+  WEAVIATE_135: 1.35.18
+  WEAVIATE_136: 1.36.12
+  WEAVIATE_137: 1.37.1-4e61e26.amd64
 
 jobs:
   lint-and-format:

ci/docker-compose.yml

@@ -31,6 +31,8 @@ services:
       DISABLE_LAZY_LOAD_SHARDS: 'true'
       GRPC_MAX_MESSAGE_SIZE: 100000000  # 100mb
       OBJECTS_TTL_DELETE_SCHEDULE: "@every 12h" # for objectTTL tests to work
+      EXPORT_ENABLED: 'true'
+      EXPORT_DEFAULT_PATH: "/var/lib/weaviate/exports"
 
   contextionary:
     environment:

docs/changelog.rst

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+Version 4.21.0
+--------------
+This minor version includes:
+    - Support for new 1.37 features:
+        - Add support for the new ``blobHash`` property data type
+        - Add support for returning profiling when making queries with the ``return_metadata=["query_profile"]`` parameter
+        - Add support for on-demaned tokenization through the ``client.tokenize`` namespace
+        - Add support for managing permissions for accessing the native MCP server
+        - Add support for collection export
+        - Add support for incremental backups
+    - Minor bug fixes and improvements:
+        - Change ``alpha`` queries and aggregations to use server-side default parameter
+        - Fixes rare flakey behaviour of ``client.batch.stream`` on server hangup
+
+
 Version 4.20.5
 --------------
 This patch version includes:

integration/conftest.py

@@ -1,5 +1,9 @@
 import os
 import time
+import sys
+import threading
+import traceback
+import pytest
 from typing import (
     Any,
     AsyncGenerator,
@@ -14,7 +18,6 @@
 )
 from typing import Callable, TypeVar
 
-import pytest
 import pytest_asyncio
 from _pytest.fixtures import SubRequest
 
@@ -500,3 +503,66 @@ def retry_on_http_error(
             raise
     # This should never be reached, but satisfies the type checker
     raise last_exception  # type: ignore
+
+
+TIMEOUT_SECONDS = 300
+
+
+def dump_all_stacks():
+    frames = sys._current_frames()
+    lines = ["\n===== DEADLOCK DETECTED — THREAD DUMP =====\n"]
+    for thread in threading.enumerate():
+        frame = frames.get(thread.ident)  # pyright: ignore
+        lines.append(f"\n--- Thread: {thread.name} (id={thread.ident}) ---")
+        if frame:
+            lines.append("".join(traceback.format_stack(frame)))
+        else:
+            lines.append("  (no frame)\n")
+    lines.append("===========================================\n")
+    return "\n".join(lines)
+
+
+class DeadlockWatchdog:
+    def __init__(self, timeout):
+        self.timeout = timeout
+        self._timer = None
+
+    def start(self, label):
+        self._label = label
+        self._timer = threading.Timer(self.timeout, self._on_timeout)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def stop(self):
+        if self._timer:
+            self._timer.cancel()
+            self._timer = None
+
+    def _on_timeout(self):
+        sys.stderr.write(f"\n[WATCHDOG] Hung at: '{self._label}' after {self.timeout}s\n")
+        sys.stderr.write(dump_all_stacks())
+        sys.stderr.flush()
+        os._exit(1)  # Hard kill — works reliably in xdist workers
+
+
+_watchdog = DeadlockWatchdog(TIMEOUT_SECONDS)
+
+
+# Covers setup + call + teardown
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_protocol(item, nextitem):
+    _watchdog.start(item.nodeid)
+    try:
+        yield
+    finally:
+        _watchdog.stop()
+
+
+# Separately watch session-scoped fixture setup
+@pytest.hookimpl(hookwrapper=True)
+def pytest_sessionstart(session):
+    _watchdog.start("session startup / session-scoped fixtures")
+    try:
+        yield
+    finally:
+        _watchdog.stop()

integration/test_batch_v4.py

@@ -718,7 +718,7 @@ def test_non_existant_collection(client_factory: ClientFactory) -> None:
     # not, so we do not check for errors here
 
 
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 def test_number_of_stored_results_in_batch(client_factory: ClientFactory) -> None:
     client, name = client_factory()
     with client.batch.dynamic() as batch:
@@ -818,7 +818,7 @@ def test_references_with_to_uuids(client_factory: ClientFactory) -> None:
 
 
 @pytest.mark.asyncio
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 async def test_add_one_hundred_thousand_objects_async_client(
     async_client_factory: AsyncClientFactory,
 ) -> None:
@@ -849,7 +849,7 @@ async def test_add_one_hundred_thousand_objects_async_client(
     await client.collections.delete(name)
 
 
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 def test_add_one_hundred_thousand_objects_sync_client(
     client_factory: ClientFactory,
 ) -> None:

integration/test_client.py

@@ -334,7 +334,7 @@ def test_collection_name_capitalization(
         client.collections.delete(name_big)
 
 
-def test_client_cluster_with_lazy_shard_loading(
+def test_client_cluster_without_lazy_shard_loading(
     client: weaviate.WeaviateClient, request: SubRequest
 ) -> None:
     try:

integration/test_collection_batch.py

@@ -271,7 +271,7 @@ def test_non_existant_collection(collection_factory_get: CollectionFactoryGet) -
 
 
 @pytest.mark.asyncio
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 async def test_batch_one_hundred_thousand_objects_async_collection(
     batch_collection_async: BatchCollectionAsync,
 ) -> None:
@@ -299,7 +299,7 @@ async def test_batch_one_hundred_thousand_objects_async_collection(
 
 
 @pytest.mark.asyncio
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 async def test_ingest_one_hundred_thousand_data_objects_async(
     batch_collection_async: BatchCollectionAsync,
 ) -> None:
@@ -321,7 +321,7 @@ async def test_ingest_one_hundred_thousand_data_objects_async(
     assert len(results.errors) == 0, [obj.message for obj in results.errors.values()]
 
 
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(60)
 def test_ingest_one_hundred_thousand_data_objects(
     batch_collection: BatchCollection,
 ) -> None: