feat: page the search results with cursors

Author: azmeuk

Containerfile

@@ -6,9 +6,9 @@ ENV UV_COMPILE_BYTECODE=1 \
 
 WORKDIR /app
 COPY pyproject.toml uv.lock README.md LICENSE ./
-RUN uv sync --locked --no-dev --extra precis --no-install-project
+RUN uv sync --locked --no-dev --extra precis --extra cursor --no-install-project
 COPY scim2_server ./scim2_server
-RUN uv sync --locked --no-dev --extra precis --no-editable
+RUN uv sync --locked --no-dev --extra precis --extra cursor --no-editable
 
 FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
 

doc/_examples/asgi_integration.py

@@ -1,4 +1,5 @@
 import json
+import os
 from urllib.parse import parse_qsl
 
 from scim2_models import SCIMException
@@ -10,7 +11,7 @@
 from scim2_server.service import ScimService
 from scim2_server.utils import load_default_provider
 
-service = ScimService(load_default_provider())
+service = ScimService(load_default_provider(), secret=os.environ["SCIM_SECRET"])
 handler = AsyncScimHandler(service, AsyncInMemoryStorage())
 
 

doc/_examples/async_cursor_storage.py

@@ -0,0 +1,53 @@
+from doc._examples.async_sqlite_storage import AsyncSQLiteStorage
+from scim2_server.storage import SearchPage
+
+
+class AsyncCursorSQLiteStorage(AsyncSQLiteStorage):
+    supports_cursors = True
+
+    async def search(self, resource_types, search_request, *, position=None):
+        types = {resource_type.name: resource_type for resource_type in resource_types}
+        [(total,)] = await self._select(types, "COUNT(*)")
+        if search_request.cursor is None:
+            return await self._search_index(types, total, search_request)
+        return await self._search_cursor(types, total, search_request, position)
+
+    async def _search_index(self, types, total, search_request):
+        count = search_request.count
+        page = [-1 if count is None else count, search_request.start_index_0]
+        rows = await self._select(types, "*", "ORDER BY id LIMIT ? OFFSET ?", page)
+        return SearchPage(total, self._to_resources(types, rows))
+
+    async def _search_cursor(self, types, total, search_request, position):
+        count = search_request.count
+        limit = -1 if count is None else count + 1
+        if position is None or "after" in position:
+            after = "" if position is None else position["after"]
+            clause = "AND id > ? ORDER BY id LIMIT ?"
+            rows = await self._select(types, "*", clause, [after, limit])
+            resources = self._to_resources(types, rows[:count])
+            more_after, more_before = len(rows) > len(resources), position is not None
+        else:
+            clause = "AND id < ? ORDER BY id DESC LIMIT ?"
+            rows = await self._select(types, "*", clause, [position["before"], limit])
+            resources = self._to_resources(types, rows[:count][::-1])
+            more_after, more_before = True, len(rows) > len(resources)
+
+        result = SearchPage(total, resources)
+        if resources and more_after:
+            result.next = {"after": resources[-1].id}
+        if resources and more_before:
+            result.previous = {"before": resources[0].id}
+        return result
+
+    async def _select(self, types, columns, clause="", parameters=()):
+        connection = await self.connect()
+        among = ", ".join("?" * len(types))
+        cursor = await connection.execute(
+            f"SELECT {columns} FROM resources WHERE resource_type IN ({among}) {clause}",
+            [*types, *parameters],
+        )
+        return list(await cursor.fetchall())
+
+    def _to_resources(self, types, rows):
+        return [self.row_to_resource(types[row["resource_type"]], row) for row in rows]

doc/_examples/async_library_storage.py

@@ -16,6 +16,7 @@
 
 from doc._examples.library_storage import TABLES
 from scim2_server.storage import AsyncScimStorage
+from scim2_server.storage import SearchPage
 
 
 class AsyncLibraryStorage(AsyncScimStorage):
@@ -131,8 +132,12 @@ async def writing(self) -> AsyncIterator[aiosqlite.Connection]:
             raise UniquenessException from None
 
     async def search(
-        self, resource_types: list[ResourceType], search_request: SearchRequest[Any]
-    ) -> tuple[int, list[Resource[Any]]]:
+        self,
+        resource_types: list[ResourceType],
+        search_request: SearchRequest[Any],
+        *,
+        position: Any = None,
+    ) -> SearchPage:
         connection = await self.connect()
         found: list[Resource[Any]] = []
         for resource_type in resource_types:
@@ -144,6 +149,5 @@ async def search(
                 resource for resource in found if search_request.filter.match(resource)
             ]
         found = search_request.sort(found)
-        start = (search_request.start_index or 1) - 1
-        stop = None if search_request.count is None else start + search_request.count
-        return len(found), found[start:stop]
+        page = found[search_request.start_index_0 : search_request.stop_index_0]
+        return SearchPage(len(found), page)

doc/_examples/async_sqlite_storage.py

@@ -19,6 +19,7 @@
 
 from doc._examples.sqlite_storage import TABLE
 from scim2_server.storage import AsyncScimStorage
+from scim2_server.storage import SearchPage
 
 
 class AsyncSQLiteStorage(AsyncScimStorage):
@@ -153,8 +154,12 @@ async def writing(self) -> AsyncIterator[aiosqlite.Connection]:
             raise UniquenessException from None
 
     async def search(
-        self, resource_types: list[ResourceType], search_request: SearchRequest[Any]
-    ) -> tuple[int, list[Resource[Any]]]:
+        self,
+        resource_types: list[ResourceType],
+        search_request: SearchRequest[Any],
+        *,
+        position: Any = None,
+    ) -> SearchPage:
         connection = await self.connect()
         found = []
         for resource_type in resource_types:
@@ -171,6 +176,5 @@ async def search(
                 resource for resource in found if search_request.filter.match(resource)
             ]
         found = search_request.sort(found)
-        start = (search_request.start_index or 1) - 1
-        stop = None if search_request.count is None else start + search_request.count
-        return len(found), found[start:stop]
+        page = found[search_request.start_index_0 : search_request.stop_index_0]
+        return SearchPage(len(found), page)

doc/_examples/cursor_storage.py

@@ -0,0 +1,51 @@
+from doc._examples.sqlite_storage import SQLiteStorage
+from scim2_server.storage import SearchPage
+
+
+class CursorSQLiteStorage(SQLiteStorage):
+    supports_cursors = True
+
+    def search(self, resource_types, search_request, *, position=None):
+        types = {resource_type.name: resource_type for resource_type in resource_types}
+        [(total,)] = self._select(types, "COUNT(*)")
+        if search_request.cursor is None:
+            return self._search_index(types, total, search_request)
+        return self._search_cursor(types, total, search_request, position)
+
+    def _search_index(self, types, total, search_request):
+        count = search_request.count
+        page = [-1 if count is None else count, search_request.start_index_0]
+        rows = self._select(types, "*", "ORDER BY id LIMIT ? OFFSET ?", page)
+        return SearchPage(total, self._to_resources(types, rows))
+
+    def _search_cursor(self, types, total, search_request, position):
+        count = search_request.count
+        limit = -1 if count is None else count + 1
+        if position is None or "after" in position:
+            after = "" if position is None else position["after"]
+            clause = "AND id > ? ORDER BY id LIMIT ?"
+            rows = self._select(types, "*", clause, [after, limit])
+            resources = self._to_resources(types, rows[:count])
+            more_after, more_before = len(rows) > len(resources), position is not None
+        else:
+            clause = "AND id < ? ORDER BY id DESC LIMIT ?"
+            rows = self._select(types, "*", clause, [position["before"], limit])
+            resources = self._to_resources(types, rows[:count][::-1])
+            more_after, more_before = True, len(rows) > len(resources)
+
+        result = SearchPage(total, resources)
+        if resources and more_after:
+            result.next = {"after": resources[-1].id}
+        if resources and more_before:
+            result.previous = {"before": resources[0].id}
+        return result
+
+    def _select(self, types, columns, clause="", parameters=()):
+        among = ", ".join("?" * len(types))
+        return self.connection.execute(
+            f"SELECT {columns} FROM resources WHERE resource_type IN ({among}) {clause}",
+            [*types, *parameters],
+        ).fetchall()
+
+    def _to_resources(self, types, rows):
+        return [self.row_to_resource(types[row["resource_type"]], row) for row in rows]

doc/_examples/library_storage.py

@@ -27,6 +27,7 @@
 from scim2_models import User
 
 from scim2_server.storage import ScimStorage
+from scim2_server.storage import SearchPage
 
 MEMBERS = """
 CREATE TABLE IF NOT EXISTS members (
@@ -233,8 +234,12 @@ def writing(self) -> Iterator[None]:
             raise UniquenessException from None
 
     def search(
-        self, resource_types: list[ResourceType], search_request: SearchRequest[Any]
-    ) -> tuple[int, list[Resource[Any]]]:
+        self,
+        resource_types: list[ResourceType],
+        search_request: SearchRequest[Any],
+        *,
+        position: Any = None,
+    ) -> SearchPage:
         found: list[Resource[Any]] = []
         for resource_type in resource_types:
             table = TABLES[resource_type.name]
@@ -245,6 +250,5 @@ def search(
                 resource for resource in found if search_request.filter.match(resource)
             ]
         found = search_request.sort(found)
-        start = (search_request.start_index or 1) - 1
-        stop = None if search_request.count is None else start + search_request.count
-        return len(found), found[start:stop]
+        page = found[search_request.start_index_0 : search_request.stop_index_0]
+        return SearchPage(len(found), page)

doc/_examples/sqlite_storage.py

@@ -17,6 +17,7 @@
 from scim2_models import UniquenessException
 
 from scim2_server.storage import ScimStorage
+from scim2_server.storage import SearchPage
 
 TABLE = """
 CREATE TABLE IF NOT EXISTS resources (
@@ -148,8 +149,12 @@ def writing(self) -> Iterator[None]:
             raise UniquenessException from None
 
     def search(
-        self, resource_types: list[ResourceType], search_request: SearchRequest[Any]
-    ) -> tuple[int, list[Resource[Any]]]:
+        self,
+        resource_types: list[ResourceType],
+        search_request: SearchRequest[Any],
+        *,
+        position: Any = None,
+    ) -> SearchPage:
         found = []
         for resource_type in resource_types:
             rows = self.connection.execute(
@@ -162,6 +167,5 @@ def search(
                 resource for resource in found if search_request.filter.match(resource)
             ]
         found = search_request.sort(found)
-        start = (search_request.start_index or 1) - 1
-        stop = None if search_request.count is None else start + search_request.count
-        return len(found), found[start:stop]
+        page = found[search_request.start_index_0 : search_request.stop_index_0]
+        return SearchPage(len(found), page)

doc/_examples/wsgi_integration.py

@@ -1,4 +1,5 @@
 import json
+import os
 from urllib.parse import parse_qsl
 from wsgiref.util import application_uri
 
@@ -11,7 +12,7 @@
 from scim2_server.service import ScimService
 from scim2_server.utils import load_default_provider
 
-service = ScimService(load_default_provider())
+service = ScimService(load_default_provider(), secret=os.environ["SCIM_SECRET"])
 handler = ScimHandler(service, InMemoryStorage())
 
 

doc/changelog.rst

@@ -1,6 +1,58 @@
 Changelog
 =========
 
+[0.8.0] - Unreleased
+--------------------
+
+Added
+^^^^^
+- Cursor pagination of :rfc:`9865`. A configuration that announces it in ``pagination`` gets
+  ``nextCursor`` and ``previousCursor`` in the search responses. The clients cannot read or
+  forge the cursors. The server refuses a cursor of another query, another ``count``, or older
+  than ``cursorTimeout``. The cursors give stable pages: a resource that exists during the whole
+  paging is returned once, even when others are created or deleted. Cursor pagination needs the
+  new ``secret`` parameter of :class:`~scim2_server.service.ScimService`, the ``cursor`` extra, and
+  a storage that sets the new
+  :attr:`~scim2_server.storage.ScimStorage.supports_cursors`, such as
+  :class:`~scim2_server.memory.InMemoryStorage`. The server refuses to start otherwise.
+- The ``scim2-server`` command pages with cursors when its ``--service-provider-config``
+  announces them. Its container image includes the ``cursor`` extra.
+- :class:`~scim2_server.applications.wsgi.WSGIApplication` and
+  :class:`~scim2_server.applications.asgi.ASGIApplication` give their default service the secret
+  of the ``SCIM2_SERVER_SECRET`` environment variable.
+- The ``maxPageSize`` of ``pagination`` bounds the ``count`` of every search.
+- :class:`~scim2_server.testing.ScimStorageContract` and
+  :class:`~scim2_server.testing.AsyncScimStorageContract` check that the cursors of the
+  storages that support them give stable pages.
+
+Changed
+^^^^^^^
+- :meth:`ScimStorage.search <scim2_server.storage.ScimStorage.search>` returns a
+  :class:`~scim2_server.storage.SearchPage` instead of a tuple, and takes the ``position`` of the
+  page when a cursor pages the resources. This also applies to code that calls it directly, such
+  as the ``search`` method of :class:`~scim2_server.memory.InMemoryStorage`.
+- :class:`~scim2_server.memory.InMemoryStorage` returns the resources in the order of their
+  identifiers when the search has no ``sortBy``, instead of the order of creation. The same goes
+  for the resources with the same sort value.
+- For the subclasses of :class:`~scim2_server.service.ScimService`:
+  :meth:`~scim2_server.service.ScimService.read_search`, ``read_search_query`` and
+  ``read_search_body`` return the search request with the position of the page its cursor asks
+  for, and :meth:`~scim2_server.service.ScimService.search_response` takes the searched resource
+  types and a :class:`~scim2_server.storage.SearchPage` instead of the number of resources and
+  the page.
+- The WSGI and ASGI applications keep the query parameters without a value, as scim2-flask does.
+  An empty ``filter``, ``sortBy`` or ``count`` gets a 400 response instead of being ignored.
+- With ``pagination`` in the configuration, a search without ``count`` gets ``defaultPageSize``
+  resources, by index as by cursor, instead of every resource up to ``maxResults``.
+- A search with a ``cursor`` gets a 501 response when the configuration does not announce
+  cursor pagination, instead of ignoring the cursor.
+
+Deprecated
+^^^^^^^^^^
+- A storage whose ``search`` method does not accept ``position`` raises a
+  :class:`DeprecationWarning`. Such a storage pages by index only. scim2-server 0.9 will require
+  ``position``.
+
 [0.7.1] - 2026-10-07
 --------------------