fix: order strings by code point on PostgreSQL

Comparisons and sorts on strings now use the C collation, so they follow the order of scim2-models. Before, they followed the collation of the column.

Author: azmeuk

doc/explanation.rst

@@ -69,9 +69,10 @@ database allows:
   This approaches the default key without reaching it. ``lower()`` follows the locale of the
   database, and only lowers ASCII letters under the ``C`` locale. Even under a UTF-8 locale, a
   few letters differ, such as ``İ`` and the final sigma. ``normalize()`` needs PostgreSQL 13 and
-  a database encoded in UTF-8. An index that serves these filters is an index on the same
-  expression. With a key other than the default one, the storage warns that it only approaches
-  the key.
+  a database encoded in UTF-8. The comparison uses the ``C`` collation, which orders strings by
+  code point, as scim2-models does, whatever the collation of the column. An index that serves
+  these filters is an index on the same expression, with the same collation. With a key other
+  than the default one, the storage warns that it only approaches the key.
 - Other databases lower both sides with their own ``lower()``. MySQL and MariaDB are not
   supported: their default collations ignore the case and the accents, even for a ``caseExact``
   attribute.

src/scim2_sqlalchemy/comparison.py

@@ -75,14 +75,15 @@ class _Normalizer(_Comparator):
     """Compare strings in PostgreSQL, normalized to NFC on both sides.
 
     The lower() of PostgreSQL follows the locale of the database, so the
-    comparison approaches the default comparison key without reaching it.
+    comparison approaches the default comparison key without reaching it. The
+    C collation orders the strings by code point, as scim2-models does.
     """
 
     def _form(self, column: _Column, expression: Any) -> ColumnElement[Any]:
         normalized = func.normalize(expression, literal_column("NFC"))
         if column.casefolded:
-            return func.normalize(func.lower(normalized), literal_column("NFC"))
-        return normalized
+            normalized = func.normalize(func.lower(normalized), literal_column("NFC"))
+        return normalized.collate("C")
 
     def column(self, column: _Column) -> ColumnElement[Any]:
         return self._form(column, column.compared)

tests/test_comparison.py

@@ -12,6 +12,7 @@
 from scim2_models import default_comparison_key
 from scim2_server.utils import load_default_service_provider_config
 from sqlalchemy import select
+from sqlalchemy import text
 from sqlalchemy.dialects import mysql
 
 from scim2_sqlalchemy.comparison import _comparator
@@ -228,3 +229,28 @@ def test_postgresql_approaches_the_default_key(session, database_url, value):
     assert session.scalar(select(expression)) == default_comparison_key(
         column.binding, value
     )
+
+
+@pytest.mark.parametrize(
+    ("parameters", "expected"),
+    [
+        ({"sort_by": "userName"}, ["e", "f", "é"]),
+        ({"filter": 'userName lt "f"'}, ["e"]),
+    ],
+)
+def test_postgresql_orders_strings_by_code_point(
+    sync_storage_factory, session, database_url, user_type, parameters, expected
+):
+    """The order of scim2-models holds whatever the collation of the column."""
+    if database_url.startswith("sqlite"):
+        pytest.skip("SQLite orders strings by code point")
+    session.execute(
+        text(
+            'ALTER TABLE users ALTER COLUMN user_name TYPE varchar COLLATE "und-x-icu"'
+        )
+    )
+    storage = sync_storage_factory()
+    for name in ("é", "f", "e"):
+        storage.create(user_type, UserModel(user_name=name))
+
+    assert user_names(storage, user_type, **parameters) == expected