Fix #2446: split_continuous_references corrupts plain bracketed lists (non-reference text)

src/memos/mem_os/utils/reference_utils.py

@@ -1,18 +1,31 @@
+import re
+
 from memos.memories.textual.item import (
     TextualMemoryItem,
 )
 
 
+# A single reference item inside a bracketed list looks like ``<int>:<id>``,
+# e.g. ``1:92ff35fb``. The id part is a hex-looking memory id, so we only
+# require it to be non-empty and free of commas / brackets. Surrounding
+# whitespace on either the ref id or the memory id is tolerated.
+_REFERENCE_ITEM_RE = re.compile(r"^\s*\d+:[^,\[\]]+\s*$")
+
+
 def split_continuous_references(text: str) -> str:
     """
     Split continuous reference tags into individual reference tags.
 
-    Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb] [4:bfe6f044]
+    Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb][4:bfe6f044].
 
-    Only processes text if:
-    1. '[' appears exactly once
-    2. ']' appears exactly once
-    3. Contains commas between '[' and ']'
+    Only processes text if all of the following hold:
+
+    1. ``[`` appears exactly once
+    2. ``]`` appears exactly once
+    3. There is at least one comma between the brackets
+    4. Every comma-separated item inside the brackets matches ``<int>:<id>``
+       (the reference tag shape). If any item does not, the block is treated
+       as ordinary prose (e.g. ``[apple, banana]``) and returned unchanged.
 
     Args:
         text (str): Text containing reference tags
@@ -41,10 +54,20 @@ def split_continuous_references(text: str) -> str:
     # Check if there's a comma between brackets
     if "," not in content_between_brackets:
         return text
-    text = text.replace(content_between_brackets, content_between_brackets.replace(", ", "]["))
-    text = text.replace(content_between_brackets, content_between_brackets.replace(",", "]["))
-
-    return text
+    # Shape guard: only rewrite when every item looks like a reference tag
+    # (``<int>:<id>``). Otherwise the block is plain bracketed prose and
+    # must be preserved verbatim (issue #2446).
+    items = content_between_brackets.split(",")
+    if not all(_REFERENCE_ITEM_RE.match(item) for item in items):
+        return text
+    # Rebuild the bracketed block in a single pass so mixed separator styles
+    # (``", "`` and bare ``","`` in the same block) are all split correctly.
+    # The previous two-step ``str.replace`` approach was broken: after the
+    # first pass rewrote ``", "`` occurrences, the original substring no
+    # longer existed in ``text`` and the second pass never fired, leaving
+    # bare commas unsplit (PR #2450 review).
+    joined = "][".join(item.strip() for item in items)
+    return text[: open_bracket_pos + 1] + joined + text[close_bracket_pos:]
 
 
 def process_streaming_references_complete(text_buffer: str) -> tuple[str, str]:

tests/mem_os/test_reference_utils.py

@@ -0,0 +1,87 @@
+"""Tests for src/memos/mem_os/utils/reference_utils.py.
+
+Focus: split_continuous_references shape guard.
+Related issue: #2446 — plain bracketed prose such as "[apple, banana]" must
+not be rewritten as "[apple][banana]". Only reference lists whose items are
+``int:<id>`` pairs (e.g. ``[1:92ff35fb, 4:bfe6f044]``) should be split.
+"""
+
+from memos.mem_os.utils.reference_utils import split_continuous_references
+
+
+class TestSplitContinuousReferences:
+    """Shape guard on ``split_continuous_references``."""
+
+    # --- happy path: real reference lists still split ------------------------
+
+    def test_splits_two_int_id_items(self):
+        assert (
+            split_continuous_references("See [1:92ff35fb, 4:bfe6f044] now")
+            == "See [1:92ff35fb][4:bfe6f044] now"
+        )
+
+    def test_splits_three_int_id_items(self):
+        assert split_continuous_references("[1:aa, 2:bb, 3:cc]") == "[1:aa][2:bb][3:cc]"
+
+    def test_handles_comma_without_space(self):
+        assert (
+            split_continuous_references("prefix [1:aa,2:bb] suffix") == "prefix [1:aa][2:bb] suffix"
+        )
+
+    def test_handles_mixed_separator_styles(self):
+        """Regression: mixed ``", "`` and bare ``","`` separators in the same
+        block must all be split (PR #2450 review).
+
+        The previous two-step ``str.replace`` implementation left the bare
+        comma between ``2:bb`` and ``3:cc`` intact, producing
+        ``"[1:aa][2:bb,3:cc]"``.
+        """
+        assert (
+            split_continuous_references("[1:aa, 2:bb,3:cc]") == "[1:aa][2:bb][3:cc]"
+        )
+
+    def test_handles_mixed_separator_styles_reversed(self):
+        """Bare comma first, then ``", "`` — symmetric to the case above."""
+        assert (
+            split_continuous_references("[1:aa,2:bb, 3:cc]") == "[1:aa][2:bb][3:cc]"
+        )
+
+    # --- shape guard: non-reference brackets stay untouched ------------------
+
+    def test_plain_prose_list_untouched(self):
+        """The regression case from issue #2446."""
+        text = "The set is [apple, banana] here"
+        assert split_continuous_references(text) == text
+
+    def test_mixed_reference_and_prose_untouched(self):
+        """If even one item is not int:<id> the whole block is preserved."""
+        text = "Look at [1:92ff35fb, banana] please"
+        assert split_continuous_references(text) == text
+
+    def test_numeric_only_items_untouched(self):
+        """Numbers without a colon are not references."""
+        text = "Pick [1, 2, 3] please"
+        assert split_continuous_references(text) == text
+
+    def test_non_integer_prefix_untouched(self):
+        """The item prefix must be a decimal integer."""
+        text = "Combine [a:1, b:2]"
+        assert split_continuous_references(text) == text
+
+    # --- boundary conditions unchanged ---------------------------------------
+
+    def test_empty_string_returns_empty(self):
+        assert split_continuous_references("") == ""
+
+    def test_no_brackets_returns_text_unchanged(self):
+        text = "no brackets, just commas"
+        assert split_continuous_references(text) == text
+
+    def test_multiple_open_brackets_returns_unchanged(self):
+        text = "many [1:aa, 2:bb] and [3:cc, 4:dd]"
+        assert split_continuous_references(text) == text
+
+    def test_single_reference_item_unchanged(self):
+        """A single item has no comma so nothing to split."""
+        text = "just [1:aa]"
+        assert split_continuous_references(text) == text