diff --git a/src/memos/mem_os/utils/reference_utils.py b/src/memos/mem_os/utils/reference_utils.py index 09b812207..fa002ee63 100644 --- a/src/memos/mem_os/utils/reference_utils.py +++ b/src/memos/mem_os/utils/reference_utils.py @@ -1,18 +1,31 @@ +import re + from memos.memories.textual.item import ( TextualMemoryItem, ) +# A single reference item inside a bracketed list looks like ``:``, +# e.g. ``1:92ff35fb``. The id part is a hex-looking memory id, so we only +# require it to be non-empty and free of commas / brackets. Surrounding +# whitespace on either the ref id or the memory id is tolerated. +_REFERENCE_ITEM_RE = re.compile(r"^\s*\d+:[^,\[\]]+\s*$") + + def split_continuous_references(text: str) -> str: """ Split continuous reference tags into individual reference tags. - Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb] [4:bfe6f044] + Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb][4:bfe6f044]. - Only processes text if: - 1. '[' appears exactly once - 2. ']' appears exactly once - 3. Contains commas between '[' and ']' + Only processes text if all of the following hold: + + 1. ``[`` appears exactly once + 2. ``]`` appears exactly once + 3. There is at least one comma between the brackets + 4. Every comma-separated item inside the brackets matches ``:`` + (the reference tag shape). If any item does not, the block is treated + as ordinary prose (e.g. ``[apple, banana]``) and returned unchanged. Args: text (str): Text containing reference tags @@ -41,10 +54,20 @@ def split_continuous_references(text: str) -> str: # Check if there's a comma between brackets if "," not in content_between_brackets: return text - text = text.replace(content_between_brackets, content_between_brackets.replace(", ", "][")) - text = text.replace(content_between_brackets, content_between_brackets.replace(",", "][")) - - return text + # Shape guard: only rewrite when every item looks like a reference tag + # (``:``). Otherwise the block is plain bracketed prose and + # must be preserved verbatim (issue #2446). + items = content_between_brackets.split(",") + if not all(_REFERENCE_ITEM_RE.match(item) for item in items): + return text + # Rebuild the bracketed block in a single pass so mixed separator styles + # (``", "`` and bare ``","`` in the same block) are all split correctly. + # The previous two-step ``str.replace`` approach was broken: after the + # first pass rewrote ``", "`` occurrences, the original substring no + # longer existed in ``text`` and the second pass never fired, leaving + # bare commas unsplit (PR #2450 review). + joined = "][".join(item.strip() for item in items) + return text[: open_bracket_pos + 1] + joined + text[close_bracket_pos:] def process_streaming_references_complete(text_buffer: str) -> tuple[str, str]: diff --git a/tests/mem_os/test_reference_utils.py b/tests/mem_os/test_reference_utils.py new file mode 100644 index 000000000..00c28cd4d --- /dev/null +++ b/tests/mem_os/test_reference_utils.py @@ -0,0 +1,87 @@ +"""Tests for src/memos/mem_os/utils/reference_utils.py. + +Focus: split_continuous_references shape guard. +Related issue: #2446 — plain bracketed prose such as "[apple, banana]" must +not be rewritten as "[apple][banana]". Only reference lists whose items are +``int:`` pairs (e.g. ``[1:92ff35fb, 4:bfe6f044]``) should be split. +""" + +from memos.mem_os.utils.reference_utils import split_continuous_references + + +class TestSplitContinuousReferences: + """Shape guard on ``split_continuous_references``.""" + + # --- happy path: real reference lists still split ------------------------ + + def test_splits_two_int_id_items(self): + assert ( + split_continuous_references("See [1:92ff35fb, 4:bfe6f044] now") + == "See [1:92ff35fb][4:bfe6f044] now" + ) + + def test_splits_three_int_id_items(self): + assert split_continuous_references("[1:aa, 2:bb, 3:cc]") == "[1:aa][2:bb][3:cc]" + + def test_handles_comma_without_space(self): + assert ( + split_continuous_references("prefix [1:aa,2:bb] suffix") == "prefix [1:aa][2:bb] suffix" + ) + + def test_handles_mixed_separator_styles(self): + """Regression: mixed ``", "`` and bare ``","`` separators in the same + block must all be split (PR #2450 review). + + The previous two-step ``str.replace`` implementation left the bare + comma between ``2:bb`` and ``3:cc`` intact, producing + ``"[1:aa][2:bb,3:cc]"``. + """ + assert ( + split_continuous_references("[1:aa, 2:bb,3:cc]") == "[1:aa][2:bb][3:cc]" + ) + + def test_handles_mixed_separator_styles_reversed(self): + """Bare comma first, then ``", "`` — symmetric to the case above.""" + assert ( + split_continuous_references("[1:aa,2:bb, 3:cc]") == "[1:aa][2:bb][3:cc]" + ) + + # --- shape guard: non-reference brackets stay untouched ------------------ + + def test_plain_prose_list_untouched(self): + """The regression case from issue #2446.""" + text = "The set is [apple, banana] here" + assert split_continuous_references(text) == text + + def test_mixed_reference_and_prose_untouched(self): + """If even one item is not int: the whole block is preserved.""" + text = "Look at [1:92ff35fb, banana] please" + assert split_continuous_references(text) == text + + def test_numeric_only_items_untouched(self): + """Numbers without a colon are not references.""" + text = "Pick [1, 2, 3] please" + assert split_continuous_references(text) == text + + def test_non_integer_prefix_untouched(self): + """The item prefix must be a decimal integer.""" + text = "Combine [a:1, b:2]" + assert split_continuous_references(text) == text + + # --- boundary conditions unchanged --------------------------------------- + + def test_empty_string_returns_empty(self): + assert split_continuous_references("") == "" + + def test_no_brackets_returns_text_unchanged(self): + text = "no brackets, just commas" + assert split_continuous_references(text) == text + + def test_multiple_open_brackets_returns_unchanged(self): + text = "many [1:aa, 2:bb] and [3:cc, 4:dd]" + assert split_continuous_references(text) == text + + def test_single_reference_item_unchanged(self): + """A single item has no comma so nothing to split.""" + text = "just [1:aa]" + assert split_continuous_references(text) == text