FIX Write non-BMP characters as surrogate pairs in UnicodeReplacementConverter

pyrit/converter/unicode_replacement_converter.py

@@ -52,7 +52,11 @@ async def convert_word_async(self, word: str) -> str:
         Returns:
             str: The converted word.
         """
-        return "".join(f"\\u{ord(ch):04x}" for ch in word)
+        # A \u escape holds exactly four hex digits, so characters outside the Basic Multilingual
+        # Plane are written as their UTF-16 surrogate pair. "surrogatepass" keeps lone surrogates
+        # (valid in a Python str) as their own code unit instead of raising.
+        code_units = word.encode("utf-16-be", errors="surrogatepass")
+        return "".join(f"\\u{int.from_bytes(code_units[i : i + 2], 'big'):04x}" for i in range(0, len(code_units), 2))
 
     def join_words(self, words: list[str]) -> str:
         """

tests/unit/converter/test_unicode_replacement_converter.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import json
+
 import pytest
 
 from pyrit.converter import ConverterResult, UnicodeReplacementConverter
@@ -50,3 +52,25 @@ async def test_unicode_replacement_input_not_supported():
     converter = UnicodeReplacementConverter()
     with pytest.raises(ValueError):
         await converter.convert_async(prompt="hello", input_type="image_path")
+
+
+async def test_unicode_replacement_non_bmp_uses_surrogate_pairs():
+    converter = UnicodeReplacementConverter()
+    result = await converter.convert_async(prompt="hi \U0001f600", input_type="text")
+    assert result.output_text == "\\u0068\\u0069 \\ud83d\\ude00"
+    assert json.loads(f'"{result.output_text}"') == "hi \U0001f600"
+
+
+@pytest.mark.parametrize(
+    "prompt, expected",
+    [
+        (json.loads(r'"\ud800"'), r"\ud800"),
+        (json.loads(r'"\udc00"'), r"\udc00"),
+        (json.loads(r'"a\udbff b"'), r"\u0061\udbff \u0062"),
+    ],
+    ids=["lone_high", "lone_low", "lone_high_in_word"],
+)
+async def test_unicode_replacement_preserves_lone_surrogates(prompt, expected):
+    converter = UnicodeReplacementConverter()
+    result = await converter.convert_async(prompt=prompt, input_type="text")
+    assert result.output_text == expected