diff --git a/pyrit/converter/unicode_replacement_converter.py b/pyrit/converter/unicode_replacement_converter.py index 779c4ee308..c4c65c7605 100644 --- a/pyrit/converter/unicode_replacement_converter.py +++ b/pyrit/converter/unicode_replacement_converter.py @@ -52,7 +52,11 @@ async def convert_word_async(self, word: str) -> str: Returns: str: The converted word. """ - return "".join(f"\\u{ord(ch):04x}" for ch in word) + # A \u escape holds exactly four hex digits, so characters outside the Basic Multilingual + # Plane are written as their UTF-16 surrogate pair. "surrogatepass" keeps lone surrogates + # (valid in a Python str) as their own code unit instead of raising. + code_units = word.encode("utf-16-be", errors="surrogatepass") + return "".join(f"\\u{int.from_bytes(code_units[i : i + 2], 'big'):04x}" for i in range(0, len(code_units), 2)) def join_words(self, words: list[str]) -> str: """ diff --git a/tests/unit/converter/test_unicode_replacement_converter.py b/tests/unit/converter/test_unicode_replacement_converter.py index a453c9a353..27d9ba69fa 100644 --- a/tests/unit/converter/test_unicode_replacement_converter.py +++ b/tests/unit/converter/test_unicode_replacement_converter.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json + import pytest from pyrit.converter import ConverterResult, UnicodeReplacementConverter @@ -50,3 +52,25 @@ async def test_unicode_replacement_input_not_supported(): converter = UnicodeReplacementConverter() with pytest.raises(ValueError): await converter.convert_async(prompt="hello", input_type="image_path") + + +async def test_unicode_replacement_non_bmp_uses_surrogate_pairs(): + converter = UnicodeReplacementConverter() + result = await converter.convert_async(prompt="hi \U0001f600", input_type="text") + assert result.output_text == "\\u0068\\u0069 \\ud83d\\ude00" + assert json.loads(f'"{result.output_text}"') == "hi \U0001f600" + + +@pytest.mark.parametrize( + "prompt, expected", + [ + (json.loads(r'"\ud800"'), r"\ud800"), + (json.loads(r'"\udc00"'), r"\udc00"), + (json.loads(r'"a\udbff b"'), r"\u0061\udbff \u0062"), + ], + ids=["lone_high", "lone_low", "lone_high_in_word"], +) +async def test_unicode_replacement_preserves_lone_surrogates(prompt, expected): + converter = UnicodeReplacementConverter() + result = await converter.convert_async(prompt=prompt, input_type="text") + assert result.output_text == expected