From c3b1816d1eb4edc5c702893c34888e4034f05604 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:55:07 +0530 Subject: [PATCH 1/2] FIX Write non-BMP characters as surrogate pairs in UnicodeReplacementConverter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `:04x` is a minimum width, so a character above U+FFFF was written with five or six hex digits (`ὠ0` for U+1F600), which every `\u` decoder reads as a different character followed by a stray digit. Encode the word as UTF-16 code units so these characters become a surrogate pair (`😀`). BMP characters are unchanged. Fixes #3130 Co-Authored-By: Claude Opus 5.5 --- pyrit/converter/unicode_replacement_converter.py | 5 ++++- .../unit/converter/test_unicode_replacement_converter.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyrit/converter/unicode_replacement_converter.py b/pyrit/converter/unicode_replacement_converter.py index 3741f45d6a..3e292ed2b4 100644 --- a/pyrit/converter/unicode_replacement_converter.py +++ b/pyrit/converter/unicode_replacement_converter.py @@ -54,7 +54,10 @@ async def convert_word_async(self, word: str) -> str: Returns: str: The converted word. """ - return "".join(f"\\u{ord(ch):04x}" for ch in word) + # A \u escape holds exactly four hex digits, so characters outside the Basic Multilingual + # Plane are written as their UTF-16 surrogate pair. + code_units = word.encode("utf-16-be") + return "".join(f"\\u{int.from_bytes(code_units[i : i + 2], 'big'):04x}" for i in range(0, len(code_units), 2)) def join_words(self, words: list[str]) -> str: """ diff --git a/tests/unit/converter/test_unicode_replacement_converter.py b/tests/unit/converter/test_unicode_replacement_converter.py index a453c9a353..2f2ae27634 100644 --- a/tests/unit/converter/test_unicode_replacement_converter.py +++ b/tests/unit/converter/test_unicode_replacement_converter.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json + import pytest from pyrit.converter import ConverterResult, UnicodeReplacementConverter @@ -50,3 +52,10 @@ async def test_unicode_replacement_input_not_supported(): converter = UnicodeReplacementConverter() with pytest.raises(ValueError): await converter.convert_async(prompt="hello", input_type="image_path") + + +async def test_unicode_replacement_non_bmp_uses_surrogate_pairs(): + converter = UnicodeReplacementConverter() + result = await converter.convert_async(prompt="hi \U0001f600", input_type="text") + assert result.output_text == "\\u0068\\u0069 \\ud83d\\ude00" + assert json.loads(f'"{result.output_text}"') == "hi \U0001f600" From 2f9ef777d71da5b0d65d3d8376d9a9ab2fd0c622 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:14:19 +0530 Subject: [PATCH 2/2] FIX Keep lone surrogates in UnicodeReplacementConverter A Python str can hold an unpaired surrogate (for example from json.loads), and strict UTF-16 encoding raised "surrogates not allowed" for it. Encode with errors="surrogatepass" so a lone surrogate is written as its own code unit, as before, while non-BMP characters are still written as surrogate pairs. Co-Authored-By: Claude Opus 5.5 --- pyrit/converter/unicode_replacement_converter.py | 5 +++-- .../test_unicode_replacement_converter.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pyrit/converter/unicode_replacement_converter.py b/pyrit/converter/unicode_replacement_converter.py index 3e292ed2b4..afe17fcb86 100644 --- a/pyrit/converter/unicode_replacement_converter.py +++ b/pyrit/converter/unicode_replacement_converter.py @@ -55,8 +55,9 @@ async def convert_word_async(self, word: str) -> str: str: The converted word. """ # A \u escape holds exactly four hex digits, so characters outside the Basic Multilingual - # Plane are written as their UTF-16 surrogate pair. - code_units = word.encode("utf-16-be") + # Plane are written as their UTF-16 surrogate pair. "surrogatepass" keeps lone surrogates + # (valid in a Python str) as their own code unit instead of raising. + code_units = word.encode("utf-16-be", errors="surrogatepass") return "".join(f"\\u{int.from_bytes(code_units[i : i + 2], 'big'):04x}" for i in range(0, len(code_units), 2)) def join_words(self, words: list[str]) -> str: diff --git a/tests/unit/converter/test_unicode_replacement_converter.py b/tests/unit/converter/test_unicode_replacement_converter.py index 2f2ae27634..27d9ba69fa 100644 --- a/tests/unit/converter/test_unicode_replacement_converter.py +++ b/tests/unit/converter/test_unicode_replacement_converter.py @@ -59,3 +59,18 @@ async def test_unicode_replacement_non_bmp_uses_surrogate_pairs(): result = await converter.convert_async(prompt="hi \U0001f600", input_type="text") assert result.output_text == "\\u0068\\u0069 \\ud83d\\ude00" assert json.loads(f'"{result.output_text}"') == "hi \U0001f600" + + +@pytest.mark.parametrize( + "prompt, expected", + [ + (json.loads(r'"\ud800"'), r"\ud800"), + (json.loads(r'"\udc00"'), r"\udc00"), + (json.loads(r'"a\udbff b"'), r"\u0061\udbff \u0062"), + ], + ids=["lone_high", "lone_low", "lone_high_in_word"], +) +async def test_unicode_replacement_preserves_lone_surrogates(prompt, expected): + converter = UnicodeReplacementConverter() + result = await converter.convert_async(prompt=prompt, input_type="text") + assert result.output_text == expected