gh-66788: Add the utf-7-imap codec (GH-153149)

Implement the modified UTF-7 encoding used for international IMAP4
mailbox names (RFC 3501, section 5.1.3).  It differs from UTF-7:
"&" is the shift character ("&-" encodes a literal "&"), "," replaces
"/" in the Base64 alphabet, and all non-printable-ASCII characters are
Base64-encoded.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: serhiy-storchaka

Doc/library/codecs.rst

@@ -1438,6 +1438,14 @@ encodings.
 |                    |         | code actually uses UTF-8  |
 |                    |         | by default.               |
 +--------------------+---------+---------------------------+
+| utf-7-imap         | mUTF-7  | Modified UTF-7 encoding   |
+|                    |         | of :rfc:`3501` for IMAP4  |
+|                    |         | mailbox names.  Only      |
+|                    |         | ``errors='strict'`` is    |
+|                    |         | supported.                |
+|                    |         |                           |
+|                    |         | .. versionadded:: next    |
++--------------------+---------+---------------------------+
 
 .. versionchanged:: 3.8
    "unicode_internal" codec is removed.

Doc/whatsnew/3.16.rst

@@ -214,6 +214,14 @@ curses
   wide-character support.
   (Contributed by Serhiy Storchaka in :gh:`133031`.)
 
+encodings
+---------
+
+* Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding
+  used for international IMAP4 mailbox names (:rfc:`3501`).
+  (Contributed by Serhiy Storchaka in :gh:`66788`.)
+
+
 gzip
 ----
 

Lib/encodings/aliases.py

@@ -600,6 +600,10 @@
     'utf7'               : 'utf_7',
     'unicode_1_1_utf_7'  : 'utf_7',
 
+    # utf_7_imap codec
+    'csutf7imap'         : 'utf_7_imap',
+    'mutf_7'             : 'utf_7_imap',
+
     # utf_8 codec
     'csutf8'             : 'utf_8',
     'u8'                 : 'utf_8',

Lib/encodings/utf_7_imap.py

@@ -0,0 +1,130 @@
+""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names,
+as specified in RFC 3501, section 5.1.3.
+
+It differs from UTF-7 (RFC 2152) as follows:
+
+* "&" (not "+") is the shift character introducing a Base64 sequence,
+  and "&-" encodes a literal "&";
+* "," is used instead of "/" in the modified Base64 alphabet;
+* only printable US-ASCII characters (except "&") may be represented
+  directly, so all other characters, including other controls, are
+  Base64-encoded.
+"""
+
+import binascii
+import codecs
+
+# The modified Base64 alphabet of RFC 3501: standard Base64 but with "," in
+# place of "/".
+_alphabet = binascii.BASE64_ALPHABET[:-1] + b','
+
+### Codec APIs
+
+def utf_7_imap_encode(input, errors='strict'):
+    if errors != 'strict':
+        raise UnicodeError(f"Unsupported error handling: {errors}")
+    res = bytearray()
+    start = 0                   # start of the current run
+    b64run = False              # is input[start:i] a Base64 run?
+    def flush(end):
+        if start < end:
+            if b64run:
+                b64 = binascii.b2a_base64(input[start:end].encode('utf-16-be'),
+                                          alphabet=_alphabet, padded=False,
+                                          newline=False)
+                res.extend(b'&' + b64 + b'-')
+            else:
+                res.extend(input[start:end].encode('ascii'))
+    for i, ch in enumerate(input):
+        if ch == '&':
+            flush(i)
+            res.extend(b'&-')
+            start = i + 1
+            b64run = False
+        elif ' ' <= ch <= '~':      # printable ASCII, represented directly
+            if b64run:
+                flush(i)
+                start = i
+                b64run = False
+        else:                        # everything else is Base64-encoded
+            if not b64run:
+                flush(i)
+                start = i
+                b64run = True
+    flush(len(input))
+    return res.take_bytes(), len(input)
+
+def utf_7_imap_decode(input, errors='strict'):
+    if errors != 'strict':
+        raise UnicodeError(f"Unsupported error handling: {errors}")
+    input = bytes(input)
+    res = []
+    start = 0                   # start of the current direct ASCII run
+    i = 0
+    n = len(input)
+    def flush(end):
+        if start < end:
+            res.append(input[start:end].decode('ascii'))
+    while i < n:
+        c = input[i]
+        if c == b'&'[0]:
+            flush(i)
+            j = input.find(b'-', i + 1)
+            if j < 0:
+                raise UnicodeDecodeError('utf-7-imap', input, i, n,
+                                         'unterminated shift sequence')
+            if j == i + 1:          # '&-'
+                res.append('&')
+            else:
+                b64 = input[i + 1:j]
+                try:
+                    data = binascii.a2b_base64(b64, alphabet=_alphabet,
+                                               strict_mode=True, padded=False)
+                except binascii.Error:
+                    data = b''
+                if not data or len(data) % 2:
+                    raise UnicodeDecodeError('utf-7-imap', input, i, j + 1,
+                                             'invalid shift sequence')
+                res.append(data.decode('utf-16-be'))
+            i = j + 1
+            start = i
+        elif b' '[0] <= c <= b'~'[0]:
+            i += 1
+        else:
+            raise UnicodeDecodeError('utf-7-imap', input, i, i + 1,
+                                     'unexpected byte')
+    flush(n)
+    return ''.join(res), len(input)
+
+class Codec(codecs.Codec):
+    def encode(self, input, errors='strict'):
+        return utf_7_imap_encode(input, errors)
+    def decode(self, input, errors='strict'):
+        return utf_7_imap_decode(input, errors)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return utf_7_imap_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    def decode(self, input, final=False):
+        return utf_7_imap_decode(input, self.errors)[0]
+
+class StreamWriter(Codec, codecs.StreamWriter):
+    pass
+
+class StreamReader(Codec, codecs.StreamReader):
+    pass
+
+### encodings module API
+
+def getregentry():
+    return codecs.CodecInfo(
+        name='utf-7-imap',
+        encode=Codec().encode,
+        decode=Codec().decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamwriter=StreamWriter,
+        streamreader=StreamReader,
+    )

Lib/test/test_codecs.py

@@ -1401,6 +1401,73 @@ def test_decode_invalid(self):
                     self.assertEqual(puny.decode("punycode", errors), expected)
 
 
+utf_7_imap_testcases = [
+    # (unicode, modified UTF-7)
+    ('', b''),
+    ('INBOX', b'INBOX'),
+    # Printable US-ASCII represents itself.
+    ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'),
+    # "&" is escaped as "&-".
+    ('&', b'&-'),
+    ('&&', b'&-&-'),
+    ('A&B', b'A&-B'),
+    # "+" and "+-" are literal, unlike in UTF-7 where "+" is the shift character.
+    ('+', b'+'),
+    ('+-', b'+-'),
+    # RFC 3501 section 5.1.3 example.
+    ('~peter/mail/台北/日本語',
+     b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'),
+    # Non-printable ASCII (including TAB) is Base64-encoded, not direct.
+    ('a\tb', b'a&AAk-b'),
+    ('\x00', b'&AAA-'),
+    ('Entw\xfcrfe', b'Entw&APw-rfe'),
+    ('ϰ', b'&A,A-'),                     # "," in the Base64 alphabet ("&A/A-" is invalid)
+    ('☃', b'&JgM-'),                     # snowman
+    ('\U0001f600', b'&2D3eAA-'),              # non-BMP (surrogate pair)
+    ('Sent &\N{DELETE}', b'Sent &-&AH8-'),
+]
+
+class Utf7ImapTest(unittest.TestCase):
+    @support.subTests('uni,encoded', utf_7_imap_testcases)
+    def test_encode(self, uni, encoded):
+        self.assertEqual(uni.encode('utf-7-imap'), encoded)
+
+    @support.subTests('uni,encoded', utf_7_imap_testcases)
+    def test_decode(self, uni, encoded):
+        self.assertEqual(encoded.decode('utf-7-imap'), uni)
+
+    # 'start' is the position of the first offending byte in each case.
+    @support.subTests('encoded,start', [
+        (b'x&', 1),             # "&" just before the end, unterminated
+        (b'&AAAA', 0),          # unterminated, though the Base64 is valid
+        (b'&AB-', 0),           # Base64 length not a multiple of a code unit
+        (b'&A/A-', 0),          # "/" not in the alphabet ("&A,A-" is valid)
+        (b'&@@@-', 0),          # invalid Base64
+        (b'a\x80b', 1),         # 8-bit byte outside a shift sequence
+        (b'a\x1fb', 1),         # control byte outside a shift sequence
+    ])
+    def test_decode_invalid(self, encoded, start):
+        with self.assertRaises(UnicodeDecodeError) as cm:
+            encoded.decode('utf-7-imap')
+        self.assertEqual(cm.exception.encoding, 'utf-7-imap')
+        self.assertEqual(cm.exception.start, start)
+
+    def test_encode_lone_surrogate(self):
+        with self.assertRaises(UnicodeEncodeError):
+            '\ud800'.encode('utf-7-imap')
+
+    def test_only_strict_errors(self):
+        with self.assertRaises(UnicodeError):
+            'x'.encode('utf-7-imap', 'replace')
+        with self.assertRaises(UnicodeError):
+            b'x'.decode('utf-7-imap', 'ignore')
+
+    def test_stateless(self):
+        # The codec is registered and exposes the standard interface.
+        info = codecs.lookup('utf-7-imap')
+        self.assertEqual(info.name, 'utf-7-imap')
+
+
 # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
 nameprep_tests = [
     # 3.1 Map to nothing.

Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst

@@ -0,0 +1,2 @@
+Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding
+used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3).