From 6f44a1beab80fcfd1a6696868339a2364e887e28 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 5 Jul 2026 23:37:13 +0300 Subject: [PATCH] gh-66788: Add the imap4-utf-7 codec Implement the modified UTF-7 encoding used for international IMAP4 mailbox names (RFC 3501, section 5.1.3). It differs from UTF-7: "&" is the shift character ("&-" encodes a literal "&"), "," replaces "/" in the Base64 alphabet, and all non-printable-ASCII characters are Base64-encoded. Co-Authored-By: Claude Fable 5 --- Doc/library/codecs.rst | 8 ++ Doc/whatsnew/3.16.rst | 8 ++ Lib/encodings/imap4_utf_7.py | 126 ++++++++++++++++++ Lib/test/test_codecs.py | 67 ++++++++++ ...6-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst | 2 + 5 files changed, 211 insertions(+) create mode 100644 Lib/encodings/imap4_utf_7.py create mode 100644 Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 99fcf35aa893e4..de9e696563c5c1 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1384,6 +1384,14 @@ encodings. | | | Only ``errors='strict'`` | | | | is supported. | +--------------------+---------+---------------------------+ +| imap4-utf-7 | | Modified UTF-7 encoding | +| | | of :rfc:`3501` for IMAP4 | +| | | mailbox names. Only | +| | | ``errors='strict'`` is | +| | | supported. | +| | | | +| | | .. versionadded:: next | ++--------------------+---------+---------------------------+ | mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | | | | ANSI codepage (CP_ACP). | diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e8c530e19d2b53..4b74dc4850005e 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -214,6 +214,14 @@ curses wide-character support. (Contributed by Serhiy Storchaka in :gh:`133031`.) +encodings +--------- + +* Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding + used for international IMAP4 mailbox names (:rfc:`3501`). + (Contributed by Serhiy Storchaka in :gh:`66788`.) + + gzip ---- diff --git a/Lib/encodings/imap4_utf_7.py b/Lib/encodings/imap4_utf_7.py new file mode 100644 index 00000000000000..f0b9df85260e44 --- /dev/null +++ b/Lib/encodings/imap4_utf_7.py @@ -0,0 +1,126 @@ +""" Codec for the modified UTF-7 encoding used for IMAP4 mailbox names, +as specified in RFC 3501, section 5.1.3. + +It differs from UTF-7 (RFC 2152) as follows: + +* "&" (not "+") is the shift character introducing a Base64 sequence, + and "&-" encodes a literal "&"; +* "," is used instead of "/" in the modified Base64 alphabet; +* only printable US-ASCII characters (except "&") may be represented + directly, so all other characters, including other controls, are + Base64-encoded. +""" + +import binascii +import codecs +from base64 import b64encode, b64decode + +### Codec APIs + +def imap4_utf_7_encode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + res = bytearray() + start = 0 # start of the current run + b64run = False # is input[start:i] a Base64 run? + def flush(end): + if start < end: + if b64run: + b64 = b64encode(input[start:end].encode('utf-16-be'), + altchars=b'+,', padded=False) + res.extend(b'&' + b64 + b'-') + else: + res.extend(input[start:end].encode('ascii')) + for i, ch in enumerate(input): + if ch == '&': + flush(i) + res.extend(b'&-') + start = i + 1 + b64run = False + elif ' ' <= ch <= '~': # printable ASCII, represented directly + if b64run: + flush(i) + start = i + b64run = False + else: # everything else is Base64-encoded + if not b64run: + flush(i) + start = i + b64run = True + flush(len(input)) + return res.take_bytes(), len(input) + +def imap4_utf_7_decode(input, errors='strict'): + if errors != 'strict': + raise UnicodeError(f"Unsupported error handling: {errors}") + input = bytes(input) + res = [] + start = 0 # start of the current direct ASCII run + i = 0 + n = len(input) + def flush(end): + if start < end: + res.append(input[start:end].decode('ascii')) + while i < n: + c = input[i] + if c == b'&'[0]: + flush(i) + j = input.find(b'-', i + 1) + if j < 0: + raise UnicodeDecodeError('imap4-utf-7', input, i, n, + 'unterminated shift sequence') + if j == i + 1: # '&-' + res.append('&') + else: + b64 = input[i + 1:j] + try: + data = b64decode(b64, altchars=b'+,', + validate=True, padded=False) + except binascii.Error: + data = b'' + if not data or len(data) % 2: + raise UnicodeDecodeError('imap4-utf-7', input, i, j + 1, + 'invalid shift sequence') + res.append(data.decode('utf-16-be')) + i = j + 1 + start = i + elif b' '[0] <= c <= b'~'[0]: + i += 1 + else: + raise UnicodeDecodeError('imap4-utf-7', input, i, i + 1, + 'unexpected byte') + flush(n) + return ''.join(res), len(input) + +class Codec(codecs.Codec): + def encode(self, input, errors='strict'): + return imap4_utf_7_encode(input, errors) + def decode(self, input, errors='strict'): + return imap4_utf_7_decode(input, errors) + +class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return imap4_utf_7_encode(input, self.errors)[0] + +class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return imap4_utf_7_decode(input, self.errors)[0] + +class StreamWriter(Codec, codecs.StreamWriter): + pass + +class StreamReader(Codec, codecs.StreamReader): + pass + +### encodings module API + +def getregentry(): + return codecs.CodecInfo( + name='imap4-utf-7', + encode=Codec().encode, + decode=Codec().decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 8fdd08df9e4f46..5b911fddd63fd2 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1401,6 +1401,73 @@ def test_decode_invalid(self): self.assertEqual(puny.decode("punycode", errors), expected) +imap4_utf_7_testcases = [ + # (unicode, modified UTF-7) + ('', b''), + ('INBOX', b'INBOX'), + # Printable US-ASCII represents itself. + ('abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~', b'abcABC123 !#$%()*+,-./:;<=>?@[]^_`{|}~'), + # "&" is escaped as "&-". + ('&', b'&-'), + ('&&', b'&-&-'), + ('A&B', b'A&-B'), + # RFC 3501 section 5.1.3 example. + ('~peter/mail/台北/日本語', + b'~peter/mail/&U,BTFw-/&ZeVnLIqe-'), + # Non-printable ASCII (including TAB) is Base64-encoded, not direct. + ('a\tb', b'a&AAk-b'), + ('\x00', b'&AAA-'), + ('Entw\xfcrfe', b'Entw&APw-rfe'), + ('☃', b'&JgM-'), # snowman + ('\U0001f600', b'&2D3eAA-'), # non-BMP (surrogate pair) + ('Sent &\N{DELETE}', b'Sent &-&AH8-'), +] + +class Imap4Utf7Test(unittest.TestCase): + def test_encode(self): + for uni, encoded in imap4_utf_7_testcases: + with self.subTest(uni=uni): + self.assertEqual(uni.encode('imap4-utf-7'), encoded) + + def test_decode(self): + for uni, encoded in imap4_utf_7_testcases: + with self.subTest(encoded=encoded): + self.assertEqual(encoded.decode('imap4-utf-7'), uni) + + def test_decode_invalid(self): + # position of the first offending byte in each case + testcases = [ + (b'&AAk', 0), # unterminated shift sequence + (b'&Jgg', 0), # unterminated shift sequence + (b'&AB-', 0), # Base64 length not a multiple of a code unit + (b'&@@@-', 0), # invalid Base64 + (b'&====-', 0), # invalid Base64 + (b'a\x80b', 1), # 8-bit byte outside a shift sequence + (b'a\x1fb', 1), # control byte outside a shift sequence + ] + for encoded, start in testcases: + with self.subTest(encoded=encoded): + with self.assertRaises(UnicodeDecodeError) as cm: + encoded.decode('imap4-utf-7') + self.assertEqual(cm.exception.encoding, 'imap4-utf-7') + self.assertEqual(cm.exception.start, start) + + def test_encode_lone_surrogate(self): + with self.assertRaises(UnicodeEncodeError): + '\ud800'.encode('imap4-utf-7') + + def test_only_strict_errors(self): + with self.assertRaises(UnicodeError): + 'x'.encode('imap4-utf-7', 'replace') + with self.assertRaises(UnicodeError): + b'x'.decode('imap4-utf-7', 'ignore') + + def test_stateless(self): + # The codec is registered and exposes the standard interface. + info = codecs.lookup('imap4-utf-7') + self.assertEqual(info.name, 'imap4-utf-7') + + # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ # 3.1 Map to nothing. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst new file mode 100644 index 00000000000000..b46adc17bd3e67 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-22-30-00.gh-issue-66788.tM7pQ2.rst @@ -0,0 +1,2 @@ +Add the ``imap4-utf-7`` codec, implementing the modified UTF-7 encoding +used for international IMAP4 mailbox names (:rfc:`3501`, section 5.1.3).