diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index 18f5da70caf..169c48324f2 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -26,10 +26,11 @@ (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. -"""#" +""" import codecs import sys +from _codecs import _normalize_encoding from . import aliases _cache = {} @@ -62,18 +63,7 @@ def normalize_encoding(encoding): "Support for non-ascii encoding names will be removed in 3.17", DeprecationWarning, stacklevel=2) - chars = [] - punct = False - for c in encoding: - if c.isalnum() or c == '.': - if punct and chars: - chars.append('_') - if c.isascii(): - chars.append(c) - punct = False - else: - punct = True - return ''.join(chars) + return _normalize_encoding(encoding) def search_function(encoding): diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index a17c025f50c..23e92f84797 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -20,6 +20,36 @@ mod _codecs { function::ArgBytesLike, }; + #[derive(FromArgs)] + struct NormalizeEncodingArgs { + #[pyarg(any)] + encoding: PyStrRef, + } + + #[pyfunction] + fn _normalize_encoding(args: NormalizeEncodingArgs, vm: &VirtualMachine) -> PyResult { + let encoding = args.encoding; + // CPython converts the entire input to UTF-8 before scanning for NUL. + if !encoding.is_utf8() { + let ctx = PyEncodeContext::new(encodings::utf8::ENCODING_NAME, &encoding, vm); + encodings::utf8::encode(ctx, &encodings::errors::Strict)?; + } + let mut normalized = String::with_capacity(encoding.byte_len()); + let mut punct = false; + for byte in encoding.as_bytes().iter().copied().take_while(|b| *b != 0) { + if byte.is_ascii_alphanumeric() || byte == b'.' { + if punct && !normalized.is_empty() { + normalized.push('_'); + } + normalized.push(char::from(byte)); + punct = false; + } else { + punct = true; + } + } + Ok(normalized) + } + #[pyfunction] fn register(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { vm.state.codec_registry.register(search_function, vm) diff --git a/extra_tests/snippets/stdlib_codecs_wide.py b/extra_tests/snippets/stdlib_codecs_wide.py index 0c45cc91cca..bd8307ffcfb 100644 --- a/extra_tests/snippets/stdlib_codecs_wide.py +++ b/extra_tests/snippets/stdlib_codecs_wide.py @@ -64,3 +64,79 @@ def record_unknown_encoding(name): assert seen_encoding_names[-1] == normalized finally: codecs.unregister(record_unknown_encoding) + + +# The encodings helper preserves case and dots, unlike registry lookup keys. +import encodings +import warnings + + +class EncodingName(str): + pass + + +for encoding, expected in ( + ("", ""), + ("UTF-8", "UTF_8"), + (" .UTF--8. ", ".UTF_8."), + ("---", ""), + ("A__-- B", "A_B"), + ("AéB", "A_B"), + ("éUTF-8Ω", "UTF_8"), + ("A😀B", "A_B"), + ("UTF-8\x00ignored", "UTF_8"), + ("\x00UTF-8", ""), +): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert _codecs._normalize_encoding(encoding) == expected + assert _codecs._normalize_encoding(encoding=EncodingName(encoding)) == expected + assert not caught + +for encoding in (b"UTF-8", bytearray(b"UTF-8"), None, 1): + try: + _codecs._normalize_encoding(encoding) + except TypeError: + pass + else: + assert False, "non-string encoding was accepted" + +# UTF-8 validation precedes scanning, including a surrogate after NUL. +for encoding in ("A\ud800B", "A\x00\ud800B"): + try: + _codecs._normalize_encoding(encoding) + except UnicodeEncodeError as error: + assert error.encoding == "utf-8" + assert error.object == encoding + assert error.start == encoding.index("\ud800") + assert error.end == error.start + 1 + else: + assert False, "surrogate encoding was accepted" + +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert encodings.normalize_encoding(b"UTF-8") == "UTF_8" + assert not caught + assert encodings.normalize_encoding("AéB") == "A_B" + assert len(caught) == 1 + assert caught[0].category is DeprecationWarning + +try: + encodings.normalize_encoding(b"UTF-\xff") +except UnicodeDecodeError: + pass +else: + assert False, "non-ASCII bytes encoding was accepted" + +for args, kwargs in ( + ((), {}), + (("UTF-8", "ASCII"), {}), + (("UTF-8",), {"encoding": "ASCII"}), + ((), {"name": "UTF-8"}), +): + try: + _codecs._normalize_encoding(*args, **kwargs) + except TypeError: + pass + else: + assert False, "invalid normalization arguments were accepted"