From cb57ba814a91b74908c27689ede2af121952a0be Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 18:31:10 +0300 Subject: [PATCH 1/3] gh-155247: Use raw string content for the extensions cache key The key of the extensions cache was built with _PyUnicode_AsUTF8NoNUL(), so importing an extension module failed with UnicodeEncodeError if its path contained characters unencodable in UTF-8, e.g. surrogate escapes of an undecodable file name. The raw content of the strings is now used. The rejection of embedded null characters, which was a side effect of the UTF-8 encoding, is now explicit in the extension module loader. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_import/__init__.py | 28 +++++++ ...-08-05-19-10-00.gh-issue-155247.extKey.rst | 4 + Python/import.c | 74 +++++++++++-------- Python/importdl.c | 9 ++- 4 files changed, 84 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 9f6dec7d1c5802c..5140bc3cd7393fe 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1262,6 +1262,34 @@ class Spec2: origin = "a\x00b" _imp.create_dynamic(Spec2()) + @unittest.skipUnless(_testsinglephase is not None, + 'requires _testsinglephase') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'requires undecodable file names') + def test_import_from_undecodable_path(self): + # gh-155247: the path of the extension module is not encodable + # in UTF-8. + origin = _testsinglephase.__file__ + # The module is cached by its path, so restore it afterwards. + self.addCleanup(restore__testsinglephase) + with os_helper.temp_dir() as tempdir: + subdir = os.path.join(os.fsencode(tempdir), + os_helper.TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) + shutil.copyfile(origin, path) + path = os.fsdecode(path) + spec = importlib.util.spec_from_file_location('_testsinglephase', + path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertEqual(module.__name__, '_testsinglephase') + self.assertEqual(module.__file__, path) + _testinternalcapi.clear_extension('_testsinglephase', path) + def test_create_builtin(self): class Spec: pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst new file mode 100644 index 000000000000000..1e08323008d5b7a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst @@ -0,0 +1,4 @@ +Fix importing an extension module whose path contains characters unencodable +in UTF-8, e.g. undecodable bytes of a file name. Previously it failed with +:exc:`UnicodeEncodeError`, which made it impossible to build or run Python in +a directory with such name. diff --git a/Python/import.c b/Python/import.c index 5ca78a971fa54c6..15bc696e440a8fe 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1276,50 +1276,66 @@ del_extensions_cache_value(void *raw) } } +/* The key of the extensions cache: the raw content of two strings. + + The UTF-8 encoding is not used, because the strings can contain lone + surrogates, e.g. a file name undecodable in the filesystem encoding. */ +struct hashtable_key { + size_t size; /* the total size of the key */ + unsigned char kind1; + unsigned char kind2; + Py_ssize_t len1; + /* followed by the raw content of both strings */ +}; + static void * -hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) +hashtable_key_from_2_strings(PyObject *str1, PyObject *str2) { - const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1); - const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2); - if (str1_data == NULL || str2_data == NULL) { - return NULL; - } - Py_ssize_t str1_len = strlen(str1_data); - Py_ssize_t str2_len = strlen(str2_data); + Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1); + Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2); + int kind1 = PyUnicode_KIND(str1); + int kind2 = PyUnicode_KIND(str2); + size_t size1 = (size_t)len1 * kind1; + size_t size2 = (size_t)len2 * kind2; - /* Make sure sep and the NULL byte won't cause an overflow. */ - assert(SIZE_MAX - str1_len - str2_len > 2); - size_t size = str1_len + 1 + str2_len + 1; + assert(SIZE_MAX - sizeof(struct hashtable_key) - size1 > size2); + size_t size = sizeof(struct hashtable_key) + size1 + size2; // XXX Use a buffer if it's a temp value (every case but "set"). - char *key = PyMem_RawMalloc(size); + struct hashtable_key *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); return NULL; } - memcpy(key, str1_data, str1_len); - key[str1_len] = sep; - memcpy(key + str1_len + 1, str2_data, str2_len); - key[size - 1] = '\0'; - assert(strlen(key) == size - 1); + key->size = size; + key->kind1 = (unsigned char)kind1; + key->kind2 = (unsigned char)kind2; + key->len1 = len1; + char *data = (char *)(key + 1); + memcpy(data, PyUnicode_DATA(str1), size1); + memcpy(data + size1, PyUnicode_DATA(str2), size2); return key; } static Py_uhash_t -hashtable_hash_str(const void *key) +hashtable_hash_key(const void *key) { - return Py_HashBuffer(key, strlen((const char *)key)); + return Py_HashBuffer(key, ((const struct hashtable_key *)key)->size); } static int -hashtable_compare_str(const void *key1, const void *key2) +hashtable_compare_key(const void *key1, const void *key2) { - return strcmp((const char *)key1, (const char *)key2) == 0; + size_t size = ((const struct hashtable_key *)key1)->size; + if (size != ((const struct hashtable_key *)key2)->size) { + return 0; + } + return memcmp(key1, key2, size) == 0; } static void -hashtable_destroy_str(void *ptr) +hashtable_destroy_key(void *ptr) { PyMem_RawFree(ptr); } @@ -1359,16 +1375,15 @@ _find_cached_def(PyModuleDef *def) } #endif -#define HTSEP ':' static int _extensions_cache_init(void) { _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key + hashtable_hash_key, + hashtable_compare_key, + hashtable_destroy_key, // key del_extensions_cache_value, // value &alloc ); @@ -1386,7 +1401,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name, if (EXTENSIONS.hashtable == NULL) { return NULL; } - void *key = hashtable_key_from_2_strings(path, name, HTSEP); + void *key = hashtable_key_from_2_strings(path, name); if (key == NULL) { return NULL; } @@ -1396,7 +1411,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name, *p_key = key; } else { - hashtable_destroy_str(key); + hashtable_destroy_key(key); } return entry; } @@ -1534,7 +1549,7 @@ _extensions_cache_set(PyObject *path, PyObject *name, finally_oldvalue: extensions_lock_release(); if (key != NULL) { - hashtable_destroy_str(key); + hashtable_destroy_key(key); } return value; @@ -1578,7 +1593,6 @@ _extensions_cache_clear_all(void) EXTENSIONS.hashtable = NULL; } -#undef HTSEP static bool diff --git a/Python/importdl.c b/Python/importdl.c index 537e8d869dc93ca..011612d97a5f48d 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -8,6 +8,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() /***********************************/ @@ -117,7 +118,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, return -1; } - info.newcontext = PyUnicode_AsUTF8(info.name); + info.newcontext = _PyUnicode_AsUTF8NoNUL(info.name); if (info.newcontext == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -130,6 +131,12 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, _Py_ext_module_loader_info_clear(&info); return -1; } + if (PyUnicode_FindChar(filename, 0, 0, + PyUnicode_GET_LENGTH(filename), 1) != -1) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + _Py_ext_module_loader_info_clear(&info); + return -1; + } info.filename = Py_NewRef(filename); #ifndef MS_WINDOWS From faa4640cc68bd83e703d700276e1e7bb10684f7b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 19:19:36 +0300 Subject: [PATCH 2/3] Fix the test on Windows A loaded extension module cannot be removed on Windows, so the temporary directory is now removed with ignore_errors. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_import/__init__.py | 37 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 5140bc3cd7393fe..f737b60271e8483 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,6 +20,7 @@ import stat import subprocess import sys +import tempfile import textwrap import threading import time @@ -1272,23 +1273,25 @@ def test_import_from_undecodable_path(self): origin = _testsinglephase.__file__ # The module is cached by its path, so restore it afterwards. self.addCleanup(restore__testsinglephase) - with os_helper.temp_dir() as tempdir: - subdir = os.path.join(os.fsencode(tempdir), - os_helper.TESTFN_UNDECODABLE) - try: - os.mkdir(subdir) - except OSError: - self.skipTest('undecodable paths are not supported') - path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) - shutil.copyfile(origin, path) - path = os.fsdecode(path) - spec = importlib.util.spec_from_file_location('_testsinglephase', - path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - self.assertEqual(module.__name__, '_testsinglephase') - self.assertEqual(module.__file__, path) - _testinternalcapi.clear_extension('_testsinglephase', path) + tempdir = tempfile.mkdtemp() + # The copied extension module stays loaded, so on Windows it cannot + # be removed. + self.addCleanup(shutil.rmtree, tempdir, ignore_errors=True) + subdir = os.path.join(os.fsencode(tempdir), + os_helper.TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) + shutil.copyfile(origin, path) + path = os.fsdecode(path) + spec = importlib.util.spec_from_file_location('_testsinglephase', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.assertEqual(module.__name__, '_testsinglephase') + self.assertEqual(module.__file__, path) + _testinternalcapi.clear_extension('_testsinglephase', path) def test_create_builtin(self): class Spec: From 7d20c518782fcc994e2a6f2d1826b06f76354094 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Aug 2026 21:18:52 +0300 Subject: [PATCH 3/3] Clear the padding in the key and fix the test on Windows The key is hashed and compared as raw bytes, so the uninitialized padding of the header made lookups miss at random. Import the extension module in a subprocess: it stays loaded, and on Windows its file cannot be removed, so the temporary directory leaked. --- Lib/test/test_import/__init__.py | 44 ++++++++++++++++---------------- Python/import.c | 2 ++ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index f737b60271e8483..46aaa4d1a0b8a19 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,7 +20,6 @@ import stat import subprocess import sys -import tempfile import textwrap import threading import time @@ -1271,27 +1270,28 @@ def test_import_from_undecodable_path(self): # gh-155247: the path of the extension module is not encodable # in UTF-8. origin = _testsinglephase.__file__ - # The module is cached by its path, so restore it afterwards. - self.addCleanup(restore__testsinglephase) - tempdir = tempfile.mkdtemp() - # The copied extension module stays loaded, so on Windows it cannot - # be removed. - self.addCleanup(shutil.rmtree, tempdir, ignore_errors=True) - subdir = os.path.join(os.fsencode(tempdir), - os_helper.TESTFN_UNDECODABLE) - try: - os.mkdir(subdir) - except OSError: - self.skipTest('undecodable paths are not supported') - path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) - shutil.copyfile(origin, path) - path = os.fsdecode(path) - spec = importlib.util.spec_from_file_location('_testsinglephase', path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - self.assertEqual(module.__name__, '_testsinglephase') - self.assertEqual(module.__file__, path) - _testinternalcapi.clear_extension('_testsinglephase', path) + with os_helper.temp_dir() as tempdir: + subdir = os.path.join(os.fsencode(tempdir), + os_helper.TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) + shutil.copyfile(origin, path) + # Import it in a subprocess: the extension module stays loaded, + # and on Windows its file cannot be removed. + script = textwrap.dedent(f""" + import importlib.util + path = {os.fsdecode(path)!a} + spec = importlib.util.spec_from_file_location( + '_testsinglephase', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert module.__name__ == '_testsinglephase', module.__name__ + assert module.__file__ == path, module.__file__ + """) + script_helper.assert_python_ok('-c', script) def test_create_builtin(self): class Spec: diff --git a/Python/import.c b/Python/import.c index 15bc696e440a8fe..87476917fa11252 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1308,6 +1308,8 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2) return NULL; } + /* Clear the padding: the key is hashed and compared as raw bytes. */ + memset(key, 0, sizeof(struct hashtable_key)); key->size = size; key->kind1 = (unsigned char)kind1; key->kind2 = (unsigned char)kind2;