From 76e9ff2f7faf99d6810205853fec486329158f06 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Aug 2026 13:12:27 +0300 Subject: [PATCH] gh-86427: Replace PyConfig.stdio_encoding with three separate members PyConfig.stdin_encoding, PyConfig.stdout_encoding and PyConfig.stderr_encoding allow the standard streams to have different encodings. In the legacy Windows stdio mode they are initialized with the encoding of the device the corresponding stream is connected to, instead of the ANSI code page. --- Doc/c-api/init_config.rst | 18 ++-- Doc/library/sys.rst | 3 +- Include/cpython/initconfig.h | 4 +- Lib/test/test_capi/test_config.py | 4 +- Lib/test/test_cmd_line.py | 36 +++++++ Lib/test/test_embed.py | 25 +++-- ...08-09-06-30-00.gh-issue-86427.threeenc.rst | 5 + ...09-06-40-00.gh-issue-86427.legacystdio.rst | 3 + Objects/unicodeobject.c | 11 +- Programs/_testembed.c | 8 +- Python/initconfig.c | 102 +++++++++++++++--- Python/pylifecycle.c | 6 +- 12 files changed, 189 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index ef09639189a6c27..0e9685d34865788 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -477,8 +477,12 @@ Configuration Options - :c:member:`skip_source_first_line ` - ``bool`` - Read-only - * - ``"stdio_encoding"`` - - :c:member:`stdio_encoding ` + * - ``"stderr_encoding"`` + - :c:member:`stderr_encoding ` + * - ``"stdin_encoding"`` + - :c:member:`stdin_encoding ` + * - ``"stdout_encoding"`` + - :c:member:`stdout_encoding ` - ``str`` - Read-only * - ``"stdio_errors"`` @@ -1863,12 +1867,14 @@ PyConfig Default: ``0``. - .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdin_encoding + .. c:member:: wchar_t* stdout_encoding + .. c:member:: wchar_t* stderr_encoding .. c:member:: wchar_t* stdio_errors - Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr` (but :data:`sys.stderr` always uses - ``"backslashreplace"`` error handler). + Encoding of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr` + respectively, and encoding errors of all of them (but :data:`sys.stderr` + always uses the ``"backslashreplace"`` error handler). Use the :envvar:`PYTHONIOENCODING` environment variable if it is non-empty. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a2668a38c6b4a2f..2fd89597498dc34 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2136,7 +2136,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only follows: * The encoding and error handling are initialized from - :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. + :c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding`, + :c:member:`PyConfig.stderr_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character devices such as disk files and pipes use the system locale diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1ccc496c63ac780..258649ad6069870 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -171,7 +171,9 @@ typedef struct PyConfig { int user_site_directory; int configure_c_stdio; int buffered_stdio; - wchar_t *stdio_encoding; + wchar_t *stdin_encoding; + wchar_t *stdout_encoding; + wchar_t *stderr_encoding; wchar_t *stdio_errors; #ifdef MS_WINDOWS int legacy_windows_stdio; diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 290126343618381..507ba3335a67b73 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -86,7 +86,9 @@ def test_config_get(self): ("show_ref_count", bool, None), ("site_import", bool, None), ("skip_source_first_line", bool, None), - ("stdio_encoding", str, None), + ("stderr_encoding", str, None), + ("stdin_encoding", str, None), + ("stdout_encoding", str, None), ("stdio_errors", str, None), ("stdlib_dir", str | None, "_stdlib_dir"), ("tracemalloc", int, None), diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 25d6d1a248b4577..caa8939f0cc211f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1067,6 +1067,42 @@ def test_python_legacy_windows_stdio(self): support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio_encoding(self): + # gh-86427: In the legacy mode the encoding of a standard stream is + # the encoding of the console it is connected to, which can differ + # for input and output. + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + try: + fin = open('CONIN$') + except OSError: + self.skipTest('no console') + # We cannot use PIPE, because the standard streams should be + # connected to the console. So we use the exit code. + code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or " + "sys.stdout.encoding != 'cp437')") + env = os.environ.copy() + env['PYTHONLEGACYWINDOWSSTDIO'] = '1' + env['PYTHONUTF8'] = '0' + env.pop('PYTHONIOENCODING', None) + old_cp = kernel32.GetConsoleCP() + old_output_cp = kernel32.GetConsoleOutputCP() + with fin, open('CONOUT$', 'w') as fout: + try: + if not kernel32.SetConsoleCP(850): + self.skipTest('cannot set the console input code page') + if not kernel32.SetConsoleOutputCP(437): + self.skipTest('cannot set the console output code page') + proc = subprocess.run([sys.executable, '-c', code], env=env, + stdin=fin, stdout=fout, + stderr=subprocess.DEVNULL) + finally: + kernel32.SetConsoleCP(old_cp) + kernel32.SetConsoleOutputCP(old_output_cp) + support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode) + self.assertEqual(proc.returncode, 0) + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") def test_python_malloc_stats(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1ff600e30bf4cbd..c776089ddd0d59b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -767,7 +767,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'configure_c_stdio': False, 'buffered_stdio': True, - 'stdio_encoding': GET_DEFAULT_CONFIG, + 'stdin_encoding': GET_DEFAULT_CONFIG, + 'stdout_encoding': GET_DEFAULT_CONFIG, + 'stderr_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, 'skip_source_first_line': False, @@ -934,7 +936,8 @@ def get_expected_config(self, expected_preconfig, expected, # there is no easy way to get the locale encoding before # setlocale(LC_CTYPE, "") is called: don't test encodings for key in ('filesystem_encoding', 'filesystem_errors', - 'stdio_encoding', 'stdio_errors'): + 'stdin_encoding', 'stdout_encoding', + 'stderr_encoding', 'stdio_errors'): expected[key] = self.IGNORE_CONFIG if expected_preconfig['utf8_mode'] == 1: @@ -942,8 +945,10 @@ def get_expected_config(self, expected_preconfig, expected, expected['filesystem_encoding'] = 'utf-8' if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG: expected['filesystem_errors'] = self.UTF8_MODE_ERRORS - if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG: - expected['stdio_encoding'] = 'utf-8' + for key in ('stdin_encoding', 'stdout_encoding', + 'stderr_encoding'): + if expected[key] is self.GET_DEFAULT_CONFIG: + expected[key] = 'utf-8' if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: expected['stdio_errors'] = 'surrogateescape' @@ -1133,7 +1138,9 @@ def test_init_from_config(self): 'malloc_stats': True, 'pymalloc_hugepages': True, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'pycache_prefix': 'conf_pycache_prefix', @@ -1205,7 +1212,9 @@ def test_init_compat_env(self): 'write_bytecode': False, 'verbose': 1, 'buffered_stdio': False, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'user_site_directory': False, 'faulthandler': True, @@ -1242,7 +1251,9 @@ def test_init_python_env(self): 'write_bytecode': False, 'verbose': 1, 'buffered_stdio': False, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'user_site_directory': False, 'faulthandler': True, diff --git a/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst b/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst new file mode 100644 index 000000000000000..e61839e5312540a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst @@ -0,0 +1,5 @@ +Replace :c:member:`!PyConfig.stdio_encoding` with three separate members: +:c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding` and +:c:member:`PyConfig.stderr_encoding`. In the legacy Windows stdio mode they +are initialized with the encoding of the device the corresponding stream is +connected to. diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst b/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst new file mode 100644 index 000000000000000..c3e2332defabfa9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst @@ -0,0 +1,3 @@ +Fix the encoding of the standard streams in the legacy Windows stdio mode +(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device +the stream is connected to, as in Python 3.7, not the ANSI code page. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 45d61c8b8b765a6..33b271ebe50fb12 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15209,9 +15209,14 @@ init_stdio_encoding(PyInterpreterState *interp) { /* Update the stdio encoding to the normalized Python codec name. */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); - if (config_get_codec_name(&config->stdio_encoding) < 0) { - return _PyStatus_ERR("failed to get the Python codec name " - "of the stdio encoding"); + wchar_t **encodings[] = {&config->stdin_encoding, + &config->stdout_encoding, + &config->stderr_encoding}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) { + if (config_get_codec_name(encodings[i]) < 0) { + return _PyStatus_ERR("failed to get the Python codec name " + "of the stdio encoding"); + } } return _PyStatus_OK(); } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 418609abc5f6b82..bbf6d9e38bdba60 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -302,7 +302,9 @@ static void check_stdio_details(const wchar_t *encoding, const wchar_t *errors) _PyConfig_InitCompatConfig(&config); /* Force the given IO encoding */ if (encoding) { - config_set_string(&config, &config.stdio_encoding, encoding); + config_set_string(&config, &config.stdin_encoding, encoding); + config_set_string(&config, &config.stdout_encoding, encoding); + config_set_string(&config, &config.stderr_encoding, encoding); } if (errors) { config_set_string(&config, &config.stdio_errors, errors); @@ -770,7 +772,9 @@ static int test_init_from_config(void) config.buffered_stdio = 0; putenv("PYTHONIOENCODING=cp424"); - config_set_string(&config, &config.stdio_encoding, L"iso8859-1"); + config_set_string(&config, &config.stdin_encoding, L"iso8859-1"); + config_set_string(&config, &config.stdout_encoding, L"iso8859-1"); + config_set_string(&config, &config.stderr_encoding, L"iso8859-1"); config_set_string(&config, &config.stdio_errors, L"replace"); putenv("PYTHONNOUSERSITE="); diff --git a/Python/initconfig.c b/Python/initconfig.c index b9bacb17a66454a..85ddb9b1d93d9ad 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -196,7 +196,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stderr_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdin_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdout_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), @@ -1074,7 +1076,9 @@ config_check_consistency(const PyConfig *config) assert(config->module_search_paths_set >= 0); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); + assert(config->stdin_encoding != NULL); + assert(config->stdout_encoding != NULL); + assert(config->stderr_encoding != NULL); assert(config->stdio_errors != NULL); #ifdef MS_WINDOWS assert(config->legacy_windows_stdio >= 0); @@ -1140,7 +1144,9 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); - CLEAR(config->stdio_encoding); + CLEAR(config->stdin_encoding); + CLEAR(config->stdout_encoding); + CLEAR(config->stderr_encoding); CLEAR(config->stdio_errors); CLEAR(config->run_command); CLEAR(config->run_module); @@ -2646,6 +2652,62 @@ config_get_locale_encoding(PyConfig *config, const PyPreConfig *preconfig, } +#ifdef MS_WINDOWS +/* The encoding of the device, or the locale encoding if it is not + a console. See also _Py_device_encoding(). */ +static PyStatus +config_get_device_encoding(PyConfig *config, const PyPreConfig *preconfig, + int fd, wchar_t **result) +{ + UINT cp = 0; + HANDLE handle = (HANDLE)_Py_get_osfhandle_noraise(fd); + if (handle != INVALID_HANDLE_VALUE && GetFileType(handle) == FILE_TYPE_CHAR) { + DWORD temp; + /* GetConsoleMode() only succeeds for a console handle. */ + if (!GetConsoleMode(handle, &temp)) { + /* Assume that access denied implies an output handle. */ + if (GetLastError() == ERROR_ACCESS_DENIED) { + cp = GetConsoleOutputCP(); + } + } + else if (GetNumberOfConsoleInputEvents(handle, &temp)) { + cp = GetConsoleCP(); + } + else { + cp = GetConsoleOutputCP(); + } + } + if (cp == 0) { + return config_get_locale_encoding(config, preconfig, result); + } + if (cp == CP_UTF8) { + return PyConfig_SetString(config, result, L"utf-8"); + } + wchar_t encoding[20]; + swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", (unsigned int)cp); + return PyConfig_SetString(config, result, encoding); +} +#endif + + +/* The default encoding of the standard stream with the file descriptor fd. */ +static PyStatus +config_get_stdio_encoding(PyConfig *config, const PyPreConfig *preconfig, + int fd, wchar_t **result) +{ + if (*result != NULL) { + return _PyStatus_OK(); + } +#ifdef MS_WINDOWS + if (config->legacy_windows_stdio) { + /* gh-86427: the standard streams are ordinary files here. */ + return config_get_device_encoding(config, preconfig, fd, result); + } +#endif + return config_get_locale_encoding(config, preconfig, result); +} + + static PyStatus config_init_stdio_encoding(PyConfig *config, const PyPreConfig *preconfig) @@ -2653,7 +2715,9 @@ config_init_stdio_encoding(PyConfig *config, PyStatus status; // Exit if encoding and errors are defined - if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { + if (config->stdin_encoding != NULL && config->stdout_encoding != NULL + && config->stderr_encoding != NULL && config->stdio_errors != NULL) + { return _PyStatus_OK(); } @@ -2676,8 +2740,14 @@ config_init_stdio_encoding(PyConfig *config, /* Does PYTHONIOENCODING contain an encoding? */ if (pythonioencoding[0]) { - if (config->stdio_encoding == NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->stdio_encoding, + wchar_t **encodings[] = {&config->stdin_encoding, + &config->stdout_encoding, + &config->stderr_encoding}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) { + if (*encodings[i] != NULL) { + continue; + } + status = CONFIG_SET_BYTES_STR(config, encodings[i], pythonioencoding, "PYTHONIOENCODING environment variable"); if (_PyStatus_EXCEPTION(status)) { @@ -2709,12 +2779,20 @@ config_init_stdio_encoding(PyConfig *config, } /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { - status = config_get_locale_encoding(config, preconfig, - &config->stdio_encoding); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = config_get_stdio_encoding(config, preconfig, 0, + &config->stdin_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = config_get_stdio_encoding(config, preconfig, 1, + &config->stdout_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = config_get_stdio_encoding(config, preconfig, 2, + &config->stderr_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; } if (config->stdio_errors == NULL) { const wchar_t *errors = config_get_stdio_errors(preconfig); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 500a1a1949a5a8a..4e2a14466d317bf 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3186,7 +3186,7 @@ init_sys_streams(PyThreadState *tstate) * GUI apps don't have valid standard streams by default. */ std = create_stdio(config, iomod, fd, 0, "", - config->stdio_encoding, + config->stdin_encoding, config->stdio_errors); if (std == NULL) goto error; @@ -3197,7 +3197,7 @@ init_sys_streams(PyThreadState *tstate) /* Set sys.stdout */ fd = fileno(stdout); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->stdout_encoding, config->stdio_errors); if (std == NULL) goto error; @@ -3209,7 +3209,7 @@ init_sys_streams(PyThreadState *tstate) /* Set sys.stderr, replaces the preliminary stderr */ fd = fileno(stderr); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->stderr_encoding, L"backslashreplace"); if (std == NULL) goto error;