diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index ef09639189a6c27..0e9685d34865788 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -477,8 +477,12 @@ Configuration Options - :c:member:`skip_source_first_line ` - ``bool`` - Read-only - * - ``"stdio_encoding"`` - - :c:member:`stdio_encoding ` + * - ``"stderr_encoding"`` + - :c:member:`stderr_encoding ` + * - ``"stdin_encoding"`` + - :c:member:`stdin_encoding ` + * - ``"stdout_encoding"`` + - :c:member:`stdout_encoding ` - ``str`` - Read-only * - ``"stdio_errors"`` @@ -1863,12 +1867,14 @@ PyConfig Default: ``0``. - .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdin_encoding + .. c:member:: wchar_t* stdout_encoding + .. c:member:: wchar_t* stderr_encoding .. c:member:: wchar_t* stdio_errors - Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr` (but :data:`sys.stderr` always uses - ``"backslashreplace"`` error handler). + Encoding of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr` + respectively, and encoding errors of all of them (but :data:`sys.stderr` + always uses the ``"backslashreplace"`` error handler). Use the :envvar:`PYTHONIOENCODING` environment variable if it is non-empty. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a2668a38c6b4a2f..2fd89597498dc34 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2136,7 +2136,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only follows: * The encoding and error handling are initialized from - :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. + :c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding`, + :c:member:`PyConfig.stderr_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character devices such as disk files and pipes use the system locale diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1ccc496c63ac780..258649ad6069870 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -171,7 +171,9 @@ typedef struct PyConfig { int user_site_directory; int configure_c_stdio; int buffered_stdio; - wchar_t *stdio_encoding; + wchar_t *stdin_encoding; + wchar_t *stdout_encoding; + wchar_t *stderr_encoding; wchar_t *stdio_errors; #ifdef MS_WINDOWS int legacy_windows_stdio; diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 290126343618381..507ba3335a67b73 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -86,7 +86,9 @@ def test_config_get(self): ("show_ref_count", bool, None), ("site_import", bool, None), ("skip_source_first_line", bool, None), - ("stdio_encoding", str, None), + ("stderr_encoding", str, None), + ("stdin_encoding", str, None), + ("stdout_encoding", str, None), ("stdio_errors", str, None), ("stdlib_dir", str | None, "_stdlib_dir"), ("tracemalloc", int, None), diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 25d6d1a248b4577..caa8939f0cc211f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1067,6 +1067,42 @@ def test_python_legacy_windows_stdio(self): support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio_encoding(self): + # gh-86427: In the legacy mode the encoding of a standard stream is + # the encoding of the console it is connected to, which can differ + # for input and output. + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + try: + fin = open('CONIN$') + except OSError: + self.skipTest('no console') + # We cannot use PIPE, because the standard streams should be + # connected to the console. So we use the exit code. + code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or " + "sys.stdout.encoding != 'cp437')") + env = os.environ.copy() + env['PYTHONLEGACYWINDOWSSTDIO'] = '1' + env['PYTHONUTF8'] = '0' + env.pop('PYTHONIOENCODING', None) + old_cp = kernel32.GetConsoleCP() + old_output_cp = kernel32.GetConsoleOutputCP() + with fin, open('CONOUT$', 'w') as fout: + try: + if not kernel32.SetConsoleCP(850): + self.skipTest('cannot set the console input code page') + if not kernel32.SetConsoleOutputCP(437): + self.skipTest('cannot set the console output code page') + proc = subprocess.run([sys.executable, '-c', code], env=env, + stdin=fin, stdout=fout, + stderr=subprocess.DEVNULL) + finally: + kernel32.SetConsoleCP(old_cp) + kernel32.SetConsoleOutputCP(old_output_cp) + support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode) + self.assertEqual(proc.returncode, 0) + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") def test_python_malloc_stats(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1ff600e30bf4cbd..c776089ddd0d59b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -767,7 +767,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'configure_c_stdio': False, 'buffered_stdio': True, - 'stdio_encoding': GET_DEFAULT_CONFIG, + 'stdin_encoding': GET_DEFAULT_CONFIG, + 'stdout_encoding': GET_DEFAULT_CONFIG, + 'stderr_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, 'skip_source_first_line': False, @@ -934,7 +936,8 @@ def get_expected_config(self, expected_preconfig, expected, # there is no easy way to get the locale encoding before # setlocale(LC_CTYPE, "") is called: don't test encodings for key in ('filesystem_encoding', 'filesystem_errors', - 'stdio_encoding', 'stdio_errors'): + 'stdin_encoding', 'stdout_encoding', + 'stderr_encoding', 'stdio_errors'): expected[key] = self.IGNORE_CONFIG if expected_preconfig['utf8_mode'] == 1: @@ -942,8 +945,10 @@ def get_expected_config(self, expected_preconfig, expected, expected['filesystem_encoding'] = 'utf-8' if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG: expected['filesystem_errors'] = self.UTF8_MODE_ERRORS - if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG: - expected['stdio_encoding'] = 'utf-8' + for key in ('stdin_encoding', 'stdout_encoding', + 'stderr_encoding'): + if expected[key] is self.GET_DEFAULT_CONFIG: + expected[key] = 'utf-8' if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: expected['stdio_errors'] = 'surrogateescape' @@ -1133,7 +1138,9 @@ def test_init_from_config(self): 'malloc_stats': True, 'pymalloc_hugepages': True, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'pycache_prefix': 'conf_pycache_prefix', @@ -1205,7 +1212,9 @@ def test_init_compat_env(self): 'write_bytecode': False, 'verbose': 1, 'buffered_stdio': False, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'user_site_directory': False, 'faulthandler': True, @@ -1242,7 +1251,9 @@ def test_init_python_env(self): 'write_bytecode': False, 'verbose': 1, 'buffered_stdio': False, - 'stdio_encoding': 'iso8859-1', + 'stdin_encoding': 'iso8859-1', + 'stdout_encoding': 'iso8859-1', + 'stderr_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'user_site_directory': False, 'faulthandler': True, diff --git a/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst b/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst new file mode 100644 index 000000000000000..e61839e5312540a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst @@ -0,0 +1,5 @@ +Replace :c:member:`!PyConfig.stdio_encoding` with three separate members: +:c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding` and +:c:member:`PyConfig.stderr_encoding`. In the legacy Windows stdio mode they +are initialized with the encoding of the device the corresponding stream is +connected to. diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst b/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst new file mode 100644 index 000000000000000..c3e2332defabfa9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst @@ -0,0 +1,3 @@ +Fix the encoding of the standard streams in the legacy Windows stdio mode +(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device +the stream is connected to, as in Python 3.7, not the ANSI code page. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 45d61c8b8b765a6..33b271ebe50fb12 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15209,9 +15209,14 @@ init_stdio_encoding(PyInterpreterState *interp) { /* Update the stdio encoding to the normalized Python codec name. */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); - if (config_get_codec_name(&config->stdio_encoding) < 0) { - return _PyStatus_ERR("failed to get the Python codec name " - "of the stdio encoding"); + wchar_t **encodings[] = {&config->stdin_encoding, + &config->stdout_encoding, + &config->stderr_encoding}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) { + if (config_get_codec_name(encodings[i]) < 0) { + return _PyStatus_ERR("failed to get the Python codec name " + "of the stdio encoding"); + } } return _PyStatus_OK(); } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 418609abc5f6b82..bbf6d9e38bdba60 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -302,7 +302,9 @@ static void check_stdio_details(const wchar_t *encoding, const wchar_t *errors) _PyConfig_InitCompatConfig(&config); /* Force the given IO encoding */ if (encoding) { - config_set_string(&config, &config.stdio_encoding, encoding); + config_set_string(&config, &config.stdin_encoding, encoding); + config_set_string(&config, &config.stdout_encoding, encoding); + config_set_string(&config, &config.stderr_encoding, encoding); } if (errors) { config_set_string(&config, &config.stdio_errors, errors); @@ -770,7 +772,9 @@ static int test_init_from_config(void) config.buffered_stdio = 0; putenv("PYTHONIOENCODING=cp424"); - config_set_string(&config, &config.stdio_encoding, L"iso8859-1"); + config_set_string(&config, &config.stdin_encoding, L"iso8859-1"); + config_set_string(&config, &config.stdout_encoding, L"iso8859-1"); + config_set_string(&config, &config.stderr_encoding, L"iso8859-1"); config_set_string(&config, &config.stdio_errors, L"replace"); putenv("PYTHONNOUSERSITE="); diff --git a/Python/initconfig.c b/Python/initconfig.c index b9bacb17a66454a..85ddb9b1d93d9ad 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -196,7 +196,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stderr_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdin_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdout_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), @@ -1074,7 +1076,9 @@ config_check_consistency(const PyConfig *config) assert(config->module_search_paths_set >= 0); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); + assert(config->stdin_encoding != NULL); + assert(config->stdout_encoding != NULL); + assert(config->stderr_encoding != NULL); assert(config->stdio_errors != NULL); #ifdef MS_WINDOWS assert(config->legacy_windows_stdio >= 0); @@ -1140,7 +1144,9 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); - CLEAR(config->stdio_encoding); + CLEAR(config->stdin_encoding); + CLEAR(config->stdout_encoding); + CLEAR(config->stderr_encoding); CLEAR(config->stdio_errors); CLEAR(config->run_command); CLEAR(config->run_module); @@ -2646,6 +2652,62 @@ config_get_locale_encoding(PyConfig *config, const PyPreConfig *preconfig, } +#ifdef MS_WINDOWS +/* The encoding of the device, or the locale encoding if it is not + a console. See also _Py_device_encoding(). */ +static PyStatus +config_get_device_encoding(PyConfig *config, const PyPreConfig *preconfig, + int fd, wchar_t **result) +{ + UINT cp = 0; + HANDLE handle = (HANDLE)_Py_get_osfhandle_noraise(fd); + if (handle != INVALID_HANDLE_VALUE && GetFileType(handle) == FILE_TYPE_CHAR) { + DWORD temp; + /* GetConsoleMode() only succeeds for a console handle. */ + if (!GetConsoleMode(handle, &temp)) { + /* Assume that access denied implies an output handle. */ + if (GetLastError() == ERROR_ACCESS_DENIED) { + cp = GetConsoleOutputCP(); + } + } + else if (GetNumberOfConsoleInputEvents(handle, &temp)) { + cp = GetConsoleCP(); + } + else { + cp = GetConsoleOutputCP(); + } + } + if (cp == 0) { + return config_get_locale_encoding(config, preconfig, result); + } + if (cp == CP_UTF8) { + return PyConfig_SetString(config, result, L"utf-8"); + } + wchar_t encoding[20]; + swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", (unsigned int)cp); + return PyConfig_SetString(config, result, encoding); +} +#endif + + +/* The default encoding of the standard stream with the file descriptor fd. */ +static PyStatus +config_get_stdio_encoding(PyConfig *config, const PyPreConfig *preconfig, + int fd, wchar_t **result) +{ + if (*result != NULL) { + return _PyStatus_OK(); + } +#ifdef MS_WINDOWS + if (config->legacy_windows_stdio) { + /* gh-86427: the standard streams are ordinary files here. */ + return config_get_device_encoding(config, preconfig, fd, result); + } +#endif + return config_get_locale_encoding(config, preconfig, result); +} + + static PyStatus config_init_stdio_encoding(PyConfig *config, const PyPreConfig *preconfig) @@ -2653,7 +2715,9 @@ config_init_stdio_encoding(PyConfig *config, PyStatus status; // Exit if encoding and errors are defined - if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { + if (config->stdin_encoding != NULL && config->stdout_encoding != NULL + && config->stderr_encoding != NULL && config->stdio_errors != NULL) + { return _PyStatus_OK(); } @@ -2676,8 +2740,14 @@ config_init_stdio_encoding(PyConfig *config, /* Does PYTHONIOENCODING contain an encoding? */ if (pythonioencoding[0]) { - if (config->stdio_encoding == NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->stdio_encoding, + wchar_t **encodings[] = {&config->stdin_encoding, + &config->stdout_encoding, + &config->stderr_encoding}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) { + if (*encodings[i] != NULL) { + continue; + } + status = CONFIG_SET_BYTES_STR(config, encodings[i], pythonioencoding, "PYTHONIOENCODING environment variable"); if (_PyStatus_EXCEPTION(status)) { @@ -2709,12 +2779,20 @@ config_init_stdio_encoding(PyConfig *config, } /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { - status = config_get_locale_encoding(config, preconfig, - &config->stdio_encoding); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = config_get_stdio_encoding(config, preconfig, 0, + &config->stdin_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = config_get_stdio_encoding(config, preconfig, 1, + &config->stdout_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = config_get_stdio_encoding(config, preconfig, 2, + &config->stderr_encoding); + if (_PyStatus_EXCEPTION(status)) { + return status; } if (config->stdio_errors == NULL) { const wchar_t *errors = config_get_stdio_errors(preconfig); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 500a1a1949a5a8a..4e2a14466d317bf 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3186,7 +3186,7 @@ init_sys_streams(PyThreadState *tstate) * GUI apps don't have valid standard streams by default. */ std = create_stdio(config, iomod, fd, 0, "", - config->stdio_encoding, + config->stdin_encoding, config->stdio_errors); if (std == NULL) goto error; @@ -3197,7 +3197,7 @@ init_sys_streams(PyThreadState *tstate) /* Set sys.stdout */ fd = fileno(stdout); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->stdout_encoding, config->stdio_errors); if (std == NULL) goto error; @@ -3209,7 +3209,7 @@ init_sys_streams(PyThreadState *tstate) /* Set sys.stderr, replaces the preliminary stderr */ fd = fileno(stderr); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->stderr_encoding, L"backslashreplace"); if (std == NULL) goto error;