Skip to content

gh-86427: Replace PyConfig.stdio_encoding with three separate members #155417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
serhiy-storchaka wants to merge 1 commit into
base: main
Choose a base branch
from
Open

gh-86427: Replace PyConfig.stdio_encoding with three separate members #155417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions Doc/c-api/init_config.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,8 +477,12 @@ Configuration Options
- :c:member:`skip_source_first_line <PyConfig.skip_source_first_line>`
- ``bool``
- Read-only
* - ``"stdio_encoding"``
- :c:member:`stdio_encoding <PyConfig.stdio_encoding>`
* - ``"stderr_encoding"``
- :c:member:`stderr_encoding <PyConfig.stderr_encoding>`
* - ``"stdin_encoding"``
- :c:member:`stdin_encoding <PyConfig.stdin_encoding>`
* - ``"stdout_encoding"``
- :c:member:`stdout_encoding <PyConfig.stdout_encoding>`
- ``str``
- Read-only
* - ``"stdio_errors"``
Expand Down Expand Up @@ -1863,12 +1867,14 @@ PyConfig

Default: ``0``.

.. c:member:: wchar_t* stdio_encoding
.. c:member:: wchar_t* stdin_encoding
.. c:member:: wchar_t* stdout_encoding
.. c:member:: wchar_t* stderr_encoding
.. c:member:: wchar_t* stdio_errors

Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and
:data:`sys.stderr` (but :data:`sys.stderr` always uses
``"backslashreplace"`` error handler).
Encoding of :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`
respectively, and encoding errors of all of them (but :data:`sys.stderr`
always uses the ``"backslashreplace"`` error handler).

Use the :envvar:`PYTHONIOENCODING` environment variable if it is
non-empty.
Expand Down
3 changes: 2 additions & 1 deletion Doc/library/sys.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2136,7 +2136,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only
follows:

* The encoding and error handling are initialized from
:c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`.
:c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding`,
:c:member:`PyConfig.stderr_encoding` and :c:member:`PyConfig.stdio_errors`.

On Windows, UTF-8 is used for the console device. Non-character
devices such as disk files and pipes use the system locale
Expand Down
4 changes: 3 additions & 1 deletion Include/cpython/initconfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,9 @@ typedef struct PyConfig {
int user_site_directory;
int configure_c_stdio;
int buffered_stdio;
wchar_t *stdio_encoding;
wchar_t *stdin_encoding;
wchar_t *stdout_encoding;
wchar_t *stderr_encoding;
wchar_t *stdio_errors;
#ifdef MS_WINDOWS
int legacy_windows_stdio;
Expand Down
4 changes: 3 additions & 1 deletion Lib/test/test_capi/test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ def test_config_get(self):
("show_ref_count", bool, None),
("site_import", bool, None),
("skip_source_first_line", bool, None),
("stdio_encoding", str, None),
("stderr_encoding", str, None),
("stdin_encoding", str, None),
("stdout_encoding", str, None),
("stdio_errors", str, None),
("stdlib_dir", str | None, "_stdlib_dir"),
("tracemalloc", int, None),
Expand Down
36 changes: 36 additions & 0 deletions Lib/test/test_cmd_line.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1067,6 +1067,42 @@ def test_python_legacy_windows_stdio(self):
support.skip_on_low_desktop_heap_memory_subprocess(p.returncode)
self.assertEqual(p.returncode, 0)

@unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows')
def test_python_legacy_windows_stdio_encoding(self):
# gh-86427: In the legacy mode the encoding of a standard stream is
# the encoding of the console it is connected to, which can differ
# for input and output.
import ctypes
kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
try:
fin = open('CONIN$')
except OSError:
self.skipTest('no console')
# We cannot use PIPE, because the standard streams should be
# connected to the console. So we use the exit code.
code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or "
"sys.stdout.encoding != 'cp437')")
env = os.environ.copy()
env['PYTHONLEGACYWINDOWSSTDIO'] = '1'
env['PYTHONUTF8'] = '0'
env.pop('PYTHONIOENCODING', None)
old_cp = kernel32.GetConsoleCP()
old_output_cp = kernel32.GetConsoleOutputCP()
with fin, open('CONOUT$', 'w') as fout:
try:
if not kernel32.SetConsoleCP(850):
self.skipTest('cannot set the console input code page')
if not kernel32.SetConsoleOutputCP(437):
self.skipTest('cannot set the console output code page')
proc = subprocess.run([sys.executable, '-c', code], env=env,
stdin=fin, stdout=fout,
stderr=subprocess.DEVNULL)
finally:
kernel32.SetConsoleCP(old_cp)
kernel32.SetConsoleOutputCP(old_output_cp)
support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode)
self.assertEqual(proc.returncode, 0)

@unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()),
"PYTHONMALLOCSTATS doesn't work with ASAN")
def test_python_malloc_stats(self):
Expand Down
25 changes: 18 additions & 7 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -767,7 +767,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'configure_c_stdio': False,
'buffered_stdio': True,

'stdio_encoding': GET_DEFAULT_CONFIG,
'stdin_encoding': GET_DEFAULT_CONFIG,
'stdout_encoding': GET_DEFAULT_CONFIG,
'stderr_encoding': GET_DEFAULT_CONFIG,
'stdio_errors': GET_DEFAULT_CONFIG,

'skip_source_first_line': False,
Expand Down Expand Up @@ -934,16 +936,19 @@ def get_expected_config(self, expected_preconfig, expected,
# there is no easy way to get the locale encoding before
# setlocale(LC_CTYPE, "") is called: don't test encodings
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
'stdin_encoding', 'stdout_encoding',
'stderr_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG

if expected_preconfig['utf8_mode'] == 1:
if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_encoding'] = 'utf-8'
if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_errors'] = self.UTF8_MODE_ERRORS
if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG:
expected['stdio_encoding'] = 'utf-8'
for key in ('stdin_encoding', 'stdout_encoding',
'stderr_encoding'):
if expected[key] is self.GET_DEFAULT_CONFIG:
expected[key] = 'utf-8'
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
expected['stdio_errors'] = 'surrogateescape'

Expand Down Expand Up @@ -1133,7 +1138,9 @@ def test_init_from_config(self):
'malloc_stats': True,
'pymalloc_hugepages': True,

'stdio_encoding': 'iso8859-1',
'stdin_encoding': 'iso8859-1',
'stdout_encoding': 'iso8859-1',
'stderr_encoding': 'iso8859-1',
'stdio_errors': 'replace',

'pycache_prefix': 'conf_pycache_prefix',
Expand Down Expand Up @@ -1205,7 +1212,9 @@ def test_init_compat_env(self):
'write_bytecode': False,
'verbose': 1,
'buffered_stdio': False,
'stdio_encoding': 'iso8859-1',
'stdin_encoding': 'iso8859-1',
'stdout_encoding': 'iso8859-1',
'stderr_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': False,
'faulthandler': True,
Expand Down Expand Up @@ -1242,7 +1251,9 @@ def test_init_python_env(self):
'write_bytecode': False,
'verbose': 1,
'buffered_stdio': False,
'stdio_encoding': 'iso8859-1',
'stdin_encoding': 'iso8859-1',
'stdout_encoding': 'iso8859-1',
'stderr_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': False,
'faulthandler': True,
Expand Down
5 changes: 5 additions & 0 deletions Misc/NEWS.d/next/C_API/2026-08-09-06-30-00.gh-issue-86427.threeenc.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Replace :c:member:`!PyConfig.stdio_encoding` with three separate members:
:c:member:`PyConfig.stdin_encoding`, :c:member:`PyConfig.stdout_encoding` and
:c:member:`PyConfig.stderr_encoding`. In the legacy Windows stdio mode they
are initialized with the encoding of the device the corresponding stream is
connected to.
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Windows/2026-08-09-06-40-00.gh-issue-86427.legacystdio.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix the encoding of the standard streams in the legacy Windows stdio mode
(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device
the stream is connected to, as in Python 3.7, not the ANSI code page.
11 changes: 8 additions & 3 deletions Objects/unicodeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15209,9 +15209,14 @@ init_stdio_encoding(PyInterpreterState *interp)
{
/* Update the stdio encoding to the normalized Python codec name. */
PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
if (config_get_codec_name(&config->stdio_encoding) < 0) {
return _PyStatus_ERR("failed to get the Python codec name "
"of the stdio encoding");
wchar_t **encodings[] = {&config->stdin_encoding,
&config->stdout_encoding,
&config->stderr_encoding};
for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) {
if (config_get_codec_name(encodings[i]) < 0) {
return _PyStatus_ERR("failed to get the Python codec name "
"of the stdio encoding");
}
}
return _PyStatus_OK();
}
Expand Down
8 changes: 6 additions & 2 deletions Programs/_testembed.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,9 @@ static void check_stdio_details(const wchar_t *encoding, const wchar_t *errors)
_PyConfig_InitCompatConfig(&config);
/* Force the given IO encoding */
if (encoding) {
config_set_string(&config, &config.stdio_encoding, encoding);
config_set_string(&config, &config.stdin_encoding, encoding);
config_set_string(&config, &config.stdout_encoding, encoding);
config_set_string(&config, &config.stderr_encoding, encoding);
}
if (errors) {
config_set_string(&config, &config.stdio_errors, errors);
Expand Down Expand Up @@ -770,7 +772,9 @@ static int test_init_from_config(void)
config.buffered_stdio = 0;

putenv("PYTHONIOENCODING=cp424");
config_set_string(&config, &config.stdio_encoding, L"iso8859-1");
config_set_string(&config, &config.stdin_encoding, L"iso8859-1");
config_set_string(&config, &config.stdout_encoding, L"iso8859-1");
config_set_string(&config, &config.stderr_encoding, L"iso8859-1");
config_set_string(&config, &config.stdio_errors, L"replace");

putenv("PYTHONNOUSERSITE=");
Expand Down
102 changes: 90 additions & 12 deletions Python/initconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site
SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(stderr_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(stdin_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(stdout_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL),
SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL),
Expand Down Expand Up @@ -1074,7 +1076,9 @@ config_check_consistency(const PyConfig *config)
assert(config->module_search_paths_set >= 0);
assert(config->filesystem_encoding != NULL);
assert(config->filesystem_errors != NULL);
assert(config->stdio_encoding != NULL);
assert(config->stdin_encoding != NULL);
assert(config->stdout_encoding != NULL);
assert(config->stderr_encoding != NULL);
assert(config->stdio_errors != NULL);
#ifdef MS_WINDOWS
assert(config->legacy_windows_stdio >= 0);
Expand Down Expand Up @@ -1140,7 +1144,9 @@ PyConfig_Clear(PyConfig *config)

CLEAR(config->filesystem_encoding);
CLEAR(config->filesystem_errors);
CLEAR(config->stdio_encoding);
CLEAR(config->stdin_encoding);
CLEAR(config->stdout_encoding);
CLEAR(config->stderr_encoding);
CLEAR(config->stdio_errors);
CLEAR(config->run_command);
CLEAR(config->run_module);
Expand Down Expand Up @@ -2646,14 +2652,72 @@ config_get_locale_encoding(PyConfig *config, const PyPreConfig *preconfig,
}


#ifdef MS_WINDOWS
/* The encoding of the device, or the locale encoding if it is not
a console. See also _Py_device_encoding(). */
static PyStatus
config_get_device_encoding(PyConfig *config, const PyPreConfig *preconfig,
int fd, wchar_t **result)
{
UINT cp = 0;
HANDLE handle = (HANDLE)_Py_get_osfhandle_noraise(fd);
if (handle != INVALID_HANDLE_VALUE && GetFileType(handle) == FILE_TYPE_CHAR) {
DWORD temp;
/* GetConsoleMode() only succeeds for a console handle. */
if (!GetConsoleMode(handle, &temp)) {
/* Assume that access denied implies an output handle. */
if (GetLastError() == ERROR_ACCESS_DENIED) {
cp = GetConsoleOutputCP();
}
}
else if (GetNumberOfConsoleInputEvents(handle, &temp)) {
cp = GetConsoleCP();
}
else {
cp = GetConsoleOutputCP();
}
}
if (cp == 0) {
return config_get_locale_encoding(config, preconfig, result);
}
if (cp == CP_UTF8) {
return PyConfig_SetString(config, result, L"utf-8");
}
wchar_t encoding[20];
swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", (unsigned int)cp);
return PyConfig_SetString(config, result, encoding);
}
#endif


/* The default encoding of the standard stream with the file descriptor fd. */
static PyStatus
config_get_stdio_encoding(PyConfig *config, const PyPreConfig *preconfig,
int fd, wchar_t **result)
{
if (*result != NULL) {
return _PyStatus_OK();
}
#ifdef MS_WINDOWS
if (config->legacy_windows_stdio) {
/* gh-86427: the standard streams are ordinary files here. */
return config_get_device_encoding(config, preconfig, fd, result);
}
#endif
return config_get_locale_encoding(config, preconfig, result);
}


static PyStatus
config_init_stdio_encoding(PyConfig *config,
const PyPreConfig *preconfig)
{
PyStatus status;

// Exit if encoding and errors are defined
if (config->stdio_encoding != NULL && config->stdio_errors != NULL) {
if (config->stdin_encoding != NULL && config->stdout_encoding != NULL
&& config->stderr_encoding != NULL && config->stdio_errors != NULL)
{
return _PyStatus_OK();
}

Expand All @@ -2676,8 +2740,14 @@ config_init_stdio_encoding(PyConfig *config,

/* Does PYTHONIOENCODING contain an encoding? */
if (pythonioencoding[0]) {
if (config->stdio_encoding == NULL) {
status = CONFIG_SET_BYTES_STR(config, &config->stdio_encoding,
wchar_t **encodings[] = {&config->stdin_encoding,
&config->stdout_encoding,
&config->stderr_encoding};
for (size_t i = 0; i < Py_ARRAY_LENGTH(encodings); i++) {
if (*encodings[i] != NULL) {
continue;
}
status = CONFIG_SET_BYTES_STR(config, encodings[i],
pythonioencoding,
"PYTHONIOENCODING environment variable");
if (_PyStatus_EXCEPTION(status)) {
Expand Down Expand Up @@ -2709,12 +2779,20 @@ config_init_stdio_encoding(PyConfig *config,
}

/* Choose the default error handler based on the current locale. */
if (config->stdio_encoding == NULL) {
status = config_get_locale_encoding(config, preconfig,
&config->stdio_encoding);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
status = config_get_stdio_encoding(config, preconfig, 0,
&config->stdin_encoding);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
status = config_get_stdio_encoding(config, preconfig, 1,
&config->stdout_encoding);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
status = config_get_stdio_encoding(config, preconfig, 2,
&config->stderr_encoding);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
if (config->stdio_errors == NULL) {
const wchar_t *errors = config_get_stdio_errors(preconfig);
Expand Down
Loading
Loading