bpo-42208: Add _Py_GetLocaleEncoding()

_io.TextIOWrapper no longer calls getpreferredencoding(False) of _bootlocale to get the locale encoding, but calls _Py_GetLocaleEncoding() instead.
python · vstinner · Oct 31, 2020 · Oct 30, 2020 · Oct 30, 2020 · Oct 30, 2020
commit f86db86e0ab6a5e500ffd2c5d15a2b4b75ee1291
diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h
@@ -50,6 +50,8 @@ PyAPI_FUNC(int) _Py_GetLocaleconvNumeric(
 
 PyAPI_FUNC(void) _Py_closerange(int first, int last);
 
+PyAPI_FUNC(PyObject*) _Py_GetLocaleEncoding(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
@@ -593,31 +593,6 @@ _PyIO_get_module_state(void)
     return state;
 }
 
-PyObject *
-_PyIO_get_locale_module(_PyIO_State *state)
-{
-    PyObject *mod;
-    if (state->locale_module != NULL) {
-        assert(PyWeakref_CheckRef(state->locale_module));
-        mod = PyWeakref_GET_OBJECT(state->locale_module);
-        if (mod != Py_None) {
-            Py_INCREF(mod);
-            return mod;
-        }
-        Py_CLEAR(state->locale_module);
-    }
-    mod = PyImport_ImportModule("_bootlocale");
-    if (mod == NULL)
-        return NULL;
-    state->locale_module = PyWeakref_NewRef(mod, NULL);
-    if (state->locale_module == NULL) {
-        Py_DECREF(mod);
-        return NULL;
-    }
-    return mod;
-}
-
-
 static int
 iomodule_traverse(PyObject *mod, visitproc visit, void *arg) {
     _PyIO_State *state = get_io_state(mod);

diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h
@@ -150,7 +150,6 @@ typedef struct {
 #define IO_STATE() _PyIO_get_module_state()
 
 extern _PyIO_State *_PyIO_get_module_state(void);
-extern PyObject *_PyIO_get_locale_module(_PyIO_State *);
 
 #ifdef MS_WINDOWS
 extern char _PyIO_get_console_type(PyObject *);

diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
@@ -10,6 +10,7 @@
 #include "Python.h"
 #include "pycore_interp.h"        // PyInterpreterState.fs_codec
 #include "pycore_long.h"          // _PyLong_GetZero()
+#include "pycore_fileutils.h"     // _Py_GetLocaleEncoding()
 #include "pycore_object.h"
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "structmember.h"         // PyMemberDef
@@ -27,7 +28,6 @@ _Py_IDENTIFIER(_dealloc_warn);
 _Py_IDENTIFIER(decode);
 _Py_IDENTIFIER(fileno);
 _Py_IDENTIFIER(flush);
-_Py_IDENTIFIER(getpreferredencoding);
 _Py_IDENTIFIER(isatty);
 _Py_IDENTIFIER(mode);
 _Py_IDENTIFIER(name);
@@ -1155,29 +1155,11 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer,
         }
     }
     if (encoding == NULL && self->encoding == NULL) {
-        PyObject *locale_module = _PyIO_get_locale_module(state);
-        if (locale_module == NULL)
-            goto catch_ImportError;
-        self->encoding = _PyObject_CallMethodIdOneArg(
-            locale_module, &PyId_getpreferredencoding, Py_False);
-        Py_DECREF(locale_module);
+        self->encoding = _Py_GetLocaleEncoding();
         if (self->encoding == NULL) {
-          catch_ImportError:
-            /*
-             Importing locale can raise an ImportError because of
-             _functools, and locale.getpreferredencoding can raise an
-             ImportError if _locale is not available.  These will happen
-             during module building.
-            */
-            if (PyErr_ExceptionMatches(PyExc_ImportError)) {
-                PyErr_Clear();
-                self->encoding = PyUnicode_FromString("ascii");
-            }
-            else
-                goto error;
+            goto error;
         }
-        else if (!PyUnicode_Check(self->encoding))
-            Py_CLEAR(self->encoding);
+        assert(PyUnicode_Check(self->encoding));
     }
     if (self->encoding != NULL) {
         encoding = PyUnicode_AsUTF8(self->encoding);

diff --git a/Python/fileutils.c b/Python/fileutils.c
@@ -1,5 +1,6 @@
 #include "Python.h"
-#include "pycore_fileutils.h"
+#include "pycore_fileutils.h"     // fileutils definitions
+#include "pycore_runtime.h"       // _PyRuntime
 #include "osdefs.h"               // SEP
 #include <locale.h>
 
@@ -820,6 +821,44 @@ _Py_EncodeLocaleEx(const wchar_t *text, char **str,
 }
 
 
+PyObject *
+_Py_GetLocaleEncoding(void)
+{
+#ifdef _Py_FORCE_UTF8_LOCALE
+    // On Android langinfo.h and CODESET are missing,
+    // and UTF-8 is always used in mbstowcs() and wcstombs().
+    return PyUnicode_FromString("UTF-8");
+#else
+    const PyPreConfig *preconfig = &_PyRuntime.preconfig;
+    if (preconfig->utf8_mode) {
+        return PyUnicode_FromString("UTF-8");
+    }
+
+#if defined(MS_WINDOWS)
+    return PyUnicode_FromFormat("cp%u", GetACP());
+#else
+    const char *encoding = nl_langinfo(CODESET);
+    if (!encoding || encoding[0] == '\0') {
+#ifdef _Py_FORCE_UTF8_FS_ENCODING
+        // nl_langinfo() can return an empty string when the setting has an
+        // invalid value. Default to UTF-8 in that case, because UTF-8 is the
+        // default charset on macOS and returning nothing will crash the
+        // interpreter.
+        return PyUnicode_FromString("UTF-8");
+#else
+        PyErr_SetString(PyExc_ValueError,
+                        "failed to get the locale encoding: "
+                        "nl_langinfo(CODESET) returns an empty string");
+        return NULL;
+#endif
+    }
+    return PyUnicode_FromString(encoding);
+#endif  // !CODESET
+
+#endif
+}
+
+
 #ifdef MS_WINDOWS
 static __int64 secs_between_epochs = 11644473600; /* Seconds between 1.1.1601 and 1.1.1970 */
-Original file line number
+Diff line change
@@ Expand Up / @@ -50,6 +50,8 @@ PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( @@
     PyAPI_FUNC(void) _Py_closerange(int first, int last);
+    PyAPI_FUNC(PyObject*) _Py_GetLocaleEncoding(void);
     #ifdef __cplusplus
     }
     #endif
@@ Expand Down @@