Skip to content

Commit 1588be6

Browse files
xdegayexdegaye
authored
bpo-28180: Fix the implementation of PEP 538 on Android (GH-4334)
1 parent 9e78dc2 commit 1588be6

File tree

6 files changed

+90
-24
lines changed

6 files changed

+90
-24
lines changed

‎Include/pylifecycle.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);
137137
#ifndef Py_LIMITED_API
138138
PyAPI_FUNC(void) _Py_CoerceLegacyLocale(void);
139139
PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void);
140+
PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category);
140141
#endif
141142

142143
#ifdef __cplusplus

‎Lib/test/test_c_locale_coercion.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import sys
77
import sysconfig
88
import shutil
9-
import subprocess
109
from collections import namedtuple
1110

1211
import test.support
@@ -18,9 +17,12 @@
1817
# Set our expectation for the default encoding used in the C locale
1918
# for the filesystem encoding and the standard streams
2019

21-
# AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
20+
# While most *nix platforms default to ASCII in the C locale, some use a
21+
# different encoding.
2222
if sys.platform.startswith("aix"):
2323
C_LOCALE_STREAM_ENCODING = "iso8859-1"
24+
elif test.support.is_android:
25+
C_LOCALE_STREAM_ENCODING = "utf-8"
2426
else:
2527
C_LOCALE_STREAM_ENCODING = "ascii"
2628

@@ -301,6 +303,19 @@ def _check_c_locale_coercion(self,
301303
# See https://bugs.python.org/issue30672 for discussion
302304
if locale_to_set == "POSIX":
303305
continue
306+
307+
# Platforms using UTF-8 in the C locale do not print
308+
# CLI_COERCION_WARNING when all the locale envt variables are
309+
# not set or set to the empty string.
310+
_expected_warnings = expected_warnings
311+
for _env_var in base_var_dict:
312+
if base_var_dict[_env_var]:
313+
break
314+
else:
315+
if (C_LOCALE_STREAM_ENCODING == "utf-8" and
316+
locale_to_set == "" and coerce_c_locale == "warn"):
317+
_expected_warnings = None
318+
304319
with self.subTest(env_var=env_var,
305320
nominal_locale=locale_to_set,
306321
PYTHONCOERCECLOCALE=coerce_c_locale):
@@ -312,7 +327,7 @@ def _check_c_locale_coercion(self,
312327
self._check_child_encoding_details(var_dict,
313328
fs_encoding,
314329
stream_encoding,
315-
expected_warnings,
330+
_expected_warnings,
316331
coercion_expected)
317332

318333
def test_test_PYTHONCOERCECLOCALE_not_set(self):

‎Misc/NEWS.d/next/Core and Builtins/2017-11-12-11-44-22.bpo-28180.HQX000.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
A new internal ``_Py_SetLocaleFromEnv(category)`` helper function has been
2+
added in order to improve the consistency of behaviour across different
3+
``libc`` implementations (e.g. Android doesn't support setting the locale from
4+
the environment by default).

‎Modules/readline.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1245,7 +1245,7 @@ call_readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
12451245
char *saved_locale = strdup(setlocale(LC_CTYPE, NULL));
12461246
if (!saved_locale)
12471247
Py_FatalError("not enough memory to save locale");
1248-
setlocale(LC_CTYPE, "");
1248+
_Py_SetLocaleFromEnv(LC_CTYPE);
12491249
#endif
12501250

12511251
if (sys_stdin != rl_instream || sys_stdout != rl_outstream) {

‎Programs/python.c‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,8 @@ main(int argc, char **argv)
5454
return 1;
5555
}
5656

57-
#ifdef __ANDROID__
58-
/* Passing "" to setlocale() on Android requests the C locale rather
59-
* than checking environment variables, so request C.UTF-8 explicitly
60-
*/
61-
setlocale(LC_ALL, "C.UTF-8");
62-
#else
6357
/* Reconfigure the locale to the default for this process */
64-
setlocale(LC_ALL, "");
65-
#endif
58+
_Py_SetLocaleFromEnv(LC_ALL);
6659

6760
/* The legacy C locale assumes ASCII as the default text encoding, which
6861
* causes problems not only for the CPython runtime, but also other

‎Python/pylifecycle.c‎

Lines changed: 65 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -459,7 +459,7 @@ _coerce_default_locale_settings(const _LocaleCoercionTarget *target)
459459
const char *newloc = target->locale_name;
460460

461461
/* Reset locale back to currently configured defaults */
462-
setlocale(LC_ALL, "");
462+
_Py_SetLocaleFromEnv(LC_ALL);
463463

464464
/* Set the relevant locale environment variable */
465465
if (setenv("LC_CTYPE", newloc, 1)) {
@@ -472,7 +472,7 @@ _coerce_default_locale_settings(const _LocaleCoercionTarget *target)
472472
}
473473

474474
/* Reconfigure with the overridden environment variables */
475-
setlocale(LC_ALL, "");
475+
_Py_SetLocaleFromEnv(LC_ALL);
476476
}
477477
#endif
478478

@@ -503,13 +503,14 @@ _Py_CoerceLegacyLocale(void)
503503
const char *new_locale = setlocale(LC_CTYPE,
504504
target->locale_name);
505505
if (new_locale != NULL) {
506-
#if !defined(__APPLE__) && defined(HAVE_LANGINFO_H) && defined(CODESET)
506+
#if !defined(__APPLE__) && !defined(__ANDROID__) && \
507+
defined(HAVE_LANGINFO_H) && defined(CODESET)
507508
/* Also ensure that nl_langinfo works in this locale */
508509
char *codeset = nl_langinfo(CODESET);
509510
if (!codeset || *codeset == '\0') {
510511
/* CODESET is not set or empty, so skip coercion */
511512
new_locale = NULL;
512-
setlocale(LC_CTYPE, "");
513+
_Py_SetLocaleFromEnv(LC_CTYPE);
513514
continue;
514515
}
515516
#endif
@@ -524,6 +525,65 @@ _Py_CoerceLegacyLocale(void)
524525
#endif
525526
}
526527

528+
/* _Py_SetLocaleFromEnv() is a wrapper around setlocale(category, "") to
529+
* isolate the idiosyncrasies of different libc implementations. It reads the
530+
* appropriate environment variable and uses its value to select the locale for
531+
* 'category'. */
532+
char *
533+
_Py_SetLocaleFromEnv(int category)
534+
{
535+
#ifdef __ANDROID__
536+
const char *locale;
537+
const char **pvar;
538+
#ifdef PY_COERCE_C_LOCALE
539+
const char *coerce_c_locale;
540+
#endif
541+
const char *utf8_locale = "C.UTF-8";
542+
const char *env_var_set[] = {
543+
"LC_ALL",
544+
"LC_CTYPE",
545+
"LANG",
546+
NULL,
547+
};
548+
549+
/* Android setlocale(category, "") doesn't check the environment variables
550+
* and incorrectly sets the "C" locale at API 24 and older APIs. We only
551+
* check the environment variables listed in env_var_set. */
552+
for (pvar=env_var_set; *pvar; pvar++) {
553+
locale = getenv(*pvar);
554+
if (locale != NULL && *locale != '\0') {
555+
if (strcmp(locale, utf8_locale) == 0 ||
556+
strcmp(locale, "en_US.UTF-8") == 0) {
557+
return setlocale(category, utf8_locale);
558+
}
559+
return setlocale(category, "C");
560+
}
561+
}
562+
563+
/* Android uses UTF-8, so explicitly set the locale to C.UTF-8 if none of
564+
* LC_ALL, LC_CTYPE, or LANG is set to a non-empty string.
565+
* Quote from POSIX section "8.2 Internationalization Variables":
566+
* "4. If the LANG environment variable is not set or is set to the empty
567+
* string, the implementation-defined default locale shall be used." */
568+
569+
#ifdef PY_COERCE_C_LOCALE
570+
coerce_c_locale = getenv("PYTHONCOERCECLOCALE");
571+
if (coerce_c_locale == NULL || strcmp(coerce_c_locale, "0") != 0) {
572+
/* Some other ported code may check the environment variables (e.g. in
573+
* extension modules), so we make sure that they match the locale
574+
* configuration */
575+
if (setenv("LC_CTYPE", utf8_locale, 1)) {
576+
fprintf(stderr, "Warning: failed setting the LC_CTYPE "
577+
"environment variable to %s\n", utf8_locale);
578+
}
579+
}
580+
#endif
581+
return setlocale(category, utf8_locale);
582+
#else /* __ANDROID__ */
583+
return setlocale(category, "");
584+
#endif /* __ANDROID__ */
585+
}
586+
527587

528588
/* Global initializations. Can be undone by Py_Finalize(). Don't
529589
call this twice without an intervening Py_Finalize() call.
@@ -599,19 +659,12 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
599659
exit(1);
600660
}
601661

602-
#ifdef __ANDROID__
603-
/* Passing "" to setlocale() on Android requests the C locale rather
604-
* than checking environment variables, so request C.UTF-8 explicitly
605-
*/
606-
setlocale(LC_CTYPE, "C.UTF-8");
607-
#else
608662
#ifndef MS_WINDOWS
609663
/* Set up the LC_CTYPE locale, so we can obtain
610664
the locale's charset without having to switch
611665
locales. */
612-
setlocale(LC_CTYPE, "");
666+
_Py_SetLocaleFromEnv(LC_CTYPE);
613667
_emit_stderr_warning_for_legacy_locale();
614-
#endif
615668
#endif
616669

617670
if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0')

0 commit comments

Comments
 (0)