#include "Python.h"

#include "pycore_fileutils.h" // fileutils definitions

#include "pycore_runtime.h" // _PyRuntime

#include "pycore_pystate.h" // _Py_AssertHoldsTstate()

#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE

#include "osdefs.h" // SEP

#include <stdlib.h> // mbstowcs()

#ifdef HAVE_UNISTD_H

# include <unistd.h> // getcwd()

#endif

#ifdef MS_WINDOWS

# include <malloc.h>

# include <windows.h>

# include <winioctl.h> // FILE_DEVICE_* constants

# include "pycore_fileutils_windows.h" // FILE_STAT_BASIC_INFORMATION

# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP)

# define PATHCCH_ALLOW_LONG_PATHS 0x01

# else

# include <pathcch.h> // PathCchCombineEx

# endif

extern int winerror_to_errno(int);

#endif

#ifdef HAVE_LANGINFO_H

# include <langinfo.h> // nl_langinfo(CODESET)

#endif

#ifdef HAVE_SYS_IOCTL_H

#include <sys/ioctl.h>

#endif

#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION

# include <iconv.h> // iconv_open()

#endif

#ifdef HAVE_FCNTL_H

# include <fcntl.h> // fcntl(F_GETFD)

#endif

#ifdef O_CLOEXEC

/* Does open() support the O_CLOEXEC flag? Possible values:

-1: unknown

0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23

1: open() supports O_CLOEXEC flag, close-on-exec is set

The flag is used by _Py_open(), _Py_open_noraise(), io.FileIO

and os.open(). */

int _Py_open_cloexec_works = -1;

#endif

// mbstowcs() and mbrtowc() errors

static const size_t DECODE_ERROR = ((size_t)-1);

#ifdef HAVE_MBRTOWC

static const size_t INCOMPLETE_CHARACTER = (size_t)-2;

#endif

static int

get_surrogateescape(_Py_error_handler errors, int *surrogateescape)

{

switch (errors)

{

case _Py_ERROR_STRICT:

*surrogateescape = 0;

return 0;

case _Py_ERROR_SURROGATEESCAPE:

*surrogateescape = 1;

return 0;

default:

return -1;

}

}

PyObject *

_Py_device_encoding(int fd)

{

int valid;

Py_BEGIN_ALLOW_THREADS

_Py_BEGIN_SUPPRESS_IPH

valid = isatty(fd);

_Py_END_SUPPRESS_IPH

Py_END_ALLOW_THREADS

if (!valid)

Py_RETURN_NONE;

#ifdef MS_WINDOWS

#ifdef HAVE_WINDOWS_CONSOLE_IO

UINT cp;

if (fd == 0)

cp = GetConsoleCP();

else if (fd == 1 || fd == 2)

cp = GetConsoleOutputCP();

else

cp = 0;

/* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application

has no console */

if (cp == 0) {

Py_RETURN_NONE;

}

return PyUnicode_FromFormat("cp%u", (unsigned int)cp);

#else

Py_RETURN_NONE;

#endif /* HAVE_WINDOWS_CONSOLE_IO */

#else

if (_PyRuntime.preconfig.utf8_mode) {

_Py_DECLARE_STR(utf_8, "utf-8");

return &_Py_STR(utf_8);

}

return _Py_GetLocaleEncodingObject();

#endif

}

static int

is_valid_wide_char(wchar_t ch)

{

#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION

/* Oracle Solaris doesn't use Unicode code points as wchar_t encoding

for non-Unicode locales, which makes values higher than _Py_MAX_UNICODE

possibly valid. */

return 1;

#endif

if (Py_UNICODE_IS_SURROGATE(ch)) {

// Reject lone surrogate characters

return 0;

}

#if SIZEOF_WCHAR_T > 2

if (ch > _Py_MAX_UNICODE) {

// bpo-35883: Reject characters outside [U+0000; U+10ffff] range.

// The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629,

// it creates characters outside the [U+0000; U+10ffff] range:

// https://sourceware.org/bugzilla/show_bug.cgi?id=2373

return 0;

}

#endif

return 1;

}

static size_t

_Py_mbstowcs(wchar_t *dest, const char *src, size_t n)

{

size_t count = mbstowcs(dest, src, n);

if (dest != NULL && count != DECODE_ERROR) {

for (size_t i=0; i < count; i++) {

wchar_t ch = dest[i];

if (!is_valid_wide_char(ch)) {

return DECODE_ERROR;

}

}

}

return count;

}

#ifdef HAVE_MBRTOWC

static size_t

_Py_mbrtowc(wchar_t *pwc, const char *str, size_t len, mbstate_t *pmbs)

{

assert(pwc != NULL);

size_t count = mbrtowc(pwc, str, len, pmbs);

if (count != 0 && count != DECODE_ERROR && count != INCOMPLETE_CHARACTER) {

if (!is_valid_wide_char(*pwc)) {

return DECODE_ERROR;

}

}

return count;

}

#endif

#if !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS)

#define USE_FORCE_ASCII

extern int _Py_normalize_encoding(const char *, char *, size_t, int);

/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale

and POSIX locale. nl_langinfo(CODESET) announces an alias of the

ASCII encoding, whereas mbstowcs() and wcstombs() functions use the

ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use

locale.getpreferredencoding() codec. For example, if command line arguments

are decoded by mbstowcs() and encoded back by os.fsencode(), we get a

UnicodeEncodeError instead of retrieving the original byte string.

The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",

nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least

one byte in range 0x80-0xff can be decoded from the locale encoding. The

workaround is also enabled on error, for example if getting the locale

failed.

On HP-UX with the C locale or the POSIX locale, nl_langinfo(CODESET)

announces "roman8" but mbstowcs() uses Latin1 in practice. Force also the

ASCII encoding in this case.

Values of force_ascii:

1: the workaround is used: Py_EncodeLocale() uses

encode_ascii_surrogateescape() and Py_DecodeLocale() uses

decode_ascii()

0: the workaround is not used: Py_EncodeLocale() uses wcstombs() and

Py_DecodeLocale() uses mbstowcs()

-1: unknown, need to call check_force_ascii() to get the value

*/

#define force_ascii (_PyRuntime.fileutils.force_ascii)

static int

check_force_ascii(void)

{

char *loc = setlocale(LC_CTYPE, NULL);

if (loc == NULL) {

goto error;

}

if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) {

/* the LC_CTYPE locale is different than C and POSIX */

return 0;

}

#if defined(HAVE_LANGINFO_H) && defined(CODESET)

const char *codeset = nl_langinfo(CODESET);

if (!codeset || codeset[0] == '\0') {

/* CODESET is not set or empty */

goto error;

}

char encoding[20]; /* longest name: "iso_646.irv_1991\0" */

if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding), 1)) {

goto error;

}

#ifdef __hpux

if (strcmp(encoding, "roman8") == 0) {

unsigned char ch;

wchar_t wch;

size_t res;

ch = (unsigned char)0xA7;

res = _Py_mbstowcs(&wch, (char*)&ch, 1);

if (res != DECODE_ERROR && wch == L'\xA7') {

/* On HP-UX with C locale or the POSIX locale,

nl_langinfo(CODESET) announces "roman8", whereas mbstowcs() uses

Latin1 encoding in practice. Force ASCII in this case.

Roman8 decodes 0xA7 to U+00CF. Latin1 decodes 0xA7 to U+00A7. */

return 1;

}

}

#else

const char* ascii_aliases[] = {

"ascii",

/* Aliases from Lib/encodings/aliases.py */

"646",

"ansi_x3.4_1968",

"ansi_x3.4_1986",

"ansi_x3_4_1968",

"cp367",

"csascii",

"ibm367",

"iso646_us",

"iso_646.irv_1991",

"iso_ir_6",

"us",

"us_ascii",

NULL

};

int is_ascii = 0;

for (const char **alias=ascii_aliases; *alias != NULL; alias++) {

if (strcmp(encoding, *alias) == 0) {

is_ascii = 1;

break;

}

}

if (!is_ascii) {

/* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */

return 0;

}

for (unsigned int i=0x80; i<=0xff; i++) {

char ch[1];

wchar_t wch[1];

size_t res;

unsigned uch = (unsigned char)i;

ch[0] = (char)uch;

res = _Py_mbstowcs(wch, ch, 1);

if (res != DECODE_ERROR) {

/* decoding a non-ASCII character from the locale encoding succeed:

the locale encoding is not ASCII, force ASCII */

return 1;

}

}

/* None of the bytes in the range 0x80-0xff can be decoded from the locale

encoding: the locale encoding is really ASCII */

#endif /* !defined(__hpux) */

return 0;

#else

/* nl_langinfo(CODESET) is not available: always force ASCII */

return 1;

#endif /* defined(HAVE_LANGINFO_H) && defined(CODESET) */

error:

/* if an error occurred, force the ASCII encoding */

return 1;

}

int

_Py_GetForceASCII(void)

{

if (force_ascii == -1) {

force_ascii = check_force_ascii();

}

return force_ascii;

}

void

_Py_ResetForceASCII(void)

{

force_ascii = -1;

}

static int

encode_ascii(const wchar_t *text, char **str,

size_t *error_pos, const char **reason,

int raw_malloc, _Py_error_handler errors)

{

char *result = NULL, *out;

size_t len, i;

wchar_t ch;

int surrogateescape;

if (get_surrogateescape(errors, &surrogateescape) < 0) {

return -3;

}

len = wcslen(text);

/* +1 for NULL byte */

if (raw_malloc) {

result = PyMem_RawMalloc(len + 1);

}

else {

result = PyMem_Malloc(len + 1);

}

if (result == NULL) {

return -1;

}

out = result;

for (i=0; i<len; i++) {

ch = text[i];

if (ch <= 0x7f) {

/* ASCII character */

*out++ = (char)ch;

}

else if (surrogateescape && 0xdc80 <= ch && ch <= 0xdcff) {

/* UTF-8b surrogate */

*out++ = (char)(ch - 0xdc00);

}

else {

if (raw_malloc) {

PyMem_RawFree(result);

}

else {

PyMem_Free(result);

}

if (error_pos != NULL) {

*error_pos = i;

}

if (reason) {

*reason = "encoding error";

}

return -2;

}

}

*out = '\0';

*str = result;

return 0;

}

#else

int

_Py_GetForceASCII(void)

{

return 0;

}

void

_Py_ResetForceASCII(void)

{

/* nothing to do */

}

#endif /* !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS) */

#if !defined(HAVE_MBRTOWC) || defined(USE_FORCE_ASCII)

static int

decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen,

const char **reason, _Py_error_handler errors)

{

wchar_t *res;

unsigned char *in;

wchar_t *out;

size_t argsize = strlen(arg) + 1;

int surrogateescape;

if (get_surrogateescape(errors, &surrogateescape) < 0) {

return -3;

}

if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {

return -1;

}

res = PyMem_RawMalloc(argsize * sizeof(wchar_t));

if (!res) {

return -1;

}

out = res;

for (in = (unsigned char*)arg; *in; in++) {

unsigned char ch = *in;

if (ch < 128) {

*out++ = ch;

}

else {

if (!surrogateescape) {

PyMem_RawFree(res);

if (wlen) {

*wlen = in - (unsigned char*)arg;

}

if (reason) {

*reason = "decoding error";

}

return -2;

}

*out++ = 0xdc00 + ch;

}

}

*out = 0;

if (wlen != NULL) {

*wlen = out - res;

}

*wstr = res;

return 0;

}

#endif /* !HAVE_MBRTOWC */

static int

decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen,

const char **reason, _Py_error_handler errors)

{

wchar_t *res;

size_t argsize;

size_t count;

#ifdef HAVE_MBRTOWC

unsigned char *in;

wchar_t *out;

mbstate_t mbs;

#endif

int surrogateescape;

if (get_surrogateescape(errors, &surrogateescape) < 0) {

return -3;

}

#ifdef HAVE_BROKEN_MBSTOWCS

/* Some platforms have a broken implementation of

* mbstowcs which does not count the characters that

* would result from conversion. Use an upper bound.

*/

argsize = strlen(arg);

#else

argsize = _Py_mbstowcs(NULL, arg, 0);

#endif

if (argsize != DECODE_ERROR) {

if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {

return -1;

}

res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t));

if (!res) {

return -1;

}

count = _Py_mbstowcs(res, arg, argsize + 1);

if (count != DECODE_ERROR) {

*wstr = res;

if (wlen != NULL) {

*wlen = count;

}

return 0;

}

PyMem_RawFree(res);

}

/* Conversion failed. Fall back to escaping with surrogateescape. */

#ifdef HAVE_MBRTOWC

/* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */

/* Overallocate; as multi-byte characters are in the argument, the

actual output could use less memory. */

argsize = strlen(arg) + 1;

if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {

return -1;

}

res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t));

if (!res) {

return -1;

}

in = (unsigned char*)arg;

out = res;

memset(&mbs, 0, sizeof mbs);

while (argsize) {

size_t converted = _Py_mbrtowc(out, (char*)in, argsize, &mbs);

if (converted == 0) {

/* Reached end of string; null char stored. */

break;

}

if (converted == DECODE_ERROR || converted == INCOMPLETE_CHARACTER) {

if (!surrogateescape) {

goto decode_error;

}

/* Decoding error. Escape as UTF-8b, and start over in the initial

shift state. */

*out++ = 0xdc00 + *in++;

argsize--;

memset(&mbs, 0, sizeof mbs);

continue;

}

// _Py_mbrtowc() reject lone surrogate characters

assert(!Py_UNICODE_IS_SURROGATE(*out));

/* successfully converted some bytes */

in += converted;

argsize -= converted;

out++;

}

if (wlen != NULL) {

*wlen = out - res;

}

*wstr = res;

return 0;

decode_error:

PyMem_RawFree(res);

if (wlen) {

*wlen = in - (unsigned char*)arg;

}

if (reason) {

*reason = "decoding error";

}

return -2;

#else /* HAVE_MBRTOWC */

/* Cannot use C locale for escaping; manually escape as if charset

is ASCII (i.e. escape all bytes > 128. This will still roundtrip

correctly in the locale's charset, which must be an ASCII superset. */

return decode_ascii(arg, wstr, wlen, reason, errors);

#endif /* HAVE_MBRTOWC */

}

/* Decode a byte string from the locale encoding.

Use the strict error handler if 'surrogateescape' is zero. Use the

surrogateescape error handler if 'surrogateescape' is non-zero: undecodable

bytes are decoded as characters in range U+DC80..U+DCFF. If a byte sequence

can be decoded as a surrogate character, escape the bytes using the

surrogateescape error handler instead of decoding them.

On success, return 0 and write the newly allocated wide character string into

*wstr (use PyMem_RawFree() to free the memory). If wlen is not NULL, write

the number of wide characters excluding the null character into *wlen.

On memory allocation failure, return -1.

On decoding error, return -2. If wlen is not NULL, write the start of

invalid byte sequence in the input string into *wlen. If reason is not NULL,

write the decoding error message into *reason.

Return -3 if the error handler 'errors' is not supported.

Use the Py_EncodeLocaleEx() function to encode the character string back to

a byte string. */

int

_Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen,

const char **reason,

int current_locale, _Py_error_handler errors)

{

if (current_locale) {

#ifdef _Py_FORCE_UTF8_LOCALE

return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,

errors);

#else

return decode_current_locale(arg, wstr, wlen, reason, errors);

#endif

}

#ifdef _Py_FORCE_UTF8_FS_ENCODING

return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,

errors);

#else

int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1);

#ifdef MS_WINDOWS

use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0);

#endif

if (use_utf8) {

return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,

errors);

}

#ifdef USE_FORCE_ASCII

if (force_ascii == -1) {

force_ascii = check_force_ascii();

}

if (force_ascii) {

/* force ASCII encoding to workaround mbstowcs() issue */

return decode_ascii(arg, wstr, wlen, reason, errors);

}

#endif

return decode_current_locale(arg, wstr, wlen, reason, errors);

#endif /* !_Py_FORCE_UTF8_FS_ENCODING */

}

/* Decode a byte string from the locale encoding with the

surrogateescape error handler: undecodable bytes are decoded as characters

in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate

character, escape the bytes using the surrogateescape error handler instead

of decoding them.

Return a pointer to a newly allocated wide character string, use

PyMem_RawFree() to free the memory. If size is not NULL, write the number of

wide characters excluding the null character into *size

Return NULL on decoding error or memory allocation error. If *size* is not

NULL, *size is set to (size_t)-1 on memory error or set to (size_t)-2 on

decoding error.

Decoding errors should never happen, unless there is a bug in the C

library.

Use the Py_EncodeLocale() function to encode the character string back to a

byte string. */

wchar_t*

Py_DecodeLocale(const char* arg, size_t *wlen)

{

wchar_t *wstr;

int res = _Py_DecodeLocaleEx(arg, &wstr, wlen,

NULL, 0,

_Py_ERROR_SURROGATEESCAPE);

if (res != 0) {

assert(res != -3);

if (wlen != NULL) {

*wlen = (size_t)res;

}

return NULL;

}

return wstr;

}

static int

encode_current_locale(const wchar_t *text, char **str,

size_t *error_pos, const char **reason,

int raw_malloc, _Py_error_handler errors)

{

const size_t len = wcslen(text);

char *result = NULL, *bytes = NULL;

size_t i, size, converted;

wchar_t c, buf[2];

int surrogateescape;

if (get_surrogateescape(errors, &surrogateescape) < 0) {

return -3;

}

/* The function works in two steps:

1. compute the length of the output buffer in bytes (size)

2. outputs the bytes */

size = 0;

buf[1] = 0;

while (1) {

for (i=0; i < len; i++) {

c = text[i];

if (c >= 0xdc80 && c <= 0xdcff) {

if (!surrogateescape) {

goto encode_error;

}

/* UTF-8b surrogate */

if (bytes != NULL) {

*bytes++ = c - 0xdc00;

size--;

}

else {

size++;

}

continue;

}

else {

buf[0] = c;

if (bytes != NULL) {

converted = wcstombs(bytes, buf, size);

}

else {

converted = wcstombs(NULL, buf, 0);

}

if (converted == DECODE_ERROR) {

goto encode_error;

}

if (bytes != NULL) {

bytes += converted;

size -= converted;

}

else {

size += converted;

}

}

}

if (result != NULL) {

*bytes = '\0';

break;

}

size += 1; /* nul byte at the end */

if (raw_malloc) {

result = PyMem_RawMalloc(size);

}

else {

result = PyMem_Malloc(size);

}

if (result == NULL) {

return -1;

}

bytes = result;

}

*str = result;

return 0;

encode_error:

if (raw_malloc) {

PyMem_RawFree(result);

}

else {

PyMem_Free(result);

}

if (error_pos != NULL) {

*error_pos = i;

}

if (reason) {

*reason = "encoding error";

}

return -2;

}

/* Encode a string to the locale encoding.

Parameters:

* raw_malloc: if non-zero, allocate memory using PyMem_RawMalloc() instead

of PyMem_Malloc().

* current_locale: if non-zero, use the current LC_CTYPE, otherwise use

Python filesystem encoding.

* errors: error handler like "strict" or "surrogateescape".

Return value:

0: success, *str is set to a newly allocated decoded string.

-1: memory allocation failure

-2: encoding error, set *error_pos and *reason (if set).

-3: the error handler 'errors' is not supported.

*/

static int

encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos,

const char **reason,

int raw_malloc, int current_locale, _Py_error_handler errors)

{

if (current_locale) {

#ifdef _Py_FORCE_UTF8_LOCALE

return _Py_EncodeUTF8Ex(text, str, error_pos, reason,

raw_malloc, errors);

#else

return encode_current_locale(text, str, error_pos, reason,

raw_malloc, errors);

#endif

}

#ifdef _Py_FORCE_UTF8_FS_ENCODING

return _Py_EncodeUTF8Ex(text, str, error_pos, reason,

raw_malloc, errors);

#else

int use_utf8 = (_PyRuntime.preconfig.utf8_mode >= 1);

#ifdef MS_WINDOWS

use_utf8 |= (_PyRuntime.preconfig.legacy_windows_fs_encoding == 0);

#endif

if (use_utf8) {

return _Py_EncodeUTF8Ex(text, str, error_pos, reason,

raw_malloc, errors);

}

#ifdef USE_FORCE_ASCII

if (force_ascii == -1) {

force_ascii = check_force_ascii();

}

if (force_ascii) {

return encode_ascii(text, str, error_pos, reason,

raw_malloc, errors);

}

#endif

return encode_current_locale(text, str, error_pos, reason,

raw_malloc, errors);

#endif /* _Py_FORCE_UTF8_FS_ENCODING */

}

static char*

encode_locale(const wchar_t *text, size_t *error_pos,

int raw_malloc, int current_locale)

{

char *str;

int res = encode_locale_ex(text, &str, error_pos, NULL,

raw_malloc, current_locale,

_Py_ERROR_SURROGATEESCAPE);

if (res != -2 && error_pos) {

*error_pos = (size_t)-1;

}

if (res != 0) {

return NULL;

}

return str;

}

/* Encode a wide character string to the locale encoding with the

surrogateescape error handler: surrogate characters in the range

U+DC80..U+DCFF are converted to bytes 0x80..0xFF.

Return a pointer to a newly allocated byte string, use PyMem_Free() to free

the memory. Return NULL on encoding or memory allocation error.

If error_pos is not NULL, *error_pos is set to (size_t)-1 on success, or set

to the index of the invalid character on encoding error.

Use the Py_DecodeLocale() function to decode the bytes string back to a wide

character string. */

char*

Py_EncodeLocale(const wchar_t *text, size_t *error_pos)

{

return encode_locale(text, error_pos, 0, 0);

}

/* Similar to Py_EncodeLocale(), but result must be freed by PyMem_RawFree()

instead of PyMem_Free(). */

char*

_Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos)

{

return encode_locale(text, error_pos, 1, 0);

}

int

_Py_EncodeLocaleEx(const wchar_t *text, char **str,

size_t *error_pos, const char **reason,

int current_locale, _Py_error_handler errors)

{

return encode_locale_ex(text, str, error_pos, reason, 1,

current_locale, errors);

}

// Get the current locale encoding name:

//

// - Return "utf-8" if _Py_FORCE_UTF8_LOCALE macro is defined (ex: on Android)

// - Return "utf-8" if the UTF-8 Mode is enabled

// - On Windows, return the ANSI code page (ex: "cp1250")

// - Return "utf-8" if nl_langinfo(CODESET) returns an empty string.

// - Otherwise, return nl_langinfo(CODESET).

//

// Return NULL on memory allocation failure.

//

// See also config_get_locale_encoding()

wchar_t*

_Py_GetLocaleEncoding(void)

{

#ifdef _Py_FORCE_UTF8_LOCALE

// On Android langinfo.h and CODESET are missing,

// and UTF-8 is always used in mbstowcs() and wcstombs().

return _PyMem_RawWcsdup(L"utf-8");

#else

#ifdef MS_WINDOWS

wchar_t encoding[23];

unsigned int ansi_codepage = GetACP();

swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", ansi_codepage);

encoding[Py_ARRAY_LENGTH(encoding) - 1] = 0;

return _PyMem_RawWcsdup(encoding);

#else

const char *encoding = nl_langinfo(CODESET);

if (!encoding || encoding[0] == '\0') {

// Use UTF-8 if nl_langinfo() returns an empty string. It can happen on

// macOS if the LC_CTYPE locale is not supported.

return _PyMem_RawWcsdup(L"utf-8");

}

wchar_t *wstr;

int res = decode_current_locale(encoding, &wstr, NULL,

NULL, _Py_ERROR_SURROGATEESCAPE);

if (res < 0) {

return NULL;

}

return wstr;

#endif // !MS_WINDOWS

#endif // !_Py_FORCE_UTF8_LOCALE

}

PyObject *

_Py_GetLocaleEncodingObject(void)

{

wchar_t *encoding = _Py_GetLocaleEncoding();

if (encoding == NULL) {

PyErr_NoMemory();

return NULL;

}

PyObject *str = PyUnicode_FromWideChar(encoding, -1);

PyMem_RawFree(encoding);

return str;

}

#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION

/* Check whether current locale uses Unicode as internal wchar_t form. */

int

_Py_LocaleUsesNonUnicodeWchar(void)

{

/* Oracle Solaris uses non-Unicode internal wchar_t form for

non-Unicode locales and hence needs conversion to UTF first. */

char* codeset = nl_langinfo(CODESET);

if (!codeset) {

return 0;

}

/* 646 refers to ISO/IEC 646 standard that corresponds to ASCII encoding */

return (strcmp(codeset, "UTF-8") != 0 && strcmp(codeset, "646") != 0);

}

static wchar_t *

_Py_ConvertWCharForm(const wchar_t *source, Py_ssize_t size,

const char *tocode, const char *fromcode)

{

static_assert(sizeof(wchar_t) == 4, "wchar_t must be 32-bit");

/* Ensure we won't overflow the size. */

if (size > (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t))) {

PyErr_NoMemory();

return NULL;

}

/* the string doesn't have to be NULL terminated */

wchar_t* target = PyMem_Malloc(size * sizeof(wchar_t));

if (target == NULL) {

PyErr_NoMemory();

return NULL;

}

iconv_t cd = iconv_open(tocode, fromcode);

if (cd == (iconv_t)-1) {

PyErr_Format(PyExc_ValueError, "iconv_open() failed");

PyMem_Free(target);

return NULL;

}

char *inbuf = (char *) source;

char *outbuf = (char *) target;

size_t inbytesleft = sizeof(wchar_t) * size;

size_t outbytesleft = inbytesleft;

size_t ret = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);

if (ret == DECODE_ERROR) {

PyErr_Format(PyExc_ValueError, "iconv() failed");

PyMem_Free(target);

iconv_close(cd);

return NULL;

}