Merge branch 'master' into disable-wchar-cache

python · serhiy-storchaka · Jul 10, 2020 · Oct 26, 2018 · Mar 19, 2019 · Mar 19, 2019
commit 3c1ab31a8b8eb889de7790047f4b05caba20aaaf
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -69,6 +69,11 @@
 except ImportError:
     resource = None
 
+try:
+    import _hashlib
+except ImportError:
+    _hashlib = None
+
 try:
     from _testcapi import unicode_legacy_string
 except ImportError:

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
@@ -1366,9 +1366,6 @@ WCharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored))
 static int
 WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored))
 {
-    Py_ssize_t result = 0;
-    Py_ssize_t len;
-
     if (value == NULL) {
         PyErr_SetString(PyExc_TypeError,
                         "can't delete attribute");
@@ -1379,38 +1376,24 @@ WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored
                         "unicode string expected instead of %s instance",
                         Py_TYPE(value)->tp_name);
         return -1;
-    } else
-        Py_INCREF(value);
+    }
 
-#if USE_UNICODE_WCHAR_CACHE
-    len = PyUnicode_GetSize(value);
+    Py_ssize_t size = self->b_size / sizeof(wchar_t);
+    Py_ssize_t len = PyUnicode_AsWideChar(value, NULL, 0);
     if (len < 0) {
-        Py_DECREF(value);
         return -1;
     }
-#else /* USE_UNICODE_WCHAR_CACHE */
-    len = PyUnicode_AsWideChar(value, NULL, 0);
-    if (len < 0) {
-        Py_DECREF(value);
+    // PyUnicode_AsWideChar() returns number of wchars including trailing null byte,
+    // when it is called with NULL.
+    assert(len > 0);
+    if (len - 1 > size) {
+        PyErr_SetString(PyExc_ValueError, "string too long");
         return -1;
     }
-    assert(len > 0);
-    len--;
-#endif /* USE_UNICODE_WCHAR_CACHE */
-    if ((size_t)len > self->b_size/sizeof(wchar_t)) {
-        PyErr_SetString(PyExc_ValueError,
-                        "string too long");
-        Py_DECREF(value);
+    if (PyUnicode_AsWideChar(value, (wchar_t *)self->b_ptr, size) < 0) {
         return -1;
     }
-    result = PyUnicode_AsWideChar(value,
-                                  (wchar_t *)self->b_ptr,
-                                  self->b_size/sizeof(wchar_t));
-    if (result >= 0 && (size_t)result < self->b_size/sizeof(wchar_t))
-        ((wchar_t *)self->b_ptr)[result] = (wchar_t)0;
-    Py_DECREF(value);
-
-    return result >= 0 ? 0 : -1;
+    return 0;
 }
 
 static PyGetSetDef WCharArray_getsets[] = {

diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
@@ -1315,8 +1315,16 @@ static PyObject *load_library(PyObject *self, PyObject *args)
 #if !USE_UNICODE_WCHAR_CACHE
     PyMem_Free(name);
 #endif /* USE_UNICODE_WCHAR_CACHE */
-    if (!hMod)
-        return PyErr_SetFromWindowsErr(GetLastError());
+    if (err == ERROR_MOD_NOT_FOUND) {
+        PyErr_Format(PyExc_FileNotFoundError,
+                     ("Could not find module '%.500S' (or one of its "
+                      "dependencies). Try using the full path with "
+                      "constructor syntax."),
+                     nameobj);
+        return NULL;
+    } else if (err) {
+        return PyErr_SetFromWindowsErr(err);
+    }
 #ifdef _WIN64
     return PyLong_FromVoidPtr(hMod);
 #else

diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c
@@ -1228,8 +1228,6 @@ U_get(void *ptr, Py_ssize_t size)
 static PyObject *
 U_set(void *ptr, PyObject *value, Py_ssize_t length)
 {
-    Py_ssize_t size;
-
     /* It's easier to calculate in characters than in bytes */
     length /= sizeof(wchar_t);
 
@@ -1240,16 +1238,14 @@ U_set(void *ptr, PyObject *value, Py_ssize_t length)
         return NULL;
     }
 
-#if USE_UNICODE_WCHAR_CACHE
-    size = PyUnicode_GetSize(value);
-    if (size < 0)
-        return NULL;
-#else /* USE_UNICODE_WCHAR_CACHE */
-    size = PyUnicode_AsWideChar(value, NULL, 0);
-    if (size < 0)
+    Py_ssize_t size = PyUnicode_AsWideChar(value, NULL, 0);
+    if (size < 0) {
         return NULL;
+    }
+    // PyUnicode_AsWideChar() returns number of wchars including trailing null byte,
+    // when it is called with NULL.
     size--;
-#endif /* USE_UNICODE_WCHAR_CACHE */
+    assert(size >= 0);
     if (size > length) {
         PyErr_Format(PyExc_ValueError,
                      "string too long (%zd, maximum length %zd)",
@@ -1424,12 +1420,8 @@ BSTR_set(void *ptr, PyObject *value, Py_ssize_t size)
     /* create a BSTR from value */
     if (value) {
         Py_ssize_t wsize;
-#if USE_UNICODE_WCHAR_CACHE
-        wvalue = PyUnicode_AsUnicodeAndSize(value, &wsize);
-#else /* USE_UNICODE_WCHAR_CACHE */
-        wvalue = PyUnicode_AsWideCharString(value, &wsize);
-#endif /* USE_UNICODE_WCHAR_CACHE */
-        if (wvalue == NULL)
+        wchar_t *wvalue = PyUnicode_AsWideCharString(value, &wsize);
+        if (wvalue == NULL) {
             return NULL;
         }
         if ((unsigned) wsize != wsize) {
@@ -1438,9 +1430,7 @@ BSTR_set(void *ptr, PyObject *value, Py_ssize_t size)
             return NULL;
         }
         bstr = SysAllocStringLen(wvalue, (unsigned)wsize);
-#if !USE_UNICODE_WCHAR_CACHE
         PyMem_Free(wvalue);
-#endif /* USE_UNICODE_WCHAR_CACHE */
     } else
         bstr = NULL;
 

diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
@@ -10255,65 +10255,10 @@ static PyObject *
 os_putenv_impl(PyObject *module, PyObject *name, PyObject *value)
 /*[clinic end generated code: output=d29a567d6b2327d2 input=ba586581c2e6105f]*/
 {
-    Py_ssize_t size;
-
-    /* Search from index 1 because on Windows starting '=' is allowed for
-       defining hidden environment variables. */
-    if (PyUnicode_GET_LENGTH(name) == 0 ||
-        PyUnicode_FindChar(name, '=', 1, PyUnicode_GET_LENGTH(name), 1) != -1)
-    {
-        PyErr_SetString(PyExc_ValueError, "illegal environment variable name");
-        return NULL;
-    }
-    PyObject *buffer = PyUnicode_FromFormat("%U=%U", name, value);
-    if (buffer == NULL) {
-        return NULL;
-    }
-
-#if USE_UNICODE_WCHAR_CACHE
-    const wchar_t *env = PyUnicode_AsUnicodeAndSize(buffer, &size);
-    if (env == NULL)
-        goto error;
-#else /* USE_UNICODE_WCHAR_CACHE */
-    size = PyUnicode_AsWideChar(buffer, NULL, 0);
-    if (size < 0) {
+    if (PySys_Audit("os.putenv", "OO", name, value) < 0) {
         return NULL;
     }
-    if ((size_t)size > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
-        return PyErr_NoMemory();
-    }
-    PyObject *bytes = PyBytes_FromStringAndSize(NULL, size * sizeof(wchar_t));
-    if (bytes == NULL) {
-        goto error;
-    }
-    wchar_t *env = (wchar_t *)PyBytes_AS_STRING(bytes);
-    size = PyUnicode_AsWideChar(buffer, env, size);
-    assert(size >= 0);
-    Py_DECREF(buffer);
-    buffer = bytes;
-#endif /* USE_UNICODE_WCHAR_CACHE */
-    if (size > _MAX_ENV) {
-        PyErr_Format(PyExc_ValueError,
-                     "the environment variable is longer than %u characters",
-                     _MAX_ENV);
-        goto error;
-    }
-    if (wcslen(env) != (size_t)size) {
-        PyErr_SetString(PyExc_ValueError, "embedded null character");
-        goto error;
-    }
-
-    if (_wputenv(env)) {
-        posix_error();
-        goto error;
-    }
-
-    posix_putenv_garbage_setitem(name, buffer);
-    Py_RETURN_NONE;
-
-error:
-    Py_DECREF(buffer);
-    return NULL;
+    return win32_putenv(name, value);
 }
 #else
 /*[clinic input]

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
@@ -541,12 +541,12 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
             else
 #endif /* HAVE_UNICODE_WCHAR_CACHE */
             {
-                ASSERT(kind == PyUnicode_1BYTE_KIND
-                       || kind == PyUnicode_2BYTE_KIND
-                       || kind == PyUnicode_4BYTE_KIND);
-                ASSERT(ascii->state.compact == 0);
-                ASSERT(ascii->state.ready == 1);
-                ASSERT(data != NULL);
+                CHECK(kind == PyUnicode_1BYTE_KIND
+                                     || kind == PyUnicode_2BYTE_KIND
+                                     || kind == PyUnicode_4BYTE_KIND);
+                CHECK(ascii->state.compact == 0);
+                CHECK(ascii->state.ready == 1);
+                CHECK(data != NULL);
                 if (ascii->state.ascii) {
                     CHECK(compact->utf8 == data);
                     CHECK(compact->utf8_length == ascii->length);
@@ -573,10 +573,10 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
 #endif /* HAVE_UNICODE_WCHAR_CACHE */
 
         if (compact->utf8 == NULL)
-            ASSERT(compact->utf8_length == 0);
+            CHECK(compact->utf8_length == 0);
 #if HAVE_UNICODE_WCHAR_CACHE
         if (ascii->wstr == NULL)
-            ASSERT(compact->wstr_length == 0);
+            CHECK(compact->wstr_length == 0);
 #endif /* HAVE_UNICODE_WCHAR_CACHE */
     }
 
@@ -1369,7 +1369,7 @@ _PyUnicode_Dump(PyObject *op)
     printf(", ");
     if (ascii->wstr == data)
         printf("shared ");
-    printf("wstr=%p", ascii->wstr);
+    printf("wstr=%p", (void *)ascii->wstr);
 #endif /* HAVE_UNICODE_WCHAR_CACHE */
 
     if (!(ascii->state.ascii == 1 && ascii->state.compact == 1)) {
@@ -1962,10 +1962,11 @@ unicode_dealloc(PyObject *unicode)
     }
 
 #if HAVE_UNICODE_WCHAR_CACHE
-    if (_PyUnicode_HAS_WSTR_MEMORY(unicode))
+    if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) {
         PyObject_DEL(_PyUnicode_WSTR(unicode));
+    }
 #endif /* HAVE_UNICODE_WCHAR_CACHE */
-    if (_PyUnicode_HAS_UTF8_MEMORY(unicode))
+    if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
         PyObject_DEL(_PyUnicode_UTF8(unicode));
     }
     if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) {