From 806ec756b9601f71c7b3845f5412251e83a98ef1 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Sat, 5 Aug 2017 15:30:59 +0300 Subject: [PATCH 01/10] bpo-16865: Support arrays >=2GB in ctypes --- Lib/ctypes/test/test_arrays.py | 7 +++++++ Modules/_ctypes/_ctypes.c | 9 +++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/ctypes/test/test_arrays.py index 4ed566b48e67e88..6e562cfd24e6638 100644 --- a/Lib/ctypes/test/test_arrays.py +++ b/Lib/ctypes/test/test_arrays.py @@ -1,4 +1,6 @@ import unittest +from test.support import bigmemtest, _2G +import sys from ctypes import * from ctypes.test import need_symbol @@ -181,5 +183,10 @@ class T(Array): _type_ = c_int _length_ = 1.87 + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=_2G, memuse=1, dry_run=False) + def test_large_array(self, size): + c_char * size + if __name__ == '__main__': unittest.main() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fe39567374d0383..be2ba64094522f8 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1325,8 +1325,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) StgDictObject *stgdict; StgDictObject *itemdict; PyObject *length_attr, *type_attr; - long length; - int overflow; + Py_ssize_t length; Py_ssize_t itemsize, itemalign; /* create the new instance (which is a class, @@ -1348,10 +1347,8 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_XDECREF(length_attr); goto error; } - length = PyLong_AsLongAndOverflow(length_attr, &overflow); - if (overflow) { - PyErr_SetString(PyExc_OverflowError, - "The '_length_' attribute is too large"); + length = PyLong_AsSsize_t(length_attr); + if (length == -1 && PyErr_Occurred()) { Py_DECREF(length_attr); goto error; } From c5e3ed7b3d1ceeb7e010a2f0ca72a440b45c46a6 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Fri, 29 Sep 2017 16:32:44 +0300 Subject: [PATCH 02/10] bpo-16865: Switch to PyLong_As{Long,LongLong}AndOverflow --- Modules/_ctypes/_ctypes.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index be2ba64094522f8..23b71a323491437 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1325,6 +1325,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) StgDictObject *stgdict; StgDictObject *itemdict; PyObject *length_attr, *type_attr; + int overflow; Py_ssize_t length; Py_ssize_t itemsize, itemalign; @@ -1347,8 +1348,15 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_XDECREF(length_attr); goto error; } - length = PyLong_AsSsize_t(length_attr); - if (length == -1 && PyErr_Occurred()) { + +#if SIZEOF_SIZE_T <= SIZEOF_LONG + length = PyLong_AsLongAndOverflow(length_attr, &overflow); +#else + length = PyLong_AsLongLongAndOverflow(length_attr, &overflow); +#endif + if (overflow) { + PyErr_SetString(PyExc_OverflowError, + "The '_length_' attribute is too large"); Py_DECREF(length_attr); goto error; } From c3c00834eb45ce538d94dd9c9b170d08302e6614 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Fri, 29 Sep 2017 16:36:32 +0300 Subject: [PATCH 03/10] bpo-16865: Minor reorganization for reduced diff --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6e0dc7b45fe462d..6a26436af46fbb4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1390,8 +1390,8 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) StgDictObject *stgdict; StgDictObject *itemdict; PyObject *length_attr, *type_attr; - int overflow; Py_ssize_t length; + int overflow; Py_ssize_t itemsize, itemalign; /* create the new instance (which is a class, From 9cf7cac926dc318acdff80596ce26b13134f8c49 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Fri, 29 Sep 2017 16:40:45 +0300 Subject: [PATCH 04/10] bpo-16865: Add NEWS entry --- .../NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst diff --git a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst new file mode 100644 index 000000000000000..d09140e34c1da1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst @@ -0,0 +1 @@ +Support arrays >=2GB in `ctypes` From b77729c1c3ff9383e80580187d83d02a8c744242 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Oct 2017 11:05:30 +0300 Subject: [PATCH 05/10] Update 2017-09-29-16-40-38.bpo-16865.l-f6I_.rst --- .../next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst index d09140e34c1da1b..499c9ac81ba809c 100644 --- a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst +++ b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst @@ -1 +1 @@ -Support arrays >=2GB in `ctypes` +Support arrays >=2GiB in `ctypes` From ff4d8ef72620958be3c660d9b6e58ceb959f3705 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Oct 2017 11:11:18 +0300 Subject: [PATCH 06/10] Update 2017-09-29-16-40-38.bpo-16865.l-f6I_.rst --- .../next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst index 499c9ac81ba809c..4dd6f84fcec00cc 100644 --- a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst +++ b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst @@ -1 +1 @@ -Support arrays >=2GiB in `ctypes` +Support arrays >=2GiB in `ctypes`. Patch by Segev Finer. From aa9e7a14c881a8030a3ee4563e923882d31239ff Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Sun, 8 Oct 2017 13:13:46 +0300 Subject: [PATCH 07/10] Revert "bpo-16865: Switch to PyLong_As{Long,LongLong}AndOverflow" This reverts commit c5e3ed7b3d1ceeb7e010a2f0ca72a440b45c46a6. --- Modules/_ctypes/_ctypes.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6a26436af46fbb4..d2a1b508091bf9f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1413,15 +1413,8 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_XDECREF(length_attr); goto error; } - -#if SIZEOF_SIZE_T <= SIZEOF_LONG - length = PyLong_AsLongAndOverflow(length_attr, &overflow); -#else - length = PyLong_AsLongLongAndOverflow(length_attr, &overflow); -#endif - if (overflow) { - PyErr_SetString(PyExc_OverflowError, - "The '_length_' attribute is too large"); + length = PyLong_AsSsize_t(length_attr); + if (length == -1 && PyErr_Occurred()) { Py_DECREF(length_attr); goto error; } From ad24633b48b534ed90b3c4dc29e509d3ee2c329f Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Sun, 8 Oct 2017 13:16:55 +0300 Subject: [PATCH 08/10] bpo-16865: Set a better exception on overflow of the _length_ attribute --- Modules/_ctypes/_ctypes.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d2a1b508091bf9f..fdc2bd930d2aacd 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1391,7 +1391,6 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) StgDictObject *itemdict; PyObject *length_attr, *type_attr; Py_ssize_t length; - int overflow; Py_ssize_t itemsize, itemalign; /* create the new instance (which is a class, @@ -1415,6 +1414,9 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } length = PyLong_AsSsize_t(length_attr); if (length == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) + PyErr_SetString(PyExc_OverflowError, + "The '_length_' attribute is too large"); Py_DECREF(length_attr); goto error; } From d69cbfac26ba14c93ec9dcead04d9f46574d4d15 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 13 May 2018 13:23:07 +0300 Subject: [PATCH 09/10] Fix rst markup. --- .../next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst index 4dd6f84fcec00cc..afaff736bf1ce75 100644 --- a/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst +++ b/Misc/NEWS.d/next/Library/2017-09-29-16-40-38.bpo-16865.l-f6I_.rst @@ -1 +1 @@ -Support arrays >=2GiB in `ctypes`. Patch by Segev Finer. +Support arrays >=2GiB in :mod:`ctypes`. Patch by Segev Finer. From eae2ca3147ebf7ba540de97ccd73546d95bc969b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 14 May 2018 20:30:39 +0300 Subject: [PATCH 10/10] Clean up the code. --- Modules/_ctypes/_ctypes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fdc2bd930d2aacd..f44a95ca3d69ce4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1413,14 +1413,14 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } length = PyLong_AsSsize_t(length_attr); + Py_DECREF(length_attr); if (length == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_SetString(PyExc_OverflowError, "The '_length_' attribute is too large"); - Py_DECREF(length_attr); + } goto error; } - Py_DECREF(length_attr); type_attr = PyObject_GetAttrString((PyObject *)result, "_type_"); if (!type_attr) {