Skip to content

Commit 3dd3e26

Browse files
serhiy-storchakaserhiy-storchaka
committed
Issue python#22896: Avoid to use PyObject_AsCharBuffer(), PyObject_AsReadBuffer()
and PyObject_AsWriteBuffer().
2 parents 0b2a6dc + 4fdb684 commit 3dd3e26

18 files changed

Lines changed: 420 additions & 395 deletions

File tree

‎Include/bytes_methods.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len);
2222
extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len);
2323

2424
/* The maketrans() static method. */
25-
extern PyObject* _Py_bytes_maketrans(PyObject *frm, PyObject *to);
25+
extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to);
2626

2727
/* Shared __doc__ strings. */
2828
extern const char _Py_isspace__doc__[];

‎Lib/ctypes/test/test_frombuffer.py‎

Lines changed: 33 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ def __init__(self):
1010
self._init_called = True
1111

1212
class Test(unittest.TestCase):
13-
def test_fom_buffer(self):
13+
def test_from_buffer(self):
1414
a = array.array("i", range(16))
1515
x = (c_int * 16).from_buffer(a)
1616

@@ -23,25 +23,37 @@ def test_fom_buffer(self):
2323
a[0], a[-1] = 200, -200
2424
self.assertEqual(x[:], a.tolist())
2525

26-
self.assertIn(a, x._objects.values())
26+
self.assertRaises(BufferError, a.append, 100)
27+
self.assertRaises(BufferError, a.pop)
2728

28-
self.assertRaises(ValueError,
29-
c_int.from_buffer, a, -1)
29+
del x; del y; gc.collect(); gc.collect(); gc.collect()
30+
a.append(100)
31+
a.pop()
32+
x = (c_int * 16).from_buffer(a)
33+
34+
self.assertIn(a, [obj.obj if isinstance(obj, memoryview) else obj
35+
for obj in x._objects.values()])
3036

3137
expected = x[:]
3238
del a; gc.collect(); gc.collect(); gc.collect()
3339
self.assertEqual(x[:], expected)
3440

35-
self.assertRaises(TypeError,
36-
(c_char * 16).from_buffer, "a" * 16)
41+
with self.assertRaises(TypeError):
42+
(c_char * 16).from_buffer(b"a" * 16)
43+
with self.assertRaises(TypeError):
44+
(c_char * 16).from_buffer("a" * 16)
3745

38-
def test_fom_buffer_with_offset(self):
46+
def test_from_buffer_with_offset(self):
3947
a = array.array("i", range(16))
4048
x = (c_int * 15).from_buffer(a, sizeof(c_int))
4149

4250
self.assertEqual(x[:], a.tolist()[1:])
43-
self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
44-
self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
51+
with self.assertRaises(ValueError):
52+
c_int.from_buffer(a, -1)
53+
with self.assertRaises(ValueError):
54+
(c_int * 16).from_buffer(a, sizeof(c_int))
55+
with self.assertRaises(ValueError):
56+
(c_int * 1).from_buffer(a, 16 * sizeof(c_int))
4557

4658
def test_from_buffer_copy(self):
4759
a = array.array("i", range(16))
@@ -56,26 +68,30 @@ def test_from_buffer_copy(self):
5668
a[0], a[-1] = 200, -200
5769
self.assertEqual(x[:], list(range(16)))
5870

59-
self.assertEqual(x._objects, None)
71+
a.append(100)
72+
self.assertEqual(x[:], list(range(16)))
6073

61-
self.assertRaises(ValueError,
62-
c_int.from_buffer, a, -1)
74+
self.assertEqual(x._objects, None)
6375

6476
del a; gc.collect(); gc.collect(); gc.collect()
6577
self.assertEqual(x[:], list(range(16)))
6678

6779
x = (c_char * 16).from_buffer_copy(b"a" * 16)
6880
self.assertEqual(x[:], b"a" * 16)
81+
with self.assertRaises(TypeError):
82+
(c_char * 16).from_buffer_copy("a" * 16)
6983

70-
def test_fom_buffer_copy_with_offset(self):
84+
def test_from_buffer_copy_with_offset(self):
7185
a = array.array("i", range(16))
7286
x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
7387

7488
self.assertEqual(x[:], a.tolist()[1:])
75-
self.assertRaises(ValueError,
76-
(c_int * 16).from_buffer_copy, a, sizeof(c_int))
77-
self.assertRaises(ValueError,
78-
(c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int))
89+
with self.assertRaises(ValueError):
90+
c_int.from_buffer_copy(a, -1)
91+
with self.assertRaises(ValueError):
92+
(c_int * 16).from_buffer_copy(a, sizeof(c_int))
93+
with self.assertRaises(ValueError):
94+
(c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int))
7995

8096
if __name__ == '__main__':
8197
unittest.main()

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ Release date: TBA
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #22896: Avoid using PyObject_AsCharBuffer(), PyObject_AsReadBuffer()
14+
and PyObject_AsWriteBuffer().
15+
1316
- Issue #21295: Revert some changes (issue #16795) to AST line numbers and
1417
column offsets that constituted a regression.
1518

‎Modules/_codecsmodule.c‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -288,8 +288,6 @@ unicode_internal_decode(PyObject *self,
288288
{
289289
PyObject *obj;
290290
const char *errors = NULL;
291-
const char *data;
292-
Py_ssize_t size;
293291

294292
if (!PyArg_ParseTuple(args, "O|z:unicode_internal_decode",
295293
&obj, &errors))
@@ -302,11 +300,16 @@ unicode_internal_decode(PyObject *self,
302300
return codec_tuple(obj, PyUnicode_GET_LENGTH(obj));
303301
}
304302
else {
305-
if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))
303+
Py_buffer view;
304+
PyObject *result;
305+
if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0)
306306
return NULL;
307307

308-
return codec_tuple(_PyUnicode_DecodeUnicodeInternal(data, size, errors),
309-
size);
308+
result = codec_tuple(
309+
_PyUnicode_DecodeUnicodeInternal(view.buf, view.len, errors),
310+
view.len);
311+
PyBuffer_Release(&view);
312+
return result;
310313
}
311314
}
312315

@@ -731,8 +734,6 @@ unicode_internal_encode(PyObject *self,
731734
{
732735
PyObject *obj;
733736
const char *errors = NULL;
734-
const char *data;
735-
Py_ssize_t len, size;
736737

737738
if (PyErr_WarnEx(PyExc_DeprecationWarning,
738739
"unicode_internal codec has been deprecated",
@@ -745,6 +746,7 @@ unicode_internal_encode(PyObject *self,
745746

746747
if (PyUnicode_Check(obj)) {
747748
Py_UNICODE *u;
749+
Py_ssize_t len, size;
748750

749751
if (PyUnicode_READY(obj) < 0)
750752
return NULL;
@@ -759,9 +761,13 @@ unicode_internal_encode(PyObject *self,
759761
PyUnicode_GET_LENGTH(obj));
760762
}
761763
else {
762-
if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))
764+
Py_buffer view;
765+
PyObject *result;
766+
if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0)
763767
return NULL;
764-
return codec_tuple(PyBytes_FromStringAndSize(data, size), size);
768+
result = codec_tuple(PyBytes_FromStringAndSize(view.buf, view.len), view.len);
769+
PyBuffer_Release(&view);
770+
return result;
765771
}
766772
}
767773

‎Modules/_ctypes/_ctypes.c‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -463,39 +463,45 @@ KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep);
463463
static PyObject *
464464
CDataType_from_buffer(PyObject *type, PyObject *args)
465465
{
466-
void *buffer;
467-
Py_ssize_t buffer_len;
466+
Py_buffer buffer;
468467
Py_ssize_t offset = 0;
469-
PyObject *obj, *result;
468+
PyObject *result, *mv;
470469
StgDictObject *dict = PyType_stgdict(type);
471470
assert (dict);
472471

473-
if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset))
474-
return NULL;
475-
476-
if (-1 == PyObject_AsWriteBuffer(obj, &buffer, &buffer_len))
472+
if (!PyArg_ParseTuple(args, "w*|n:from_buffer", &buffer, &offset))
477473
return NULL;
478474

479475
if (offset < 0) {
480476
PyErr_SetString(PyExc_ValueError,
481477
"offset cannot be negative");
478+
PyBuffer_Release(&buffer);
482479
return NULL;
483480
}
484-
if (dict->size > buffer_len - offset) {
481+
if (dict->size > buffer.len - offset) {
485482
PyErr_Format(PyExc_ValueError,
486483
"Buffer size too small (%zd instead of at least %zd bytes)",
487-
buffer_len, dict->size + offset);
484+
buffer.len, dict->size + offset);
485+
PyBuffer_Release(&buffer);
488486
return NULL;
489487
}
490488

491-
result = PyCData_AtAddress(type, (char *)buffer + offset);
492-
if (result == NULL)
489+
result = PyCData_AtAddress(type, (char *)buffer.buf + offset);
490+
if (result == NULL) {
491+
PyBuffer_Release(&buffer);
493492
return NULL;
493+
}
494494

495-
Py_INCREF(obj);
496-
if (-1 == KeepRef((CDataObject *)result, -1, obj)) {
495+
mv = PyMemoryView_FromBuffer(&buffer);
496+
if (mv == NULL) {
497+
PyBuffer_Release(&buffer);
497498
return NULL;
498499
}
500+
/* Hack the memoryview so that it will release the buffer. */
501+
((PyMemoryViewObject *)mv)->mbuf->master.obj = buffer.obj;
502+
((PyMemoryViewObject *)mv)->view.obj = buffer.obj;
503+
if (-1 == KeepRef((CDataObject *)result, -1, mv))
504+
result = NULL;
499505
return result;
500506
}
501507

@@ -508,37 +514,36 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
508514
static PyObject *
509515
CDataType_from_buffer_copy(PyObject *type, PyObject *args)
510516
{
511-
const void *buffer;
512-
Py_ssize_t buffer_len;
517+
Py_buffer buffer;
513518
Py_ssize_t offset = 0;
514-
PyObject *obj, *result;
519+
PyObject *result;
515520
StgDictObject *dict = PyType_stgdict(type);
516521
assert (dict);
517522

518-
if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset))
519-
return NULL;
520-
521-
if (-1 == PyObject_AsReadBuffer(obj, (const void**)&buffer, &buffer_len))
523+
if (!PyArg_ParseTuple(args, "y*|n:from_buffer", &buffer, &offset))
522524
return NULL;
523525

524526
if (offset < 0) {
525527
PyErr_SetString(PyExc_ValueError,
526528
"offset cannot be negative");
529+
PyBuffer_Release(&buffer);
527530
return NULL;
528531
}
529532

530-
if (dict->size > buffer_len - offset) {
533+
if (dict->size > buffer.len - offset) {
531534
PyErr_Format(PyExc_ValueError,
532535
"Buffer size too small (%zd instead of at least %zd bytes)",
533-
buffer_len, dict->size + offset);
536+
buffer.len, dict->size + offset);
537+
PyBuffer_Release(&buffer);
534538
return NULL;
535539
}
536540

537541
result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL);
538-
if (result == NULL)
539-
return NULL;
540-
memcpy(((CDataObject *)result)->b_ptr,
541-
(char *)buffer+offset, dict->size);
542+
if (result != NULL) {
543+
memcpy(((CDataObject *)result)->b_ptr,
544+
(char *)buffer.buf + offset, dict->size);
545+
}
546+
PyBuffer_Release(&buffer);
542547
return result;
543548
}
544549

‎Modules/_io/bytesio.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -553,28 +553,30 @@ PyDoc_STRVAR(readinto_doc,
553553
"is set not to block as has no data to read.");
554554

555555
static PyObject *
556-
bytesio_readinto(bytesio *self, PyObject *buffer)
556+
bytesio_readinto(bytesio *self, PyObject *arg)
557557
{
558-
void *raw_buffer;
558+
Py_buffer buffer;
559559
Py_ssize_t len, n;
560560

561561
CHECK_CLOSED(self, NULL);
562562

563-
if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1)
563+
if (!PyArg_Parse(arg, "w*", &buffer))
564564
return NULL;
565565

566566
/* adjust invalid sizes */
567+
len = buffer.len;
567568
n = self->string_size - self->pos;
568569
if (len > n) {
569570
len = n;
570571
if (len < 0)
571572
len = 0;
572573
}
573574

574-
memcpy(raw_buffer, self->buf + self->pos, len);
575+
memcpy(buffer.buf, self->buf + self->pos, len);
575576
assert(self->pos + len < PY_SSIZE_T_MAX);
576577
assert(len >= 0);
577578
self->pos += len;
579+
PyBuffer_Release(&buffer);
578580

579581
return PyLong_FromSsize_t(len);
580582
}

‎Modules/_sqlite/connection.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -522,19 +522,20 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val)
522522
return -1;
523523
sqlite3_result_text(context, str, -1, SQLITE_TRANSIENT);
524524
} else if (PyObject_CheckBuffer(py_val)) {
525-
const char* buffer;
526-
Py_ssize_t buflen;
527-
if (PyObject_AsCharBuffer(py_val, &buffer, &buflen) != 0) {
525+
Py_buffer view;
526+
if (PyObject_GetBuffer(py_val, &view, PyBUF_SIMPLE) != 0) {
528527
PyErr_SetString(PyExc_ValueError,
529528
"could not convert BLOB to buffer");
530529
return -1;
531530
}
532-
if (buflen > INT_MAX) {
531+
if (view.len > INT_MAX) {
533532
PyErr_SetString(PyExc_OverflowError,
534533
"BLOB longer than INT_MAX bytes");
534+
PyBuffer_Release(&view);
535535
return -1;
536536
}
537-
sqlite3_result_blob(context, buffer, (int)buflen, SQLITE_TRANSIENT);
537+
sqlite3_result_blob(context, view.buf, (int)view.len, SQLITE_TRANSIENT);
538+
PyBuffer_Release(&view);
538539
} else {
539540
return -1;
540541
}

‎Modules/_sqlite/statement.c‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con
9494
int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObject* parameter)
9595
{
9696
int rc = SQLITE_OK;
97-
const char* buffer;
9897
char* string;
9998
Py_ssize_t buflen;
10099
parameter_type paramtype;
@@ -145,18 +144,22 @@ int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObjec
145144
}
146145
rc = sqlite3_bind_text(self->st, pos, string, (int)buflen, SQLITE_TRANSIENT);
147146
break;
148-
case TYPE_BUFFER:
149-
if (PyObject_AsCharBuffer(parameter, &buffer, &buflen) != 0) {
147+
case TYPE_BUFFER: {
148+
Py_buffer view;
149+
if (PyObject_GetBuffer(parameter, &view, PyBUF_SIMPLE) != 0) {
150150
PyErr_SetString(PyExc_ValueError, "could not convert BLOB to buffer");
151151
return -1;
152152
}
153-
if (buflen > INT_MAX) {
153+
if (view.len > INT_MAX) {
154154
PyErr_SetString(PyExc_OverflowError,
155155
"BLOB longer than INT_MAX bytes");
156+
PyBuffer_Release(&view);
156157
return -1;
157158
}
158-
rc = sqlite3_bind_blob(self->st, pos, buffer, buflen, SQLITE_TRANSIENT);
159+
rc = sqlite3_bind_blob(self->st, pos, view.buf, (int)view.len, SQLITE_TRANSIENT);
160+
PyBuffer_Release(&view);
159161
break;
162+
}
160163
case TYPE_UNKNOWN:
161164
rc = -1;
162165
}

0 commit comments

Comments
 (0)