From 98d883b7dae8dcc54388b707a5111dcf6d9e1092 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jul 2023 14:20:38 +0200 Subject: [PATCH] WIP: gh-106593: PyResource API * Add PyUnicode_AsUTF8Resource() * Add PyBytes_AsStringResource() * Add PySequence_AsObjectArray() * Add Include/pyresource.h * Add PyResource_Close() to the stable ABI * compute_abstract_methods(): Replace PySequence_Fast() with PySequence_AsObjectArray() --- Doc/data/stable_abi.dat | 2 + Include/Python.h | 1 + Include/abstract.h | 6 +++ Include/bytearrayobject.h | 3 +- Include/bytesobject.h | 3 +- Include/ceval.h | 3 +- Include/cpython/unicodeobject.h | 15 +++++-- Include/pycapsule.h | 5 ++- Include/pyresource.h | 21 ++++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + Makefile.pre.in | 1 + Misc/stable_abi.toml | 5 +++ Modules/_abc.c | 44 ++++++++++++-------- Objects/abstract.c | 66 ++++++++++++++++++++++++++++++ Objects/bytearrayobject.c | 15 ++++++- Objects/bytesobject.c | 13 ++++++ Objects/capsule.c | 14 +++++++ Objects/object.c | 14 +++++++ Objects/unicodeobject.c | 36 +++++++++++++++- PC/python3dll.c | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ Python/ceval.c | 24 +++++++++++ 23 files changed, 271 insertions(+), 26 deletions(-) create mode 100644 Include/pyresource.h diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index e3dd3dab27a0358..9d82d706a1c4a71 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -545,6 +545,8 @@ function,PyObject_VectorcallMethod,3.12,, var,PyProperty_Type,3.2,, var,PyRangeIter_Type,3.2,, var,PyRange_Type,3.2,, +type,PyResource,3.13,,full-abi +function,PyResource_Close,3.13,, var,PyReversed_Type,3.2,, function,PySeqIter_New,3.2,, var,PySeqIter_Type,3.2,, diff --git a/Include/Python.h b/Include/Python.h index 07f6c202a7f1261..e004b6085b27a8f 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -39,6 +39,7 @@ #include "pymacro.h" #include "pymath.h" #include "pymem.h" +#include "pyresource.h" #include "pytypedefs.h" #include "pybuffer.h" #include "object.h" diff --git a/Include/abstract.h b/Include/abstract.h index dd915004e7834ed..de2aab7a373a5c3 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -706,6 +706,12 @@ PyAPI_FUNC(PyObject *) PySequence_List(PyObject *o); TypeError exception with 'm' as the message text. */ PyAPI_FUNC(PyObject *) PySequence_Fast(PyObject *o, const char* m); +PyAPI_FUNC(int) PySequence_AsObjectArray( + PyObject *, + PyObject ***parray, + Py_ssize_t *psize, + PyResource *res); + /* Return the size of the sequence 'o', assuming that 'o' was returned by PySequence_Fast and is not NULL. */ #define PySequence_Fast_GET_SIZE(o) \ diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 3d53fdba6432672..db9e8fe003471ea 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -29,7 +29,8 @@ PyAPI_FUNC(PyObject *) PyByteArray_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyByteArray_Concat(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyByteArray_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *); -PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *); +PyAPI_FUNC(char*) PyByteArray_AsString(PyObject *op); +PyAPI_FUNC(char*) PyByteArray_AsStringRes(PyObject *op, PyResource *res); PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); #ifndef Py_LIMITED_API diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ee448cd02bdab38..5b7a4255b1c1643 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -39,7 +39,8 @@ PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); -PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); +PyAPI_FUNC(char*) PyBytes_AsString(PyObject *op); +PyAPI_FUNC(const char*) PyBytes_AsStringRes(PyObject *op, PyResource *res); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); diff --git a/Include/ceval.h b/Include/ceval.h index 9885bdb7febc21a..7978caacc28aab9 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -56,7 +56,8 @@ PyAPI_FUNC(int) Py_GetRecursionLimit(void); PyAPI_FUNC(int) Py_EnterRecursiveCall(const char *where); PyAPI_FUNC(void) Py_LeaveRecursiveCall(void); -PyAPI_FUNC(const char *) PyEval_GetFuncName(PyObject *); +PyAPI_FUNC(const char *) PyEval_GetFuncName(PyObject *func); +PyAPI_FUNC(const char *) PyEval_GetFuncNameRes(PyObject *func, PyResource *res); PyAPI_FUNC(const char *) PyEval_GetFuncDesc(PyObject *); PyAPI_FUNC(PyObject *) PyEval_EvalFrame(PyFrameObject *); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index e75b5e154943dcf..9b18670aa9c3fe8 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -438,7 +438,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); -/* --- Manage the default encoding ---------------------------------------- */ +/* --- Manage the UTF-8 encoding ------------------------------------------ */ /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode. @@ -452,11 +452,20 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( Use of this API is DEPRECATED since no size information can be extracted from the returned data. */ - -PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); +PyAPI_FUNC(const char*) PyUnicode_AsUTF8(PyObject *unicode); #define _PyUnicode_AsString PyUnicode_AsUTF8 + +PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSizeRes( + PyObject *unicode, + Py_ssize_t *size, + PyResource *res); + +PyAPI_FUNC(const char*) PyUnicode_AsUTF8Res( + PyObject *unicode, + PyResource *res); + /* === Characters Type APIs =============================================== */ /* These should not be used directly. Use the Py_UNICODE_IS* and diff --git a/Include/pycapsule.h b/Include/pycapsule.h index 929a9a685259fb3..c0861db6c2d122f 100644 --- a/Include/pycapsule.h +++ b/Include/pycapsule.h @@ -34,7 +34,10 @@ PyAPI_FUNC(void *) PyCapsule_GetPointer(PyObject *capsule, const char *name); PyAPI_FUNC(PyCapsule_Destructor) PyCapsule_GetDestructor(PyObject *capsule); -PyAPI_FUNC(const char *) PyCapsule_GetName(PyObject *capsule); +PyAPI_FUNC(const char*) PyCapsule_GetName(PyObject *capsule); +#ifndef Py_LIMITED_API +PyAPI_FUNC(const char*) PyCapsule_GetNameRes(PyObject *capsule, PyResource *res); +#endif PyAPI_FUNC(void *) PyCapsule_GetContext(PyObject *capsule); diff --git a/Include/pyresource.h b/Include/pyresource.h new file mode 100644 index 000000000000000..ceb36c8069178cf --- /dev/null +++ b/Include/pyresource.h @@ -0,0 +1,21 @@ +#ifndef Py_RESOURCE_H +#define Py_RESOURCE_H +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + void (*close_func) (void *data); + void *data; +} PyResource; + +PyAPI_FUNC(void) PyResource_Close(PyResource *res); + +#ifdef Py_BUILD_CORE +PyAPI_FUNC(void) _PyResource_DECREF(void *data); +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_RESOURCE_H diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e92e986b293377a..387d772001e8a0a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -564,6 +564,7 @@ def test_windows_feature_macros(self): "PyProperty_Type", "PyRangeIter_Type", "PyRange_Type", + "PyResource_Close", "PyReversed_Type", "PySeqIter_New", "PySeqIter_Type", diff --git a/Makefile.pre.in b/Makefile.pre.in index 553b2aa480c1845..bd6f199089d4234 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1657,6 +1657,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pymath.h \ $(srcdir)/Include/pymem.h \ $(srcdir)/Include/pyport.h \ + $(srcdir)/Include/pyresource.h \ $(srcdir)/Include/pystate.h \ $(srcdir)/Include/pystats.h \ $(srcdir)/Include/pystrcmp.h \ diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8ea8fde68b833a1..e9d13c6d4a4cde2 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2444,3 +2444,8 @@ added = '3.13' [function.PyMapping_GetOptionalItemString] added = '3.13' +[struct.PyResource] + added = '3.13' + struct_abi_kind = 'full-abi' +[function.PyResource_Close] + added = '3.13' diff --git a/Modules/_abc.c b/Modules/_abc.c index 8a3aa9cb88880f2..7a617253f52fab4 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -317,34 +317,46 @@ compute_abstract_methods(PyObject *self) } assert(PyList_Check(items)); for (Py_ssize_t pos = 0; pos < PyList_GET_SIZE(items); pos++) { - PyObject *it = PySequence_Fast( - PyList_GET_ITEM(items, pos), - "items() returned non-iterable"); - if (!it) { + PyObject *seq = PyList_GET_ITEM(items, pos); + PyResource res; + PyObject **items; + Py_ssize_t nitem; + if (PySequence_AsObjectArray(seq, &items, &nitem, &res) < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_SetString(PyExc_TypeError, "items() returned non-iterable"); + } goto error; } - if (PySequence_Fast_GET_SIZE(it) != 2) { + if (nitem != 2) { PyErr_SetString(PyExc_TypeError, "items() returned item which size is not 2"); - Py_DECREF(it); + PyResource_Close(&res); goto error; } - // borrowed - PyObject *key = PySequence_Fast_GET_ITEM(it, 0); - PyObject *value = PySequence_Fast_GET_ITEM(it, 1); // items or it may be cleared while accessing __abstractmethod__ // So we need to keep strong reference for key - Py_INCREF(key); + PyObject *key = Py_NewRef(items[0]); + PyObject *value = Py_NewRef(items[1]); + PyResource_Close(&res); + int is_abstract = _PyObject_IsAbstract(value); - if (is_abstract < 0 || - (is_abstract && PySet_Add(abstracts, key) < 0)) { - Py_DECREF(it); - Py_DECREF(key); - goto error; + int err; + if (is_abstract < 0) { + err = 1; + } + else if (is_abstract) { + err = PySet_Add(abstracts, key) < 0; + } + else { + err = 0; } Py_DECREF(key); - Py_DECREF(it); + Py_DECREF(value); + + if (err) { + goto error; + } } /* Stage 2: inherited abstract methods. */ diff --git a/Objects/abstract.c b/Objects/abstract.c index b4edcec60077102..c58a3fe57ed896a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2205,6 +2205,72 @@ PySequence_Fast(PyObject *v, const char *m) return v; } +static void pysequence_decref_track(void *data) +{ + PyObject *obj = _PyObject_CAST(data); + _PyObject_GC_TRACK(obj); + Py_DECREF(obj); +} + +int +PySequence_AsObjectArray(PyObject *v, PyObject ***parray, Py_ssize_t *psize, + PyResource *res) +{ + if (v == NULL) { + null_error(); + goto error; + } + + if (PyTuple_CheckExact(v)) { + if (_PyObject_GC_IS_TRACKED(v)) { + // Don't track the tuple while it's being used + _PyObject_GC_UNTRACK(v); + res->close_func = pysequence_decref_track; + } + else { + res->close_func = _PyResource_DECREF; + } + res->data = Py_NewRef(v); + *parray = _PyTuple_ITEMS(v); + *psize = PyTuple_GET_SIZE(v); + return 0; + } + + if (PyList_CheckExact(v)) { + // Don't track the list while it's being used + _PyObject_GC_UNTRACK(v); + res->close_func = pysequence_decref_track; + res->data = Py_NewRef(v); + *parray = _PyList_ITEMS(v); + *psize = PyList_GET_SIZE(v); + return 0; + } + + PyObject *it = PyObject_GetIter(v); + if (it == NULL) { + goto error; + } + + v = PySequence_List(it); + Py_DECREF(it); + if (v == NULL) { + goto error; + } + + // Don't track the list while it's being used + _PyObject_GC_UNTRACK(v); + res->close_func = pysequence_decref_track; + res->data = v; + *parray = _PyList_ITEMS(v); + *psize = PyList_GET_SIZE(v); + return 0; + +error: + *parray = NULL; + *psize = 0; + return -1; +} + /* Iterate over seq. Result depends on the operation: PY_ITERSEARCH_COUNT: -1 if error, else # of times obj appears in seq. PY_ITERSEARCH_INDEX: 0-based index of first occurrence of obj in seq; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 18a24a369a64c13..e02b9d4bcc8fe98 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -157,7 +157,7 @@ PyByteArray_Size(PyObject *self) return PyByteArray_GET_SIZE(self); } -char * +char* PyByteArray_AsString(PyObject *self) { assert(self != NULL); @@ -166,6 +166,19 @@ PyByteArray_AsString(PyObject *self) return PyByteArray_AS_STRING(self); } + +char* +PyByteArray_AsStringRes(PyObject *self, PyResource *res) +{ + assert(self != NULL); + assert(PyByteArray_Check(self)); + + res->close_func = _PyResource_DECREF; + res->data = Py_NewRef(self); + return PyByteArray_AS_STRING(self); +} + + int PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) { diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 6b9231a9fa7693f..51df31a317da383 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1220,6 +1220,19 @@ PyBytes_AsString(PyObject *op) return ((PyBytesObject *)op)->ob_sval; } +const char* +PyBytes_AsStringRes(PyObject *op, PyResource *res) +{ + if (!PyBytes_Check(op)) { + PyErr_Format(PyExc_TypeError, + "expected bytes, %.200s found", Py_TYPE(op)->tp_name); + return NULL; + } + res->close_func = _PyResource_DECREF; + res->data = Py_NewRef(op); + return ((PyBytesObject *)op)->ob_sval; +} + int PyBytes_AsStringAndSize(PyObject *obj, char **s, diff --git a/Objects/capsule.c b/Objects/capsule.c index baaddb3f1f0849c..9ed671ffc7b02da 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -106,6 +106,20 @@ PyCapsule_GetName(PyObject *o) } +const char * +PyCapsule_GetNameRes(PyObject *o, PyResource *res) +{ + PyCapsule *capsule = (PyCapsule *)o; + + if (!is_legal_capsule(capsule, "PyCapsule_GetName")) { + return NULL; + } + res->close_func = _PyResource_DECREF; + res->data = Py_NewRef(capsule); + return capsule->name; +} + + PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *o) { diff --git a/Objects/object.c b/Objects/object.c index d30e048335ab63f..2c931b860e36145 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2678,6 +2678,20 @@ int Py_IsFalse(PyObject *x) return Py_Is(x, Py_False); } +void _PyResource_DECREF(void *data) +{ + PyObject *obj = _PyObject_CAST(data); + Py_DECREF(obj); +} + +void PyResource_Close(PyResource *res) +{ + if (res->close_func == NULL) { + return; + } + res->close_func(res->data); +} + #ifdef __cplusplus } #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f543c0a65b49f65..a023551e918e33c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3781,7 +3781,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) static int unicode_fill_utf8(PyObject *unicode); -const char * +const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { if (!PyUnicode_Check(unicode)) { @@ -3800,12 +3800,44 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return PyUnicode_UTF8(unicode); } -const char * +const char* PyUnicode_AsUTF8(PyObject *unicode) { return PyUnicode_AsUTF8AndSize(unicode, NULL); } + +const char* +PyUnicode_AsUTF8AndSizeRes(PyObject *unicode, Py_ssize_t *psize, PyResource *res) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + + if (PyUnicode_UTF8(unicode) == NULL) { + if (unicode_fill_utf8(unicode) == -1) { + return NULL; + } + } + + if (psize) { + *psize = PyUnicode_UTF8_LENGTH(unicode); + } + + res->close_func = _PyResource_DECREF; + res->data = Py_NewRef(unicode); + return PyUnicode_UTF8(unicode); +} + + +const char* +PyUnicode_AsUTF8Res(PyObject *unicode, PyResource *res) +{ + return PyUnicode_AsUTF8AndSizeRes(unicode, NULL, res); +} + + /* PyUnicode_GetSize() has been deprecated since Python 3.3 because it returned length of Py_UNICODE. diff --git a/PC/python3dll.c b/PC/python3dll.c index 8f2df6950cfc052..a8d2ec1b9181f85 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -512,6 +512,7 @@ EXPORT_FUNC(PyOS_string_to_double) EXPORT_FUNC(PyOS_strtol) EXPORT_FUNC(PyOS_strtoul) EXPORT_FUNC(PyOS_vsnprintf) +EXPORT_FUNC(PyResource_Close) EXPORT_FUNC(PySeqIter_New) EXPORT_FUNC(PySequence_Check) EXPORT_FUNC(PySequence_Concat) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 760962e4c4b6a99..3f6eb72457a2f9b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -308,6 +308,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index aaebe1908e30dae..370572b2a4c4cbc 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -171,6 +171,9 @@ Include + + Include + Include diff --git a/Python/ceval.c b/Python/ceval.c index f13ba9883d98144..7097ed0b020ccb1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2298,6 +2298,7 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) const char * PyEval_GetFuncName(PyObject *func) { + // Should be kept in sync with PyEval_GetFuncNameRes() if (PyMethod_Check(func)) return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); else if (PyFunction_Check(func)) @@ -2308,6 +2309,29 @@ PyEval_GetFuncName(PyObject *func) return Py_TYPE(func)->tp_name; } +const char * +PyEval_GetFuncNameRes(PyObject *func, PyResource *res) +{ + // Should be kept in sync with PyEval_GetFuncName() + if (PyMethod_Check(func)) { + return PyEval_GetFuncNameRes(PyMethod_GET_FUNCTION(func), res); + } + else if (PyFunction_Check(func)) { + return PyUnicode_AsUTF8Res(((PyFunctionObject*)func)->func_name, res); + } + else if (PyCFunction_Check(func)) { + res->close_func = _PyResource_DECREF; + res->data = Py_NewRef(func); + return ((PyCFunctionObject*)func)->m_ml->ml_name; + } + else { + PyTypeObject *type = (PyTypeObject*)Py_NewRef(Py_TYPE(func)); + res->close_func = _PyResource_DECREF; + res->data = (PyObject *)type; // steal the reference + return type->tp_name; + } +} + const char * PyEval_GetFuncDesc(PyObject *func) {