Skip to content

[WIP, DO NOT MERGE] bpo-41188: Prepare CPython for opague PyObject structure. #21262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
WildCard65 wants to merge 6 commits into from
Closed

[WIP, DO NOT MERGE] bpo-41188: Prepare CPython for opague PyObject structure. #21262

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3d5826b
Added new type member 'tp_obj_offset'.
WildCard65 Jul 1, 2020
ff05339
Added static library "slib_pythoncore" to act as a medium between CPy…
WildCard65 Jul 1, 2020
8aa000e
Added basic implementation of "Py_TPFLAGS_OMIT_PYOBJECT_SIZE".
WildCard65 Jul 2, 2020
8774b02
Fixed GCC error
WildCard65 Jul 2, 2020
da3b98a
Add forgotten member documentation.
WildCard65 Jul 2, 2020
686d657
Trick PyObject into believing it's an opaque type.
WildCard65 Jul 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added basic implementation of "Py_TPFLAGS_OMIT_PYOBJECT_SIZE".
  • Loading branch information
@WildCard65
WildCard65 committed Jul 2, 2020
commit 8aa000ee3bdc20c4985c201474a361582065eda6
4 changes: 3 additions & 1 deletion Include/cpython/object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,10 @@ struct _typeobject {

destructor tp_finalize;
vectorcallfunc tp_vectorcall;

/* INTERNAL USE ONLY! MODIFYING THIS CAN CRASH PYTHON! */
const Py_ssize_t tp_obj_offset;
const Py_ssize_t tp_obj_offset; /* Offset from "PyObject *" pointer */
Copy link
Copy Markdown
Member

@vstinner vstinner Jul 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that this is needed. When I get a "PyObject *ob", the offset to the PyObject structure is 0: "PyObject copy = *ob;" in Python internals is valid.

Copy link
Copy Markdown
Contributor Author

@WildCard65 WildCard65 Jul 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not required if everything inherited ONLY PyObject, that will never be true.

The offset is calculated from the immediate base type of the type, which can be for example: list, tuple, dict, type, custom types.

Same thing applies to "tp_obj_size", without these members, heap corruption can be very likely.

const Py_ssize_t tp_obj_size; /* Total memory allocation size */
Copy link
Copy Markdown
Member

@vstinner vstinner Jul 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see the point of adding a new member. You can just check for Py_TPFLAGS_USES_OPAQUE_OBJECT flag in functions like PyType_GenericAlloc(), _PyObject_SIZE() and _PyObject_VAR_SIZE(). If the flag is set, add sizeof(PyObject).

Copy link
Copy Markdown
Contributor Author

@WildCard65 WildCard65 Jul 2, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See previous comment.

};

/* The *real* layout of a type object when allocated on the heap */
Expand Down
9 changes: 5 additions & 4 deletions Include/cpython/objimpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@
# error "_PyObject_VAR_SIZE requires SIZEOF_VOID_P be a power of 2"
#endif

#define _PyObject_VAR_SIZE(typeobj, nitems) \
_Py_SIZE_ROUND_UP((typeobj)->tp_basicsize + \
(nitems)*(typeobj)->tp_itemsize, \
SIZEOF_VOID_P)
static inline Py_ssize_t _PyObject_VAR_SIZE(PyTypeObject *typeobj, Py_ssize_t nitems)
{
Py_ssize_t size = (PyType_HasFeature(typeobj, Py_TPFLAGS_OMIT_PYOBJECT_SIZE) ? typeobj->tp_obj_size : typeobj->tp_basicsize);
return _Py_SIZE_ROUND_UP((size + (nitems * (typeobj->tp_itemsize))), SIZEOF_VOID_P);
}


/* This example code implements an object constructor with a custom
Expand Down
22 changes: 15 additions & 7 deletions Include/object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,21 +102,26 @@ typedef struct _typeobject PyTypeObject;
* by hand. Similarly every pointer to a variable-size Python object can,
* in addition, be cast to PyVarObject*.
*/
typedef struct _object {

typedef struct _object PyObject;
typedef struct _varobject PyVarObject;

// TODO: Send "_object" and "_varobject" to CPython internal only.
struct _object {
_PyObject_HEAD_EXTRA
Py_ssize_t ob_refcnt;
PyTypeObject *ob_type;
} PyObject;
};

struct _varobject {
PyObject ob_base;
Py_ssize_t ob_size; /* Number of items in variable part */
};

/* Cast argument to PyObject* type. */
#define _PyObject_CAST(op) ((PyObject*)(op))
#define _PyObject_CAST_CONST(op) ((const PyObject*)(op))

typedef struct {
PyObject ob_base;
Py_ssize_t ob_size; /* Number of items in variable part */
} PyVarObject;

/* Cast argument to PyVarObject* type. */
#define _PyVarObject_CAST(op) ((PyVarObject*)(op))
#define _PyVarObject_CAST_CONST(op) ((const PyVarObject*)(op))
Expand All @@ -136,6 +141,9 @@ Py_SLIB_LOCAL(void) PyVarObject_SetSize(PyVarObject *ob, const Py_ssize_t size);
Py_SLIB_LOCAL(int) PyObject_IsType(const PyObject *ob, const PyTypeObject *type);


Py_SLIB_LOCAL(void *) PyObject_GetStructure(const PyObject *ob, const PyTypeObject *type);
#define Py_GET_STRUCTURE(ob, type) PyObject_GetStructure(_PyObject_CAST_CONST(ob), (const PyTypeObject *)type)

#ifdef Py_USE_SLIB
#define Py_REFCNT(ob) PyObject_GetRefCount(_PyObject_CAST_CONST(ob))
#define Py_TYPE(ob) PyObject_GetType(_PyObject_CAST_CONST(ob))
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_sys.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1334,7 +1334,7 @@ def delx(self): del self.__x
check((1,2,3), vsize('') + 3*self.P)
# type
# static type: PyTypeObject
fmt = 'P2nPI13Pl4Pn9Pn11PIPP'
fmt = 'P2nPI13Pl4Pn9Pn11PIPPll'
s = vsize(fmt)
check(int, s)
# class
Expand Down
21 changes: 18 additions & 3 deletions Objects/typeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2086,11 +2086,13 @@ best_base(PyObject *bases)
return base;
}

#define GET_TYPE_TOTALSIZE(type) (_PyType_HasFeature(type, Py_TPFLAGS_OMIT_PYOBJECT_SIZE) ? type->tp_obj_size : type->tp_basicsize)

static int
extra_ivars(PyTypeObject *type, PyTypeObject *base)
{
size_t t_size = type->tp_basicsize;
size_t b_size = base->tp_basicsize;
size_t t_size = GET_TYPE_TOTALSIZE(type);
size_t b_size = GET_TYPE_TOTALSIZE(base);

assert(t_size >= b_size); /* Else type smaller than base! */
if (type->tp_itemsize || base->tp_itemsize) {
Expand Down Expand Up @@ -4846,7 +4848,7 @@ object___sizeof___impl(PyObject *self)
isize = Py_TYPE(self)->tp_itemsize;
if (isize > 0)
res = Py_SIZE(self) * isize;
res += Py_TYPE(self)->tp_basicsize;
res += GET_TYPE_TOTALSIZE(Py_TYPE(self));

return PyLong_FromSsize_t(res);
}
Expand Down Expand Up @@ -5364,6 +5366,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)

static int add_operators(PyTypeObject *);

#define SET_SPECIAL_TYPEVAR(var, value) *((Py_ssize_t *)&type->##var) = value

int
PyType_Ready(PyTypeObject *type)
{
Expand Down Expand Up @@ -5426,6 +5430,17 @@ PyType_Ready(PyTypeObject *type)
goto error;
}

if (type->tp_flags & Py_TPFLAGS_OMIT_PYOBJECT_SIZE) {
// tp_basicsize doesn't include the size of it's base type.
Py_ssize_t base_size = GET_TYPE_TOTALSIZE(base);

SET_SPECIAL_TYPEVAR(tp_obj_offset, (base->tp_obj_offset + base_size));
SET_SPECIAL_TYPEVAR(tp_obj_size, (base_size + type->tp_basicsize));
} else { // Type uses the size of PyObject, so the special variables are not used.
SET_SPECIAL_TYPEVAR(tp_obj_offset, 0);
SET_SPECIAL_TYPEVAR(tp_obj_size, 0);
}

/* Initialize ob_type if NULL. This means extensions that want to be
compilable separately on Windows can call PyType_Ready() instead of
initializing the ob_type field of their type objects. */
Expand Down
5 changes: 5 additions & 0 deletions PCbuild/python3dll.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@
<ItemGroup>
<ResourceCompile Include="..\PC\python_nt.rc" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="slib_pythoncore.vcxproj">
<Project>{983bba6b-a34b-40f8-927c-a57c714f1887}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
Expand Down
2 changes: 1 addition & 1 deletion PCbuild/slib_pythoncore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
<AdditionalOptions>/Zm200 %(AdditionalOptions)</AdditionalOptions>
<AdditionalIncludeDirectories>$(PySourcePath)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>_USRDLL;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>_USRDLL;Py_BUILD_SLIB;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
Expand Down
10 changes: 10 additions & 0 deletions SLib/object.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,16 @@ Py_SLIB_LOCAL(int) PyObject_IsType(const PyObject *ob, const PyTypeObject *type)
}


Py_SLIB_LOCAL(void *) PyObject_GetStructure(const PyObject *ob, const PyTypeObject *type)
{
if (!PyType_HasFeature((PyTypeObject *)type, Py_TPFLAGS_OMIT_PYOBJECT_SIZE) ||
type->tp_obj_offset == type->tp_obj_size) // This checks to see if tp_basicsize was 0 (IE: It has no internal structure)
return (void *)ob;

return (void *)(((unsigned char *)ob) + type->tp_obj_offset);
}


Py_SLIB_LOCAL(void) PyObject_IncRef(PyObject *ob)
{
#ifdef Py_REF_DEBUG
Expand Down