Skip to content

gh-101277: Isolate itertools, add group and _grouper types to module state #101302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erlend-aasland merged 14 commits into from
Feb 1, 2023
Merged

gh-101277: Isolate itertools, add group and _grouper types to module state #101302

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3ab0289
Add immutable types tests
erlend-aasland Jan 24, 2023
99406f8
gh-101277: establish global state
erlend-aasland Jan 24, 2023
50410bf
gh-101277: Add groupby and _grouper types to module state
erlend-aasland Jan 24, 2023
6dc2032
Merge branch 'main' into isolate-itertools/batch-1
erlend-aasland Jan 25, 2023
2cfe214
Sync with main
erlend-aasland Jan 29, 2023
f1e877b
Store state in groupbyobject
erlend-aasland Jan 29, 2023
a0f9b42
In _grouper_create, fetch state from groupbyobject->state
erlend-aasland Jan 29, 2023
51b3c6e
Removed now unused variable
erlend-aasland Jan 29, 2023
7aea9f6
Update Modules/itertoolsmodule.c
erlend-aasland Jan 31, 2023
10cb0bc
Include pycore_moduleobject.h bco. _PyModule_GetState()
erlend-aasland Jan 31, 2023
a64e69c
Make sure the include is inserted in alphabetical order
erlend-aasland Jan 31, 2023
4783565
Update Modules/itertoolsmodule.c
erlend-aasland Jan 31, 2023
cea6d40
Sync with main
erlend-aasland Jan 31, 2023
8a3b6df
Add get-state-by-cls helper and use it for subclass check
erlend-aasland Jan 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
gh-101277: Add groupby and _grouper types to module state
  • Loading branch information
@erlend-aasland
erlend-aasland committed Jan 24, 2023
commit 50410bf33091ec9731c6ad664c3f921dafa1c5bc
10 changes: 5 additions & 5 deletions Modules/clinic/itertoolsmodule.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

223 changes: 118 additions & 105 deletions Modules/itertoolsmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,33 @@
by Raymond D. Hettinger <python@rcn.com>
*/

typedef struct {
PyTypeObject *groupby_type;
PyTypeObject *_grouper_type;
} itertools_state;

static inline itertools_state *
get_module_state(PyObject *mod)
{
void *state = PyModule_GetState(mod);
Comment thread
erlend-aasland marked this conversation as resolved.
Outdated
assert(state != NULL);
return (itertools_state *)state;
}

static struct PyModuleDef itertoolsmodule;
Comment thread
erlend-aasland marked this conversation as resolved.
static inline itertools_state *
find_state_by_type(PyTypeObject *tp)
{
PyObject *mod = PyType_GetModuleByDef(tp, &itertoolsmodule);
assert(mod != NULL);
return get_module_state(mod);
}
#define clinic_state() (find_state_by_type(type))

/*[clinic input]
module itertools
class itertools.groupby "groupbyobject *" "&groupby_type"
class itertools._grouper "_grouperobject *" "&_grouper_type"
class itertools.groupby "groupbyobject *" "clinic_state()->groupby_type"
class itertools._grouper "_grouperobject *" "clinic_state()->_grouper_type"
class itertools.teedataobject "teedataobject *" "&teedataobject_type"
class itertools._tee "teeobject *" "&tee_type"
class itertools.batched "batchedobject *" "&batched_type"
Expand All @@ -31,10 +54,8 @@ class itertools.filterfalse "filterfalseobject *" "&filterfalse_type"
class itertools.count "countobject *" "&count_type"
class itertools.pairwise "pairwiseobject *" "&pairwise_type"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=1168b274011ce21b]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=424108522584b55b]*/

static PyTypeObject groupby_type;
static PyTypeObject _grouper_type;
static PyTypeObject teedataobject_type;
static PyTypeObject tee_type;
static PyTypeObject batched_type;
Expand All @@ -51,14 +72,8 @@ static PyTypeObject filterfalse_type;
static PyTypeObject count_type;
static PyTypeObject pairwise_type;

typedef struct {
} itertools_state;

static itertools_state global_state;

#define GLOBAL_STATE (&global_state)

#include "clinic/itertoolsmodule.c.h"
#undef clinic_state

/* batched object ************************************************************/

Expand Down Expand Up @@ -421,18 +436,21 @@ itertools_groupby_impl(PyTypeObject *type, PyObject *it, PyObject *keyfunc)
static void
groupby_dealloc(groupbyobject *gbo)
{
PyTypeObject *tp = Py_TYPE(gbo);
PyObject_GC_UnTrack(gbo);
Py_XDECREF(gbo->it);
Py_XDECREF(gbo->keyfunc);
Py_XDECREF(gbo->tgtkey);
Py_XDECREF(gbo->currkey);
Py_XDECREF(gbo->currvalue);
Py_TYPE(gbo)->tp_free(gbo);
tp->tp_free(gbo);
Py_DECREF(tp);
}

static int
groupby_traverse(groupbyobject *gbo, visitproc visit, void *arg)
{
Py_VISIT(Py_TYPE(gbo));
Py_VISIT(gbo->it);
Py_VISIT(gbo->keyfunc);
Py_VISIT(gbo->tgtkey);
Expand Down Expand Up @@ -553,50 +571,26 @@ static PyMethodDef groupby_methods[] = {
{NULL, NULL} /* sentinel */
};

static PyTypeObject groupby_type = {
Comment thread
erlend-aasland marked this conversation as resolved.
PyVarObject_HEAD_INIT(NULL, 0)
"itertools.groupby", /* tp_name */
sizeof(groupbyobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)groupby_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
itertools_groupby__doc__, /* tp_doc */
(traverseproc)groupby_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(iternextfunc)groupby_next, /* tp_iternext */
groupby_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
itertools_groupby, /* tp_new */
PyObject_GC_Del, /* tp_free */
static PyType_Slot groupby_slots[] = {
{Py_tp_dealloc, groupby_dealloc},
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_doc, (void *)itertools_groupby__doc__},
{Py_tp_traverse, groupby_traverse},
{Py_tp_iter, PyObject_SelfIter},
{Py_tp_iternext, groupby_next},
{Py_tp_methods, groupby_methods},
{Py_tp_new, itertools_groupby},
{Py_tp_free, PyObject_GC_Del},
{0, NULL},
};

static PyType_Spec groupby_spec = {
.name = "itertools.groupby",
.basicsize= sizeof(groupbyobject),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_IMMUTABLETYPE),
.slots = groupby_slots,
};

/* _grouper object (internal) ************************************************/

Expand All @@ -610,25 +604,25 @@ typedef struct {
@classmethod
itertools._grouper.__new__

parent: object(subclass_of='&groupby_type')
parent: object(subclass_of='clinic_state()->groupby_type')
tgtkey: object
/
[clinic start generated code]*/

static PyObject *
itertools__grouper_impl(PyTypeObject *type, PyObject *parent,
PyObject *tgtkey)
/*[clinic end generated code: output=462efb1cdebb5914 input=dc180d7771fc8c59]*/
/*[clinic end generated code: output=462efb1cdebb5914 input=626b30a78e38cf7d]*/
{
return _grouper_create((groupbyobject*) parent, tgtkey);
}

static PyObject *
_grouper_create(groupbyobject *parent, PyObject *tgtkey)
{
_grouperobject *igo;

igo = PyObject_GC_New(_grouperobject, &_grouper_type);
PyTypeObject *tp = Py_TYPE(parent);
itertools_state *state = find_state_by_type(tp);
Comment thread
erlend-aasland marked this conversation as resolved.
Outdated
_grouperobject *igo = PyObject_GC_New(_grouperobject, state->_grouper_type);
if (igo == NULL)
return NULL;
igo->parent = Py_NewRef(parent);
Expand All @@ -642,15 +636,18 @@ _grouper_create(groupbyobject *parent, PyObject *tgtkey)
static void
_grouper_dealloc(_grouperobject *igo)
{
PyTypeObject *tp = Py_TYPE(igo);
PyObject_GC_UnTrack(igo);
Py_DECREF(igo->parent);
Py_DECREF(igo->tgtkey);
PyObject_GC_Del(igo);
Py_DECREF(tp);
}

static int
_grouper_traverse(_grouperobject *igo, visitproc visit, void *arg)
{
Py_VISIT(Py_TYPE(igo));
Py_VISIT(igo->parent);
Py_VISIT(igo->tgtkey);
return 0;
Expand Down Expand Up @@ -698,48 +695,24 @@ static PyMethodDef _grouper_methods[] = {
{NULL, NULL} /* sentinel */
};

static PyType_Slot _grouper_slots[] = {
{Py_tp_dealloc, _grouper_dealloc},
{Py_tp_getattro, PyObject_GenericGetAttr},
{Py_tp_traverse, _grouper_traverse},
{Py_tp_iter, PyObject_SelfIter},
{Py_tp_iternext, _grouper_next},
{Py_tp_methods, _grouper_methods},
{Py_tp_new, itertools__grouper},
{Py_tp_free, PyObject_GC_Del},
{0, NULL},
};

static PyTypeObject _grouper_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"itertools._grouper", /* tp_name */
sizeof(_grouperobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)_grouper_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
0, /* tp_doc */
(traverseproc)_grouper_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(iternextfunc)_grouper_next, /* tp_iternext */
_grouper_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
itertools__grouper, /* tp_new */
PyObject_GC_Del, /* tp_free */
static PyType_Spec _grouper_spec = {
.name = "itertools._grouper",
.basicsize = sizeof(_grouperobject),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_IMMUTABLETYPE),
.slots = _grouper_slots,
};


Expand Down Expand Up @@ -4986,8 +4959,47 @@ combinations_with_replacement(p, r)\n\
");

static int
itertoolsmodule_exec(PyObject *m)
itertoolsmodule_traverse(PyObject *mod, visitproc visit, void *arg)
{
itertools_state *state = get_module_state(mod);
Py_VISIT(state->groupby_type);
Py_VISIT(state->_grouper_type);
return 0;
}

static int
itertoolsmodule_clear(PyObject *mod)
{
itertools_state *state = get_module_state(mod);
Py_CLEAR(state->groupby_type);
Py_CLEAR(state->_grouper_type);
return 0;
}

static void
itertoolsmodule_free(void *mod)
{
(void)itertoolsmodule_clear((PyObject *)mod);
}

#define ADD_TYPE(module, type, spec) \
do { \
type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, NULL); \
if (type == NULL) { \
return -1; \
} \
if (PyModule_AddType(module, type) < 0) { \
return -1; \
} \
} while (0)

static int
itertoolsmodule_exec(PyObject *mod)
{
itertools_state *state = get_module_state(mod);
ADD_TYPE(mod, state->groupby_type, &groupby_spec);
ADD_TYPE(mod, state->_grouper_type, &_grouper_spec);

PyTypeObject *typelist[] = {
&accumulate_type,
&batched_type,
Expand All @@ -5007,16 +5019,14 @@ itertoolsmodule_exec(PyObject *m)
&permutations_type,
&product_type,
&repeat_type,
&groupby_type,
&_grouper_type,
&tee_type,
&teedataobject_type
};

Py_SET_TYPE(&teedataobject_type, &PyType_Type);

for (size_t i = 0; i < Py_ARRAY_LENGTH(typelist); i++) {
if (PyModule_AddType(m, typelist[i]) < 0) {
if (PyModule_AddType(mod, typelist[i]) < 0) {
return -1;
}
}
Expand All @@ -5042,6 +5052,9 @@ static struct PyModuleDef itertoolsmodule = {
.m_size = sizeof(itertools_state),
.m_methods = module_methods,
.m_slots = itertoolsmodule_slots,
.m_traverse = itertoolsmodule_traverse,
.m_clear = itertoolsmodule_clear,
.m_free = itertoolsmodule_free,
};

PyMODINIT_FUNC
Expand Down