From 26afdc6639a8c03113e4fefae1ff7298eeb161df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Jul 2017 15:59:36 +0200 Subject: [PATCH 01/38] POC implementation of __base_subclass__ --- Python/bltinmodule.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6215a638c94b71..3ecf3149406bb9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -53,7 +53,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, { PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; PyObject *cls = NULL, *cell = NULL; + PyObject *base_types; int isclass = 0; /* initialize to prevent gcc warning */ + int i, modified_bases = 0; if (nargs < 2) { PyErr_SetString(PyExc_TypeError, @@ -76,6 +78,30 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, if (bases == NULL) return NULL; + for (i = 2; i < nargs; i++){ + PyObject *base, *new_base; + base = args[i]; + if PyType_Check(base){ + continue; + } + new_base = PyObject_GetAttrString(base, "__base_subclass__"); + if (new_base == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + Py_DECREF(bases); + return NULL; + } + } + else { + Py_INCREF(new_base); + PyTuple_SET_ITEM(bases, i - 2, new_base); + Py_DECREF(base); + modified_bases = 1; + } + } + if (kwnames == NULL) { meta = NULL; mkw = NULL; @@ -168,6 +194,10 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { + if (modified_bases){ + base_types = _PyStack_AsTupleSlice(args, nargs, 2, nargs); + PyMapping_SetItemString(ns, "__orig_bases__", base_types); + } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { From 3285fb6898739dfb844412f04d7638136e39d747 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 01:00:00 +0200 Subject: [PATCH 02/38] POC implementation of __class_getitem__ --- Objects/typeobject.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2a8118b43c5a0a..3f6cc9721c4260 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3488,6 +3488,32 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } +static PyObject * +generic_subscript(PyTypeObject *tp, PyObject *key) +{ + PyObject* stack[2] = {(PyObject *)tp, key}; + PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); + const char* msg = "'%.200s' object is not subscriptable"; + if (meth){ + return _PyObject_FastCall(meth, stack, 2); + } + else{ + if (PyErr_ExceptionMatches(PyExc_AttributeError)){ + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); + return NULL; + } + return NULL; + } + return NULL; +} + +static PyMappingMethods generic_as_mapping = { + NULL, /*mp_length*/ + (binaryfunc)generic_subscript, /*mp_subscript*/ + NULL, /*mp_ass_subscript*/ +}; + PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ @@ -3501,7 +3527,7 @@ PyTypeObject PyType_Type = { (reprfunc)type_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ + &generic_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ From 779f85dcba319311d788e8aa7e6b87c923bf9935 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 02:01:52 +0200 Subject: [PATCH 03/38] Modify __base_subclass__ API to support dynamic evaluation and base removal --- Python/bltinmodule.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3ecf3149406bb9..a7ad7edc8f2b6d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -51,7 +51,7 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *new_bases; PyObject *cls = NULL, *cell = NULL; PyObject *base_types; int isclass = 0; /* initialize to prevent gcc warning */ @@ -79,13 +79,14 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, return NULL; for (i = 2; i < nargs; i++){ - PyObject *base, *new_base; + PyObject *base, *new_base, *new_base_meth; + PyObject* stack[1]; base = args[i]; if PyType_Check(base){ continue; } - new_base = PyObject_GetAttrString(base, "__base_subclass__"); - if (new_base == NULL) { + new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); + if (new_base_meth == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); } @@ -95,6 +96,8 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } else { + stack[0] = bases; + new_base = _PyObject_FastCall(new_base_meth, stack, 1); Py_INCREF(new_base); PyTuple_SET_ITEM(bases, i - 2, new_base); Py_DECREF(base); @@ -102,6 +105,29 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } + if (modified_bases){ + int ind, tot_nones = 0; + for (i = 0; i < nargs - 2; i++){ + if (PyTuple_GET_ITEM(bases, i) == Py_None){ + tot_nones++; + } + } + ind = 0; + /* Remove all None's from base classes */ + new_bases = PyTuple_New(nargs - 2 - tot_nones); + for (i = 0; i < nargs - 2; i++){ + PyObject* a_base; + a_base = PyTuple_GET_ITEM(bases, i); + if (a_base != Py_None){ + Py_INCREF(a_base); + PyTuple_SET_ITEM(new_bases, ind, a_base); + ind++; + } + } + Py_DECREF(bases); + bases = new_bases; + } + if (kwnames == NULL) { meta = NULL; mkw = NULL; From 5d5211d9172660913f5b1507183117d837c26ba3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 23:20:43 +0200 Subject: [PATCH 04/38] Make __base_subclass__ faster and safer --- Python/bltinmodule.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a7ad7edc8f2b6d..ba78b1364dedd9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -74,6 +74,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } + bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); if (bases == NULL) return NULL; @@ -96,31 +97,42 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, } } else { + if (!PyCallable_Check(new_base_meth)) { + PyErr_Format(PyExc_TypeError, + "attribute of type '%.200s' is not callable", + Py_TYPE(new_base_meth)->tp_name); + Py_DECREF(bases); + return NULL; + } stack[0] = bases; new_base = _PyObject_FastCall(new_base_meth, stack, 1); + if (new_base == NULL){ + Py_DECREF(bases); + return NULL; + } Py_INCREF(new_base); - PyTuple_SET_ITEM(bases, i - 2, new_base); - Py_DECREF(base); + args[i] = new_base; modified_bases = 1; } } if (modified_bases){ int ind, tot_nones = 0; - for (i = 0; i < nargs - 2; i++){ - if (PyTuple_GET_ITEM(bases, i) == Py_None){ + for (i = 2; i < nargs; i++){ + if (args[i] == Py_None){ tot_nones++; } } + ind = 0; /* Remove all None's from base classes */ new_bases = PyTuple_New(nargs - 2 - tot_nones); - for (i = 0; i < nargs - 2; i++){ - PyObject* a_base; - a_base = PyTuple_GET_ITEM(bases, i); - if (a_base != Py_None){ - Py_INCREF(a_base); - PyTuple_SET_ITEM(new_bases, ind, a_base); + for (i = 2; i < nargs; i++){ + PyObject* base; + base = args[i]; + if (base != Py_None){ + Py_INCREF(base); + PyTuple_SET_ITEM(new_bases, ind, base); ind++; } } From 41fa7e9ce186efb3953ce99d0094f71f38fdc18f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jul 2017 23:59:25 +0200 Subject: [PATCH 05/38] Factor out base update in a separate helper --- Python/bltinmodule.c | 104 ++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ba78b1364dedd9..c5652165931f06 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -46,39 +46,15 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" -/* AC: cannot convert yet, waiting for *args support */ -static PyObject * -builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, - PyObject *kwnames) +static PyObject* +update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *new_bases; - PyObject *cls = NULL, *cell = NULL; - PyObject *base_types; - int isclass = 0; /* initialize to prevent gcc warning */ - int i, modified_bases = 0; - - if (nargs < 2) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: not enough arguments"); - return NULL; - } - func = args[0]; /* Better be callable */ - if (!PyFunction_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: func must be a function"); - return NULL; - } - name = args[1]; - if (!PyUnicode_Check(name)) { - PyErr_SetString(PyExc_TypeError, - "__build_class__: name is not a string"); - return NULL; - } - - bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); - if (bases == NULL) - return NULL; + int i, ind, tot_nones; + PyObject *new_bases; + assert(PyTuple_Check(bases)); + /* We have a separate cycle to calculate replacements with the idea that + most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ PyObject *base, *new_base, *new_base_meth; PyObject* stack[1]; @@ -92,7 +68,6 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyErr_Clear(); } else { - Py_DECREF(bases); return NULL; } } @@ -101,32 +76,31 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyErr_Format(PyExc_TypeError, "attribute of type '%.200s' is not callable", Py_TYPE(new_base_meth)->tp_name); - Py_DECREF(bases); return NULL; } stack[0] = bases; new_base = _PyObject_FastCall(new_base_meth, stack, 1); if (new_base == NULL){ - Py_DECREF(bases); return NULL; } Py_INCREF(new_base); args[i] = new_base; - modified_bases = 1; + *modified_bases = 1; } } - if (modified_bases){ - int ind, tot_nones = 0; + if (*modified_bases){ + /* Find out have many bases wants to be removed to pre-allocate + the tuple for new bases */ + tot_nones = 0; for (i = 2; i < nargs; i++){ if (args[i] == Py_None){ tot_nones++; } } - - ind = 0; - /* Remove all None's from base classes */ new_bases = PyTuple_New(nargs - 2 - tot_nones); + /* Remove all None's from base classes */ + ind = 0; for (i = 2; i < nargs; i++){ PyObject* base; base = args[i]; @@ -136,7 +110,54 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, ind++; } } + return new_bases; + } + else{ + return bases; + } +} + + +/* AC: cannot convert yet, waiting for *args support */ +static PyObject * +builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *new_bases, *old_bases = NULL; + PyObject *cls = NULL, *cell = NULL; + int isclass = 0; /* initialize to prevent gcc warning */ + int modified_bases = 0; + + if (nargs < 2) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: not enough arguments"); + return NULL; + } + func = args[0]; /* Better be callable */ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: func must be a function"); + return NULL; + } + name = args[1]; + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, + "__build_class__: name is not a string"); + return NULL; + } + + bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); + if (bases == NULL) + return NULL; + + new_bases = update_bases(bases, args, nargs, &modified_bases); + if (new_bases == NULL){ Py_DECREF(bases); + return NULL; + } + else{ + old_bases = bases; bases = new_bases; } @@ -233,8 +254,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { if (modified_bases){ - base_types = _PyStack_AsTupleSlice(args, nargs, 2, nargs); - PyMapping_SetItemString(ns, "__orig_bases__", base_types); + PyMapping_SetItemString(ns, "__orig_bases__", old_bases); } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); From 5aeebab8950d5efc3f2c6c36c3e1e7a78e7d146d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 00:05:12 +0200 Subject: [PATCH 06/38] Formatting --- Python/bltinmodule.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c5652165931f06..1d5da34962fadf 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -59,7 +59,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyObject *base, *new_base, *new_base_meth; PyObject* stack[1]; base = args[i]; - if PyType_Check(base){ + if (PyType_Check(base)) { continue; } new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); @@ -93,18 +93,18 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) /* Find out have many bases wants to be removed to pre-allocate the tuple for new bases */ tot_nones = 0; - for (i = 2; i < nargs; i++){ - if (args[i] == Py_None){ + for (i = 2; i < nargs; i++) { + if (args[i] == Py_None) { tot_nones++; } } new_bases = PyTuple_New(nargs - 2 - tot_nones); /* Remove all None's from base classes */ ind = 0; - for (i = 2; i < nargs; i++){ + for (i = 2; i < nargs; i++) { PyObject* base; base = args[i]; - if (base != Py_None){ + if (base != Py_None) { Py_INCREF(base); PyTuple_SET_ITEM(new_bases, ind, base); ind++; @@ -112,7 +112,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } return new_bases; } - else{ + else { return bases; } } @@ -146,17 +146,16 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } - bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); if (bases == NULL) return NULL; new_bases = update_bases(bases, args, nargs, &modified_bases); - if (new_bases == NULL){ + if (new_bases == NULL) { Py_DECREF(bases); return NULL; } - else{ + else { old_bases = bases; bases = new_bases; } From e858ea7501b6bf0ccee2a4afd3104c8b7b6da7bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 00:41:46 +0200 Subject: [PATCH 07/38] Also make __class_getitem__ safer --- Objects/typeobject.c | 13 +++++++++---- Python/bltinmodule.c | 7 +++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3f6cc9721c4260..9215a9f5a3c2af 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3492,13 +3492,18 @@ static PyObject * generic_subscript(PyTypeObject *tp, PyObject *key) { PyObject* stack[2] = {(PyObject *)tp, key}; - PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); const char* msg = "'%.200s' object is not subscriptable"; - if (meth){ + PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); + if (meth) { + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__class_getitem__ must be callable"); + return NULL; + } return _PyObject_FastCall(meth, stack, 2); } - else{ - if (PyErr_ExceptionMatches(PyExc_AttributeError)){ + else { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 1d5da34962fadf..49b2a9a368b4b9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -53,7 +53,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyObject *new_bases; assert(PyTuple_Check(bases)); - /* We have a separate cycle to calculate replacements with the idea that + /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ PyObject *base, *new_base, *new_base_meth; @@ -73,9 +73,8 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } else { if (!PyCallable_Check(new_base_meth)) { - PyErr_Format(PyExc_TypeError, - "attribute of type '%.200s' is not callable", - Py_TYPE(new_base_meth)->tp_name); + PyErr_SetString(PyExc_TypeError, + "__base_subclass__ must be callable"); return NULL; } stack[0] = bases; From 5b8d45367b2adb0015c37e7d592f00bba32d973a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 21 Jul 2017 15:47:47 +0200 Subject: [PATCH 08/38] Simplify some code --- Objects/typeobject.c | 1 - Python/bltinmodule.c | 81 +++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9215a9f5a3c2af..66fb921c3c615e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3510,7 +3510,6 @@ generic_subscript(PyTypeObject *tp, PyObject *key) } return NULL; } - return NULL; } static PyMappingMethods generic_as_mapping = { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 49b2a9a368b4b9..215a2059b518f3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -50,73 +50,62 @@ static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { int i, ind, tot_nones; - PyObject *new_bases; + PyObject *base, *new_base, *new_base_meth, *new_bases; + PyObject* stack[1] = {bases}; assert(PyTuple_Check(bases)); /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ - PyObject *base, *new_base, *new_base_meth; - PyObject* stack[1]; base = args[i]; if (PyType_Check(base)) { continue; } new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); - if (new_base_meth == NULL) { + if (!new_base_meth) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); + continue; } - else { - return NULL; - } - } - else { - if (!PyCallable_Check(new_base_meth)) { - PyErr_SetString(PyExc_TypeError, - "__base_subclass__ must be callable"); - return NULL; - } - stack[0] = bases; - new_base = _PyObject_FastCall(new_base_meth, stack, 1); - if (new_base == NULL){ - return NULL; - } - Py_INCREF(new_base); - args[i] = new_base; - *modified_bases = 1; + return NULL; } - } - - if (*modified_bases){ - /* Find out have many bases wants to be removed to pre-allocate - the tuple for new bases */ - tot_nones = 0; - for (i = 2; i < nargs; i++) { - if (args[i] == Py_None) { - tot_nones++; - } + if (!PyCallable_Check(new_base_meth)) { + PyErr_SetString(PyExc_TypeError, + "__base_subclass__ must be callable"); + return NULL; } - new_bases = PyTuple_New(nargs - 2 - tot_nones); - /* Remove all None's from base classes */ - ind = 0; - for (i = 2; i < nargs; i++) { - PyObject* base; - base = args[i]; - if (base != Py_None) { - Py_INCREF(base); - PyTuple_SET_ITEM(new_bases, ind, base); - ind++; - } + new_base = _PyObject_FastCall(new_base_meth, stack, 1); + if (!new_base){ + return NULL; } - return new_bases; + Py_INCREF(new_base); + args[i] = new_base; + *modified_bases = 1; } - else { + if (!*modified_bases){ return bases; } + /* Find out have many bases wants to be removed to pre-allocate + the tuple for new bases, then keep only non-None's in bases.*/ + tot_nones = 0; + for (i = 2; i < nargs; i++) { + if (args[i] == Py_None) { + tot_nones++; + } + } + new_bases = PyTuple_New(nargs - 2 - tot_nones); + ind = 0; + for (i = 2; i < nargs; i++) { + base = args[i]; + if (base != Py_None) { + Py_INCREF(base); + PyTuple_SET_ITEM(new_bases, ind, base); + ind++; + } + } + return new_bases; } - /* AC: cannot convert yet, waiting for *args support */ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, From b3e52f12341fb840811cddd03c97e4bfe53ac8fe Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Jul 2017 19:49:09 +0200 Subject: [PATCH 09/38] Initial work on typing2 --- Lib/typing2.py | 1347 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1347 insertions(+) create mode 100644 Lib/typing2.py diff --git a/Lib/typing2.py b/Lib/typing2.py new file mode 100644 index 00000000000000..c504bf5ec318ff --- /dev/null +++ b/Lib/typing2.py @@ -0,0 +1,1347 @@ +import abc +from abc import abstractmethod, abstractproperty +import collections +import contextlib +import functools +import re as stdlib_re # Avoid confusion with the re we export. +import sys +import types +import collections.abc +try: + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +except ImportError: + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) + + +# Please keep __all__ alphabetized within each category. +__all__ = [ + # Super-special typing primitives. + 'Any', + 'Callable', + 'ClassVar', + 'Generic', + 'Optional', + 'Tuple', + 'Type', + 'TypeVar', + 'Union', + + # ABCs (from collections.abc). + 'AbstractSet', # collections.abc.Set. + 'ByteString', + 'Container', + 'ContextManager', + 'Hashable', + 'ItemsView', + 'Iterable', + 'Iterator', + 'KeysView', + 'Mapping', + 'MappingView', + 'MutableMapping', + 'MutableSequence', + 'MutableSet', + 'Sequence', + 'Sized', + 'ValuesView', + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'Collection', + 'AsyncGenerator', + #AsyncContextManager + + # Structural checks, a.k.a. protocols. + 'Reversible', + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', + 'SupportsInt', + 'SupportsRound', + + # Concrete collection types. + 'Counter', + 'Deque', + 'Dict', + 'DefaultDict', + 'List', + 'Set', + 'FrozenSet', + 'NamedTuple', # Not really a type. + 'Generator', + + # One-off things. + 'AnyStr', + 'cast', + 'get_type_hints', + 'NewType', + 'no_type_check', + 'no_type_check_decorator', + 'overload', + 'Text', + 'TYPE_CHECKING', +] + +# The pseudo-submodules 're' and 'io' are part of the public +# namespace, but excluded from __all__ because they might stomp on +# legitimate imports of those modules. + +########################### +# Internal helper functions +########################### + +def _trim_name(nm): + whitelist = ('_TypingBase', '_FinalTypingBase') + if nm.startswith('_') and nm not in whitelist: + nm = nm[1:] + return nm + + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + If obj is a type, we return a shorter version than the default + type.__repr__, based on the module and qualified name, which is + typically enough to uniquely identify a type. For everything + else, we fall back on repr(obj). + """ + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + qname = obj.__qualname__ + if obj.__module__ == 'typing2': + qname = _trim_name(qname) + return '%s.%s' % (obj.__module__, qname) + if obj is ...: + return('...') + if isinstance(obj, types.FunctionType): + return obj.__name__ + return repr(obj) + + +def _type_vars(types): + tvars = [] + for t in types: + if isinstance(t, _TypingBase): + t._get_type_vars(tvars) + elif isinstance(t, list): + tvars.extend(_type_vars(t)) + return tuple(tvars) + + +def _eval_type(t, globalns, localns): + if isinstance(t, _TypingBase): + return t._eval_type(globalns, localns) + return t + + +def _type_check(arg, msg): + """Check that the argument is a type, and return it (internal helper). + As a special case, accept None and return type(None) instead. + The msg argument is a human-readable error message, e.g. + + "Union[arg, ...]: arg should be a type." + + We append the repr() of the actual value (truncated to 100 chars). + """ + if arg is None: + return type(None) + if isinstance(arg, str): + arg = _ForwardRef(arg) + if ( + isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or + not isinstance(arg, (type, _TypingBase)) and not callable(arg) + ): + raise TypeError(msg + " Got %.100r." % (arg,)) + # Bare Union etc. are not valid as type arguments + if ( + type(arg).__name__ in ('_Union', '_Optional') and + not getattr(arg, '__origin__', None) or + isinstance(arg, _TypingBase) and arg in (Generic, Protocol) + ): + raise TypeError("Plain %s is not valid as type argument" % arg) + return arg + + +# Special typing constructs Union, Optional, Generic, Callable and Tuple +# use three special attributes for internal bookkeeping of generic types: +# * __parameters__ is a tuple of unique free type parameters of a generic +# type, for example, Dict[T, T].__parameters__ == (T,); +# * __origin__ keeps a reference to a type that was subscripted, +# e.g., Union[T, int].__origin__ == Union; +# * __args__ is a tuple of all arguments used in subscripting, +# e.g., Dict[T, int].__args__ == (T, int). + + +def _update_args(cls, args): + """An internal helper function: calculate substitution tree + for generic cls after replacing its type parameters with + substitutions in tvars -> args (if any). + Repeat the same following __origin__'s. + + Return a list of arguments with all possible substitutions + performed. Arguments that are generic classes themselves are represented + as tuples (so that no new classes are created by this function). + For example: _subs_tree(List[Tuple[int, T]][str]) == [(Tuple, int, str)] + """ + + return GenericType(cls.__origin__, args) + + +def _remove_dups_flatten(parameters): + """An internal helper for Union creation and substitution: flatten Union's + among parameters, then remove duplicates and strict subclasses. + """ + + # Flatten out Union[Union[...], ...]. + params = [] + for p in parameters: + if isinstance(p, _Union) and p.__origin__ is Union: + params.extend(p.__args__) + elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: + params.extend(p[1:]) + else: + params.append(p) + # Weed out strict duplicates, preserving the first of each occurrence. + all_params = set(params) + if len(all_params) < len(params): + new_params = [] + for t in params: + if t in all_params: + new_params.append(t) + all_params.remove(t) + params = new_params + assert not all_params, all_params + # Weed out subclasses. + # E.g. Union[int, Employee, Manager] == Union[int, Employee]. + # If object is present it will be sole survivor among proper classes. + # Never discard type variables. + # (In particular, Union[str, AnyStr] != AnyStr.) + all_params = set(params) + for t1 in params: + if not isinstance(t1, type): + continue + if any(isinstance(t2, type) and issubclass(t1, t2) + for t2 in all_params - {t1} + if not (isinstance(t2, GenericMeta) and + t2.__origin__ is not None)): + all_params.remove(t1) + return tuple(t for t in params if t in all_params) + + +def _check_generic(cls, parameters): + # Check correct count for parameters of a generic cls (internal helper). + if not cls.__parameters__: + raise TypeError("%s is not a generic class" % repr(cls)) + alen = len(parameters) + elen = len(cls.__parameters__) + if alen != elen: + raise TypeError("Too %s parameters for %s; actual %s, expected %s" % + ("many" if alen > elen else "few", repr(cls), alen, elen)) + + +########################################################### +# Special bases for everything: general, and for singletons +########################################################### + +class _TypingBase: + """Internal indicator of special typing constructs.""" + + __slots__ = ('__weakref__') + + def __new__(cls, *args, **kwds): + """Constructor. + + This only exists to give a better error message in case + someone tries to subclass a special typing object (not a good idea). + """ + if (len(args) == 3 and + isinstance(args[0], str) and + isinstance(args[1], tuple)): + # Close enough. + raise TypeError("Cannot subclass %r" % _type_repr(cls)) + return super().__new__(cls) + + def _eval_type(self, globalns, localns): + """Override this in subclasses to interpret forward references. + + For example, List['C'] is internally stored as + List[_ForwardRef('C')], which should evaluate to List[C], + where C is an object found in globalns or localns (searching + localns first, of course). + """ + return self + + def _get_type_vars(self, tvars): + pass + + def __repr__(self): + cls = type(self) + qname = _trim_name(cls.__qualname__) + return '%s.%s' % (cls.__module__, qname) + + def __call__(self, *args, **kwds): + raise TypeError("Cannot instantiate %r" % type(self)) + + def __instancecheck__(self, obj): + raise TypeError(f"{_type_repr(self)} cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError(f"{_type_repr(self)} cannot be used with issubclass().") + + +class _FinalTypingBase(_TypingBase): + """Internal mix-in class to prevent instantiation. + + Prevents instantiation unless _root=True is given in class call. + It is used to create pseudo-singleton instances Any, Union, Optional, etc. + """ + + __slots__ = () + + def __new__(cls, *args, _root=False, **kwds): + self = super().__new__(cls, *args, **kwds) + if _root is True: + return self + raise TypeError("Cannot instantiate %r" % cls) + + def __reduce__(self): + return _trim_name(type(self).__name__) + + +######################################################################## +# Two non-singleton classes _ForwardRef (internal), and TypeVar (public) +######################################################################## + + +class _ForwardRef(_TypingBase): + """Internal wrapper to hold a forward reference.""" + + __slots__ = ('__forward_arg__', '__forward_code__', + '__forward_evaluated__', '__forward_value__') + + def __init__(self, arg): + super().__init__(arg) + if not isinstance(arg, str): + raise TypeError('Forward reference must be a string -- got %r' % (arg,)) + try: + code = compile(arg, '', 'eval') + except SyntaxError: + raise SyntaxError('Forward reference must be an expression -- got %r' % + (arg,)) + self.__forward_arg__ = arg + self.__forward_code__ = code + self.__forward_evaluated__ = False + self.__forward_value__ = None + + def _eval_type(self, globalns, localns): + if not self.__forward_evaluated__ or localns is not globalns: + if globalns is None and localns is None: + globalns = localns = {} + elif globalns is None: + globalns = localns + elif localns is None: + localns = globalns + self.__forward_value__ = _type_check( + eval(self.__forward_code__, globalns, localns), + "Forward references must evaluate to types.") + self.__forward_evaluated__ = True + return self.__forward_value__ + + def __eq__(self, other): + if not isinstance(other, _ForwardRef): + return NotImplemented + return (self.__forward_arg__ == other.__forward_arg__ and + self.__forward_value__ == other.__forward_value__) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_value__)) + + def __repr__(self): + return '%r*' % (self.__forward_arg__,) + + +class TypeVar(_TypingBase): + """Type variable. + + Usage:: + + T = TypeVar('T') # Can be anything + A = TypeVar('A', str, bytes) # Must be str or bytes + + Type variables exist primarily for the benefit of static type + checkers. They serve as the parameters for generic types as well + as for generic function definitions. See class Generic for more + information on generic types. Generic functions work as follows: + + def repeat(x: T, n: int) -> List[T]: + '''Return a list containing n references to x.''' + return [x]*n + + def longest(x: A, y: A) -> A: + '''Return the longest of two strings.''' + return x if len(x) >= len(y) else y + + The latter example's signature is essentially the overloading + of (str, str) -> str and (bytes, bytes) -> bytes. Also note + that if the arguments are instances of some subclass of str, + the return type is still plain str. + + At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. + + Type variables defined with covariant=True or contravariant=True + can be used do declare covariant or contravariant generic types. + See PEP 484 for more details. By default generic types are invariant + in all type variables. + + Type variables can be introspected. e.g.: + + T.__name__ == 'T' + T.__constraints__ == () + T.__covariant__ == False + T.__contravariant__ = False + A.__constraints__ == (str, bytes) + """ + + __slots__ = ('__name__', '__bound__', '__constraints__', + '__covariant__', '__contravariant__') + + def __init__(self, name, *constraints, bound=None, + covariant=False, contravariant=False): + self.__name__ = name + if covariant and contravariant: + raise ValueError("Bivariant types are not supported.") + self.__covariant__ = bool(covariant) + self.__contravariant__ = bool(contravariant) + if constraints and bound is not None: + raise TypeError("Constraints cannot be combined with bound=...") + if constraints and len(constraints) == 1: + raise TypeError("A single constraint is not allowed") + msg = "TypeVar(name, constraint, ...): constraints must be types." + self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) + if bound: + self.__bound__ = _type_check(bound, "Bound must be a type.") + else: + self.__bound__ = None + + def _get_type_vars(self, tvars): + if self not in tvars: + tvars.append(self) + + def __repr__(self): + if self.__covariant__: + prefix = '+' + elif self.__contravariant__: + prefix = '-' + else: + prefix = '~' + return prefix + self.__name__ + +# Some unconstrained type variables. These are used by the container types. +# (These are not for export.) +T = TypeVar('T') # Any type. +KT = TypeVar('KT') # Key type. +VT = TypeVar('VT') # Value type. +T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. +V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. +VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. +T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. + +# A useful type variable with constraints. This represents string types. +# (This one *is* for export!) +AnyStr = TypeVar('AnyStr', bytes, str) + +#################################################### +# Two non-subscriptable singletons: Any and NoReturn +#################################################### + +class _Any(_FinalTypingBase): + """Special type indicating an unconstrained type. + + - Any is compatible with every type. + - Any assumed to have all methods. + - All values assumed to be instances of Any. + + Note that all the above statements are true from the point of view of + static type checkers. At runtime, Any should not be used with instance + or class checks. + """ + + __slots__ = () + + +Any = _Any(_root=True) + + +class _NoReturn(_FinalTypingBase): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + +NoReturn = _NoReturn(_root=True) + +################################################################## +# The main class: represents subscriptions of all singletons below +################################################################## + +class GenericType(_TypingBase): + + __slots__ = ('__origin__', '__args__', '__parameters__', + '_name', '_call') + + def __init__(self, origin, args, name=None, call=True): + self.__origin__ = origin + self.__args__ = args + self.__parameters__ = () + self._call = call + if name: + self._name = name + elif isinstance(origin, type): + self._name = origin.__qualname__ + else: + self._name = repr(origin) + def __call__(self, *args, **kwargs): + if not self._call: + raise TypeError(f'{self} is not callable') + return self.__origin__(*args, **kwargs) + def __base_subclass__(self, bases): + return self.__origin__ + def __getitem__(self, args): + return _update_args(self, args) + def _eval_type(self): + return self + def __repr__(self): + args = ', '.join([_type_repr(a) for a in self.__args__]) + return self._name + f'[{args}]' + + +###################################################################### +# Subscriptable non-subclassable singletons: ClassVar, Union, Optional +###################################################################### + +class _ClassVar(_FinalTypingBase): + """Special type construct to mark class variables. + + An annotation wrapped in ClassVar indicates that a given + attribute is intended to be used as a class variable and + should not be set on instances of that class. Usage:: + + class Starship: + stats: ClassVar[Dict[str, int]] = {} # class variable + damage: int = 10 # instance variable + + ClassVar accepts only types and cannot be further subscribed. + + Note that ClassVar is not a class itself, and should not + be used with isinstance() or issubclass(). + """ + + __slots__ = () + + def __getitem__(self, item): + return GenericType(self, [item]) + + +ClassVar = _ClassVar(_root=True) + + +class _Union(_FinalTypingBase): + """Union type; Union[X, Y] means either X or Y. + + To define a union, use e.g. Union[int, str]. Details: + + - The arguments must be types and there must be at least one. + + - None as an argument is a special case and is replaced by + type(None). + + - Unions of unions are flattened, e.g.:: + + Union[Union[int, str], float] == Union[int, str, float] + + - Unions of a single argument vanish, e.g.:: + + Union[int] == int # The constructor actually returns int + + - Redundant arguments are skipped, e.g.:: + + Union[int, str, int] == Union[int, str] + + - When comparing unions, the argument order is ignored, e.g.:: + + Union[int, str] == Union[str, int] + + - When two arguments have a subclass relationship, the least + derived argument is kept, e.g.:: + + class Employee: pass + class Manager(Employee): pass + Union[int, Employee, Manager] == Union[int, Employee] + Union[Manager, int, Employee] == Union[int, Employee] + Union[Employee, Manager] == Employee + + - Similar for object:: + + Union[int, object] == object + + - You cannot subclass or instantiate a union. + + - You can use Optional[X] as a shorthand for Union[X, None]. + """ + + __slots__ = () + + def __getitem__(self, parameters): + if parameters == (): + raise TypeError("Cannot take a Union of no types.") + if not isinstance(parameters, tuple): + parameters = (parameters,) + msg = "Union[arg, ...]: each arg must be a type." + parameters = tuple(_type_check(p, msg) for p in parameters) + return GenericType(self, parameters) + +Union = _Union(_root=True) + + +class _Optional(_FinalTypingBase): + """Optional type. + + Optional[X] is equivalent to Union[X, None]. + """ + + __slots__ = () + + def __getitem__(self, arg): + arg = _type_check(arg, "Optional[t] requires a single type.") + return Union[arg, type(None)] + + +Optional = _Optional(_root=True) + + +############################################################# +# Subclassable singletons: Generic, Callable, Tuple, Protocol +############################################################# + +class Generic(_FinalTypingBase): + """Abstract base class for generic types. + + A generic type is typically declared by inheriting from + this class parameterized with one or more type variables. + For example, a generic mapping type might be defined as:: + + class Mapping(Generic[KT, VT]): + def __getitem__(self, key: KT) -> VT: + ... + # Etc. + + This class can then be used as follows:: + + def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: + try: + return mapping[key] + except KeyError: + return default + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(self, args) + + +class Tuple(_FinalTypingBase): + """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + + Example: Tuple[T1, T2] is a tuple of two elements corresponding + to type variables T1 and T2. Tuple[int, float, str] is a tuple + of an int, a float and a string. + + To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(tuple, args, name='Tuple', call=False) + + +class Callable(_FinalTypingBase): + """Callable type; Callable[[int], str] is a function of (int) -> str. + + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or ellipsis; the return type must be a single type. + + There is no syntax to indicate optional or keyword arguments, + such function types are rarely used as callback types. + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(collections.abc.Callable, args) + + +class Protocol(_FinalTypingBase): + """Internal base class for protocol classes. + + This implements a simple-minded structural issubclass check + (similar but more general than the one-offs in collections.abc + such as Hashable). + """ + + __slots__ = () + + def __class_getitem__(self, args): + return GenericType(self, args) + + +######################################### +# Fancy classes: NamedTuple and TypedDict +######################################### + +def _make_nmtuple(name, types): + msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" + types = [(n, _type_check(t, msg)) for n, t in types] + nm_tpl = collections.namedtuple(name, [n for n, t in types]) + # Prior to PEP 526, only _field_types attribute was assigned. + # Now, both __annotations__ and _field_types are used to maintain compatibility. + nm_tpl.__annotations__ = nm_tpl._field_types = collections.OrderedDict(types) + try: + nm_tpl.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + return nm_tpl + + +_PY36 = sys.version_info[:2] >= (3, 6) + +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict', '_source') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + + +class NamedTupleMeta(type): + + def __new__(cls, typename, bases, ns): + if ns.get('_root', False): + return super().__new__(cls, typename, bases, ns) + if not _PY36: + raise TypeError("Class syntax for NamedTuple is only supported" + " in Python 3.6+") + types = ns.get('__annotations__', {}) + nm_tpl = _make_nmtuple(typename, types.items()) + defaults = [] + defaults_dict = {} + for field_name in types: + if field_name in ns: + default_value = ns[field_name] + defaults.append(default_value) + defaults_dict[field_name] = default_value + elif defaults: + raise TypeError("Non-default namedtuple field {field_name} cannot " + "follow default field(s) {default_names}" + .format(field_name=field_name, + default_names=', '.join(defaults_dict.keys()))) + nm_tpl.__new__.__defaults__ = tuple(defaults) + nm_tpl._field_defaults = defaults_dict + # update from user namespace without overriding special namedtuple attributes + for key in ns: + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: + setattr(nm_tpl, key, ns[key]) + return nm_tpl + + +class NamedTuple(metaclass=NamedTupleMeta): + """Typed version of namedtuple. + + Usage in Python versions >= 3.6:: + + class Employee(NamedTuple): + name: str + id: int + + This is equivalent to:: + + Employee = collections.namedtuple('Employee', ['name', 'id']) + + The resulting class has extra __annotations__ and _field_types + attributes, giving an ordered dict mapping field names to types. + __annotations__ should be preferred, while _field_types + is kept to maintain pre PEP 526 compatibility. (The field names + are in the _fields attribute, which is part of the namedtuple + API.) Alternative equivalent keyword syntax is also accepted:: + + Employee = NamedTuple('Employee', name=str, id=int) + + In Python versions <= 3.5 use:: + + Employee = NamedTuple('Employee', [('name', str), ('id', int)]) + """ + _root = True + + def __new__(self, typename, fields=None, **kwargs): + if kwargs and not _PY36: + raise TypeError("Keyword syntax for NamedTuple is only supported" + " in Python 3.6+") + if fields is None: + fields = kwargs.items() + elif kwargs: + raise TypeError("Either list of fields or keywords" + " can be provided to NamedTuple, not both") + return _make_nmtuple(typename, fields) + + +################################# +# Public functions (with helpers) +################################# + +def cast(typ, val): + """Cast a value to a type. + + This returns the value unchanged. To the type checker this + signals that the return value has the designated type, but at + runtime we intentionally don't check anything (we want this + to be as fast as possible). + """ + return val + + +def _get_defaults(func): + """Internal helper to extract the default arguments, by name.""" + try: + code = func.__code__ + except AttributeError: + # Some built-in functions don't have __code__, __defaults__, etc. + return {} + pos_count = code.co_argcount + arg_names = code.co_varnames + arg_names = arg_names[:pos_count] + defaults = func.__defaults__ or () + kwdefaults = func.__kwdefaults__ + res = dict(kwdefaults) if kwdefaults else {} + pos_offset = pos_count - len(defaults) + for name, value in zip(arg_names[pos_offset:], defaults): + assert name not in res + res[name] = value + return res + + +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) + + +def get_type_hints(obj, globalns=None, localns=None): + """Return type hints for an object. + + This is often the same as obj.__annotations__, but it handles + forward references encoded as string literals, and if necessary + adds Optional[t] if a default value equal to None is set. + + The argument may be a module, class, method, or function. The annotations + are returned as a dictionary. For classes, annotations include also + inherited members. + + TypeError is raised if the argument is not of a type that can contain + annotations, and an empty dictionary is returned if no annotations are + present. + + BEWARE -- the behavior of globalns and localns is counterintuitive + (unless you are familiar with how eval() and exec() work). The + search order is locals first, then globals. + + - If no dict arguments are passed, an attempt is made to use the + globals from obj, and these are also used as the locals. If the + object does not appear to have globals, an exception is raised. + + - If one dict argument is passed, it is used for both globals and + locals. + + - If two dict arguments are passed, they specify globals and + locals, respectively. + """ + + if getattr(obj, '__no_type_check__', None): + return {} + if globalns is None: + globalns = getattr(obj, '__globals__', {}) + if localns is None: + localns = globalns + elif localns is None: + localns = globalns + # Classes require a special treatment. + if isinstance(obj, type): + hints = {} + for base in reversed(obj.__mro__): + ann = base.__dict__.get('__annotations__', {}) + for name, value in ann.items(): + if value is None: + value = type(None) + if isinstance(value, str): + value = _ForwardRef(value) + value = _eval_type(value, globalns, localns) + hints[name] = value + return hints + hints = getattr(obj, '__annotations__', None) + if hints is None: + # Return empty annotations for something that _could_ have them. + if isinstance(obj, _allowed_types): + return {} + else: + raise TypeError('{!r} is not a module, class, method, ' + 'or function.'.format(obj)) + defaults = _get_defaults(obj) + hints = dict(hints) + for name, value in hints.items(): + if value is None: + value = type(None) + if isinstance(value, str): + value = _ForwardRef(value) + value = _eval_type(value, globalns, localns) + if name in defaults and defaults[name] is None: + value = Optional[value] + hints[name] = value + return hints + + +def no_type_check(arg): + """Decorator to indicate that annotations are not type hints. + + The argument must be a class or function; if it is a class, it + applies recursively to all methods and classes defined in that class + (but not to methods defined in its superclasses or subclasses). + + This mutates the function(s) or class(es) in place. + """ + if isinstance(arg, type): + arg_attrs = arg.__dict__.copy() + for attr, val in arg.__dict__.items(): + if val in arg.__bases__ + (arg,): + arg_attrs.pop(attr) + for obj in arg_attrs.values(): + if isinstance(obj, types.FunctionType): + obj.__no_type_check__ = True + if isinstance(obj, type): + no_type_check(obj) + try: + arg.__no_type_check__ = True + except TypeError: # built-in classes + pass + return arg + + +def no_type_check_decorator(decorator): + """Decorator to give another decorator the @no_type_check effect. + + This wraps the decorator with something that wraps the decorated + function in @no_type_check. + """ + + @functools.wraps(decorator) + def wrapped_decorator(*args, **kwds): + func = decorator(*args, **kwds) + func = no_type_check(func) + return func + + return wrapped_decorator + + +def _overload_dummy(*args, **kwds): + """Helper for @overload to raise when called.""" + raise NotImplementedError( + "You should not call an overloaded function. " + "A series of @overload-decorated functions " + "outside a stub module should always be followed " + "by an implementation that is not @overload-ed.") + + +def overload(func): + """Decorator for overloaded functions/methods. + + In a stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload. For example: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + + In a non-stub file (i.e. a regular .py file), do the same but + follow it with an implementation. The implementation should *not* + be decorated with @overload. For example: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + def utf8(value): + # implementation goes here + """ + return _overload_dummy + + +def NewType(name, tp): + """NewType creates simple unique types with almost zero + runtime overhead. NewType(name, tp) is considered a subtype of tp + by static type checkers. At runtime, NewType(name, tp) returns + a dummy function that simply returns its argument. Usage:: + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + UserId('user') # Fails type check + + name_by_id(42) # Fails type check + name_by_id(UserId(42)) # OK + + num = UserId(5) + 1 # type: int + """ + + def new_type(x): + return x + + new_type.__name__ = name + new_type.__supertype__ = tp + return new_type + + +###################################################################### +# Various generics mimicking those in collections.abc and +# additional protocols. A few are simply re-exported for completeness. +###################################################################### + +Hashable = collections.abc.Hashable # Not generic. + +Awaitable = GenericType(collections.abc.Awaitable, [T_co]) +Coroutine = GenericType(collections.abc.Coroutine, [T_co, T_contra, V_co]) +AsyncIterable = GenericType(collections.abc.AsyncIterable, [T_co]) +AsyncIterator = GenericType(collections.abc.AsyncIterator, [T_co]) + +Iterable = GenericType(collections.abc.Iterable, [T_co]) +Iterator = GenericType(collections.abc.Iterator, [T_co]) + +class SupportsInt(Protocol): + __slots__ = () + + @abstractmethod + def __int__(self) -> int: + pass + + +class SupportsFloat(Protocol): + __slots__ = () + + @abstractmethod + def __float__(self) -> float: + pass + + +class SupportsComplex(Protocol): + __slots__ = () + + @abstractmethod + def __complex__(self) -> complex: + pass + + +class SupportsBytes(Protocol): + __slots__ = () + + @abstractmethod + def __bytes__(self) -> bytes: + pass + + +class SupportsAbs(Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __abs__(self) -> T_co: + pass + + +class SupportsRound(Protocol[T_co]): + __slots__ = () + + @abstractmethod + def __round__(self, ndigits: int = 0) -> T_co: + pass + + +Reversible = GenericType(collections.abc.Reversible, [T_co]) +Sized = collections.abc.Sized # Not generic. +Container = GenericType(collections.abc.Container, [T_co]) +Collection = GenericType(collections.abc.Collection, [T_co]) + +# Callable was defined earlier. + +AbstractSet = GenericType(collections.abc.Set, [T_co]) +MutableSet = GenericType(collections.abc.MutableSet, [T]) +Set = GenericType(set, [T], name='Set') +FrozenSet = GenericType(frozenset, [T_co], name='FrozenSet') + +# NOTE: It is only covariant in the value type. +Mapping = GenericType(collections.abc.Mapping, [KT, VT_co]) +MutableMapping = GenericType(collections.abc.MutableMapping, [KT, VT]) +Dict = GenericType(dict, [KT, VT], call=False, name='Dict') +DefaultDict = GenericType(collections.defaultdict, [KT, VT], call=False, name='DefaultDict') +Counter = GenericType(collections.Counter, [T], call=False) +ChainMap = GenericType(collections.ChainMap, [KT, VT], call=False) + +Sequence = GenericType(collections.abc.Sequence, [T_co]) +MutableSequence = GenericType(collections.abc.MutableSequence, [T]) +ByteString = collections.abc.ByteString # Not generic. +List = GenericType(list, [T], name='List') +Deque = GenericType(collections.deque, [T], name='Deque') + +MappingView = GenericType(collections.abc.MappingView, [T_co]) +KeysView = GenericType(collections.abc.KeysView, [KT]) +ItemsView = GenericType(collections.abc.ItemsView, [KT, VT_co]) +ValuesView = GenericType(collections.abc.ValuesView, [VT_co]) + +ContextManager = GenericType(contextlib.AbstractContextManager, [T_co]) +if hasattr(contextlib, 'AbstractAsyncContextManager'): + AsyncContextManager = GenericType(contextlib.AbstractAsyncContextManager, [T_co]) + __all__.append('AsyncContextManager') + +Generator = GenericType(collections.abc.Generator, [T_co, T_contra, V_co]) +AsyncGenerator = GenericType(collections.abc.AsyncGenerator, [T_co, T_contra]) + +# Internal type variable used for Type[]. +CT_co = TypeVar('CT_co', covariant=True, bound=type) +# This is not a real generic class. Don't use outside annotations. +Type = GenericType(type, [CT_co], name='Type') +"""A special construct usable to annotate class objects. + +For example, suppose we have the following classes:: + + class User: ... # Abstract base for User classes + class BasicUser(User): ... + class ProUser(User): ... + class TeamUser(User): ... + +And a function that takes a class argument that's a subclass of +User and returns an instance of the corresponding class:: + + U = TypeVar('U', bound=User) + def new_user(user_class: Type[U]) -> U: + user = user_class() + # (Here we could write the user object to a database) + return user + + joe = new_user(BasicUser) + +At this point the type checker knows that joe has type BasicUser. +""" + +################## +# Public constants +################## + +# Python-version-specific alias (Python 2: unicode; Python 3: str) +Text = str + + +# Constant that's True when type checking, but False here. +TYPE_CHECKING = False + +######################################### +# Special io types and re generic aliases +######################################### + + +class IO(Generic[AnyStr]): + """Generic base class for TextIO and BinaryIO. + + This is an abstract, generic version of the return of open(). + + NOTE: This does not distinguish between the different possible + classes (text vs. binary, read vs. write vs. read/write, + append-only, unbuffered). The TextIO and BinaryIO subclasses + below capture the distinctions between text vs. binary, which is + pervasive in the interface; however we currently do not offer a + way to track the other distinctions in the type system. + """ + + __slots__ = () + + @abstractproperty + def mode(self) -> str: + pass + + @abstractproperty + def name(self) -> str: + pass + + @abstractmethod + def close(self) -> None: + pass + + @abstractmethod + def closed(self) -> bool: + pass + + @abstractmethod + def fileno(self) -> int: + pass + + @abstractmethod + def flush(self) -> None: + pass + + @abstractmethod + def isatty(self) -> bool: + pass + + @abstractmethod + def read(self, n: int = -1) -> AnyStr: + pass + + @abstractmethod + def readable(self) -> bool: + pass + + @abstractmethod + def readline(self, limit: int = -1) -> AnyStr: + pass + + @abstractmethod + def readlines(self, hint: int = -1) -> List[AnyStr]: + pass + + @abstractmethod + def seek(self, offset: int, whence: int = 0) -> int: + pass + + @abstractmethod + def seekable(self) -> bool: + pass + + @abstractmethod + def tell(self) -> int: + pass + + @abstractmethod + def truncate(self, size: int = None) -> int: + pass + + @abstractmethod + def writable(self) -> bool: + pass + + @abstractmethod + def write(self, s: AnyStr) -> int: + pass + + @abstractmethod + def writelines(self, lines: List[AnyStr]) -> None: + pass + + @abstractmethod + def __enter__(self) -> 'IO[AnyStr]': + pass + + @abstractmethod + def __exit__(self, type, value, traceback) -> None: + pass + + +class BinaryIO(IO[bytes]): + """Typed version of the return of open() in binary mode.""" + + __slots__ = () + + @abstractmethod + def write(self, s: Union[bytes, bytearray]) -> int: + pass + + @abstractmethod + def __enter__(self) -> 'BinaryIO': + pass + + +class TextIO(IO[str]): + """Typed version of the return of open() in text mode.""" + + __slots__ = () + + @abstractproperty + def buffer(self) -> BinaryIO: + pass + + @abstractproperty + def encoding(self) -> str: + pass + + @abstractproperty + def errors(self) -> Optional[str]: + pass + + @abstractproperty + def line_buffering(self) -> bool: + pass + + @abstractproperty + def newlines(self) -> Any: + pass + + @abstractmethod + def __enter__(self) -> 'TextIO': + pass + + +class io: + """Wrapper namespace for IO generic classes.""" + + __all__ = ['IO', 'TextIO', 'BinaryIO'] + IO = IO + TextIO = TextIO + BinaryIO = BinaryIO + + +io.__name__ = __name__ + '.io' +sys.modules[io.__name__] = io + + +Pattern = GenericType(type(stdlib_re.compile('')), [AnyStr], name='Pattern') +Match = GenericType(type(stdlib_re.match('', '')), [AnyStr], name='Match') + + +class re: + """Wrapper namespace for re type aliases.""" + + __all__ = ['Pattern', 'Match'] + Pattern = Pattern + Match = Match + + +re.__name__ = __name__ + '.re' +sys.modules[re.__name__] = re From 6e7b575273aed0a4c5f181fde1825d82d9924b8d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 3 Sep 2017 20:57:35 +0200 Subject: [PATCH 10/38] Remove test implementation for typing --- Lib/typing2.py | 1347 ------------------------------------------------ 1 file changed, 1347 deletions(-) delete mode 100644 Lib/typing2.py diff --git a/Lib/typing2.py b/Lib/typing2.py deleted file mode 100644 index c504bf5ec318ff..00000000000000 --- a/Lib/typing2.py +++ /dev/null @@ -1,1347 +0,0 @@ -import abc -from abc import abstractmethod, abstractproperty -import collections -import contextlib -import functools -import re as stdlib_re # Avoid confusion with the re we export. -import sys -import types -import collections.abc -try: - from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType -except ImportError: - WrapperDescriptorType = type(object.__init__) - MethodWrapperType = type(object().__str__) - MethodDescriptorType = type(str.join) - - -# Please keep __all__ alphabetized within each category. -__all__ = [ - # Super-special typing primitives. - 'Any', - 'Callable', - 'ClassVar', - 'Generic', - 'Optional', - 'Tuple', - 'Type', - 'TypeVar', - 'Union', - - # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. - 'ByteString', - 'Container', - 'ContextManager', - 'Hashable', - 'ItemsView', - 'Iterable', - 'Iterator', - 'KeysView', - 'Mapping', - 'MappingView', - 'MutableMapping', - 'MutableSequence', - 'MutableSet', - 'Sequence', - 'Sized', - 'ValuesView', - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'Collection', - 'AsyncGenerator', - #AsyncContextManager - - # Structural checks, a.k.a. protocols. - 'Reversible', - 'SupportsAbs', - 'SupportsBytes', - 'SupportsComplex', - 'SupportsFloat', - 'SupportsInt', - 'SupportsRound', - - # Concrete collection types. - 'Counter', - 'Deque', - 'Dict', - 'DefaultDict', - 'List', - 'Set', - 'FrozenSet', - 'NamedTuple', # Not really a type. - 'Generator', - - # One-off things. - 'AnyStr', - 'cast', - 'get_type_hints', - 'NewType', - 'no_type_check', - 'no_type_check_decorator', - 'overload', - 'Text', - 'TYPE_CHECKING', -] - -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. - -########################### -# Internal helper functions -########################### - -def _trim_name(nm): - whitelist = ('_TypingBase', '_FinalTypingBase') - if nm.startswith('_') and nm not in whitelist: - nm = nm[1:] - return nm - - -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - If obj is a type, we return a shorter version than the default - type.__repr__, based on the module and qualified name, which is - typically enough to uniquely identify a type. For everything - else, we fall back on repr(obj). - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - qname = obj.__qualname__ - if obj.__module__ == 'typing2': - qname = _trim_name(qname) - return '%s.%s' % (obj.__module__, qname) - if obj is ...: - return('...') - if isinstance(obj, types.FunctionType): - return obj.__name__ - return repr(obj) - - -def _type_vars(types): - tvars = [] - for t in types: - if isinstance(t, _TypingBase): - t._get_type_vars(tvars) - elif isinstance(t, list): - tvars.extend(_type_vars(t)) - return tuple(tvars) - - -def _eval_type(t, globalns, localns): - if isinstance(t, _TypingBase): - return t._eval_type(globalns, localns) - return t - - -def _type_check(arg, msg): - """Check that the argument is a type, and return it (internal helper). - As a special case, accept None and return type(None) instead. - The msg argument is a human-readable error message, e.g. - - "Union[arg, ...]: arg should be a type." - - We append the repr() of the actual value (truncated to 100 chars). - """ - if arg is None: - return type(None) - if isinstance(arg, str): - arg = _ForwardRef(arg) - if ( - isinstance(arg, _TypingBase) and type(arg).__name__ == '_ClassVar' or - not isinstance(arg, (type, _TypingBase)) and not callable(arg) - ): - raise TypeError(msg + " Got %.100r." % (arg,)) - # Bare Union etc. are not valid as type arguments - if ( - type(arg).__name__ in ('_Union', '_Optional') and - not getattr(arg, '__origin__', None) or - isinstance(arg, _TypingBase) and arg in (Generic, Protocol) - ): - raise TypeError("Plain %s is not valid as type argument" % arg) - return arg - - -# Special typing constructs Union, Optional, Generic, Callable and Tuple -# use three special attributes for internal bookkeeping of generic types: -# * __parameters__ is a tuple of unique free type parameters of a generic -# type, for example, Dict[T, T].__parameters__ == (T,); -# * __origin__ keeps a reference to a type that was subscripted, -# e.g., Union[T, int].__origin__ == Union; -# * __args__ is a tuple of all arguments used in subscripting, -# e.g., Dict[T, int].__args__ == (T, int). - - -def _update_args(cls, args): - """An internal helper function: calculate substitution tree - for generic cls after replacing its type parameters with - substitutions in tvars -> args (if any). - Repeat the same following __origin__'s. - - Return a list of arguments with all possible substitutions - performed. Arguments that are generic classes themselves are represented - as tuples (so that no new classes are created by this function). - For example: _subs_tree(List[Tuple[int, T]][str]) == [(Tuple, int, str)] - """ - - return GenericType(cls.__origin__, args) - - -def _remove_dups_flatten(parameters): - """An internal helper for Union creation and substitution: flatten Union's - among parameters, then remove duplicates and strict subclasses. - """ - - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, _Union) and p.__origin__ is Union: - params.extend(p.__args__) - elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: - params.extend(p[1:]) - else: - params.append(p) - # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - # Weed out subclasses. - # E.g. Union[int, Employee, Manager] == Union[int, Employee]. - # If object is present it will be sole survivor among proper classes. - # Never discard type variables. - # (In particular, Union[str, AnyStr] != AnyStr.) - all_params = set(params) - for t1 in params: - if not isinstance(t1, type): - continue - if any(isinstance(t2, type) and issubclass(t1, t2) - for t2 in all_params - {t1} - if not (isinstance(t2, GenericMeta) and - t2.__origin__ is not None)): - all_params.remove(t1) - return tuple(t for t in params if t in all_params) - - -def _check_generic(cls, parameters): - # Check correct count for parameters of a generic cls (internal helper). - if not cls.__parameters__: - raise TypeError("%s is not a generic class" % repr(cls)) - alen = len(parameters) - elen = len(cls.__parameters__) - if alen != elen: - raise TypeError("Too %s parameters for %s; actual %s, expected %s" % - ("many" if alen > elen else "few", repr(cls), alen, elen)) - - -########################################################### -# Special bases for everything: general, and for singletons -########################################################### - -class _TypingBase: - """Internal indicator of special typing constructs.""" - - __slots__ = ('__weakref__') - - def __new__(cls, *args, **kwds): - """Constructor. - - This only exists to give a better error message in case - someone tries to subclass a special typing object (not a good idea). - """ - if (len(args) == 3 and - isinstance(args[0], str) and - isinstance(args[1], tuple)): - # Close enough. - raise TypeError("Cannot subclass %r" % _type_repr(cls)) - return super().__new__(cls) - - def _eval_type(self, globalns, localns): - """Override this in subclasses to interpret forward references. - - For example, List['C'] is internally stored as - List[_ForwardRef('C')], which should evaluate to List[C], - where C is an object found in globalns or localns (searching - localns first, of course). - """ - return self - - def _get_type_vars(self, tvars): - pass - - def __repr__(self): - cls = type(self) - qname = _trim_name(cls.__qualname__) - return '%s.%s' % (cls.__module__, qname) - - def __call__(self, *args, **kwds): - raise TypeError("Cannot instantiate %r" % type(self)) - - def __instancecheck__(self, obj): - raise TypeError(f"{_type_repr(self)} cannot be used with isinstance().") - - def __subclasscheck__(self, cls): - raise TypeError(f"{_type_repr(self)} cannot be used with issubclass().") - - -class _FinalTypingBase(_TypingBase): - """Internal mix-in class to prevent instantiation. - - Prevents instantiation unless _root=True is given in class call. - It is used to create pseudo-singleton instances Any, Union, Optional, etc. - """ - - __slots__ = () - - def __new__(cls, *args, _root=False, **kwds): - self = super().__new__(cls, *args, **kwds) - if _root is True: - return self - raise TypeError("Cannot instantiate %r" % cls) - - def __reduce__(self): - return _trim_name(type(self).__name__) - - -######################################################################## -# Two non-singleton classes _ForwardRef (internal), and TypeVar (public) -######################################################################## - - -class _ForwardRef(_TypingBase): - """Internal wrapper to hold a forward reference.""" - - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__') - - def __init__(self, arg): - super().__init__(arg) - if not isinstance(arg, str): - raise TypeError('Forward reference must be a string -- got %r' % (arg,)) - try: - code = compile(arg, '', 'eval') - except SyntaxError: - raise SyntaxError('Forward reference must be an expression -- got %r' % - (arg,)) - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - - def _eval_type(self, globalns, localns): - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - self.__forward_value__ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.") - self.__forward_evaluated__ = True - return self.__forward_value__ - - def __eq__(self, other): - if not isinstance(other, _ForwardRef): - return NotImplemented - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_value__)) - - def __repr__(self): - return '%r*' % (self.__forward_arg__,) - - -class TypeVar(_TypingBase): - """Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows: - - def repeat(x: T, n: int) -> List[T]: - '''Return a list containing n references to x.''' - return [x]*n - - def longest(x: A, y: A) -> A: - '''Return the longest of two strings.''' - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of (str, str) -> str and (bytes, bytes) -> bytes. Also note - that if the arguments are instances of some subclass of str, - the return type is still plain str. - - At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. - - Type variables defined with covariant=True or contravariant=True - can be used do declare covariant or contravariant generic types. - See PEP 484 for more details. By default generic types are invariant - in all type variables. - - Type variables can be introspected. e.g.: - - T.__name__ == 'T' - T.__constraints__ == () - T.__covariant__ == False - T.__contravariant__ = False - A.__constraints__ == (str, bytes) - """ - - __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__') - - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False): - self.__name__ = name - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - if constraints and bound is not None: - raise TypeError("Constraints cannot be combined with bound=...") - if constraints and len(constraints) == 1: - raise TypeError("A single constraint is not allowed") - msg = "TypeVar(name, constraint, ...): constraints must be types." - self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None - - def _get_type_vars(self, tvars): - if self not in tvars: - tvars.append(self) - - def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) -T = TypeVar('T') # Any type. -KT = TypeVar('KT') # Key type. -VT = TypeVar('VT') # Value type. -T_co = TypeVar('T_co', covariant=True) # Any type covariant containers. -V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. -VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. -T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. - -# A useful type variable with constraints. This represents string types. -# (This one *is* for export!) -AnyStr = TypeVar('AnyStr', bytes, str) - -#################################################### -# Two non-subscriptable singletons: Any and NoReturn -#################################################### - -class _Any(_FinalTypingBase): - """Special type indicating an unconstrained type. - - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. - - Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance - or class checks. - """ - - __slots__ = () - - -Any = _Any(_root=True) - - -class _NoReturn(_FinalTypingBase): - """Special type indicating functions that never return. - Example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise Exception('no way') - - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. - """ - - __slots__ = () - - -NoReturn = _NoReturn(_root=True) - -################################################################## -# The main class: represents subscriptions of all singletons below -################################################################## - -class GenericType(_TypingBase): - - __slots__ = ('__origin__', '__args__', '__parameters__', - '_name', '_call') - - def __init__(self, origin, args, name=None, call=True): - self.__origin__ = origin - self.__args__ = args - self.__parameters__ = () - self._call = call - if name: - self._name = name - elif isinstance(origin, type): - self._name = origin.__qualname__ - else: - self._name = repr(origin) - def __call__(self, *args, **kwargs): - if not self._call: - raise TypeError(f'{self} is not callable') - return self.__origin__(*args, **kwargs) - def __base_subclass__(self, bases): - return self.__origin__ - def __getitem__(self, args): - return _update_args(self, args) - def _eval_type(self): - return self - def __repr__(self): - args = ', '.join([_type_repr(a) for a in self.__args__]) - return self._name + f'[{args}]' - - -###################################################################### -# Subscriptable non-subclassable singletons: ClassVar, Union, Optional -###################################################################### - -class _ClassVar(_FinalTypingBase): - """Special type construct to mark class variables. - - An annotation wrapped in ClassVar indicates that a given - attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - ClassVar accepts only types and cannot be further subscribed. - - Note that ClassVar is not a class itself, and should not - be used with isinstance() or issubclass(). - """ - - __slots__ = () - - def __getitem__(self, item): - return GenericType(self, [item]) - - -ClassVar = _ClassVar(_root=True) - - -class _Union(_FinalTypingBase): - """Union type; Union[X, Y] means either X or Y. - - To define a union, use e.g. Union[int, str]. Details: - - - The arguments must be types and there must be at least one. - - - None as an argument is a special case and is replaced by - type(None). - - - Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - - When two arguments have a subclass relationship, the least - derived argument is kept, e.g.:: - - class Employee: pass - class Manager(Employee): pass - Union[int, Employee, Manager] == Union[int, Employee] - Union[Manager, int, Employee] == Union[int, Employee] - Union[Employee, Manager] == Employee - - - Similar for object:: - - Union[int, object] == object - - - You cannot subclass or instantiate a union. - - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - - __slots__ = () - - def __getitem__(self, parameters): - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - return GenericType(self, parameters) - -Union = _Union(_root=True) - - -class _Optional(_FinalTypingBase): - """Optional type. - - Optional[X] is equivalent to Union[X, None]. - """ - - __slots__ = () - - def __getitem__(self, arg): - arg = _type_check(arg, "Optional[t] requires a single type.") - return Union[arg, type(None)] - - -Optional = _Optional(_root=True) - - -############################################################# -# Subclassable singletons: Generic, Callable, Tuple, Protocol -############################################################# - -class Generic(_FinalTypingBase): - """Abstract base class for generic types. - - A generic type is typically declared by inheriting from - this class parameterized with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: - try: - return mapping[key] - except KeyError: - return default - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(self, args) - - -class Tuple(_FinalTypingBase): - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. - - Example: Tuple[T1, T2] is a tuple of two elements corresponding - to type variables T1 and T2. Tuple[int, float, str] is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, use Tuple[T, ...]. - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(tuple, args, name='Tuple', call=False) - - -class Callable(_FinalTypingBase): - """Callable type; Callable[[int], str] is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. - - There is no syntax to indicate optional or keyword arguments, - such function types are rarely used as callback types. - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(collections.abc.Callable, args) - - -class Protocol(_FinalTypingBase): - """Internal base class for protocol classes. - - This implements a simple-minded structural issubclass check - (similar but more general than the one-offs in collections.abc - such as Hashable). - """ - - __slots__ = () - - def __class_getitem__(self, args): - return GenericType(self, args) - - -######################################### -# Fancy classes: NamedTuple and TypedDict -######################################### - -def _make_nmtuple(name, types): - msg = "NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type" - types = [(n, _type_check(t, msg)) for n, t in types] - nm_tpl = collections.namedtuple(name, [n for n, t in types]) - # Prior to PEP 526, only _field_types attribute was assigned. - # Now, both __annotations__ and _field_types are used to maintain compatibility. - nm_tpl.__annotations__ = nm_tpl._field_types = collections.OrderedDict(types) - try: - nm_tpl.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass - return nm_tpl - - -_PY36 = sys.version_info[:2] >= (3, 6) - -# attributes prohibited to set in NamedTuple class syntax -_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', - '_fields', '_field_defaults', '_field_types', - '_make', '_replace', '_asdict', '_source') - -_special = ('__module__', '__name__', '__qualname__', '__annotations__') - - -class NamedTupleMeta(type): - - def __new__(cls, typename, bases, ns): - if ns.get('_root', False): - return super().__new__(cls, typename, bases, ns) - if not _PY36: - raise TypeError("Class syntax for NamedTuple is only supported" - " in Python 3.6+") - types = ns.get('__annotations__', {}) - nm_tpl = _make_nmtuple(typename, types.items()) - defaults = [] - defaults_dict = {} - for field_name in types: - if field_name in ns: - default_value = ns[field_name] - defaults.append(default_value) - defaults_dict[field_name] = default_value - elif defaults: - raise TypeError("Non-default namedtuple field {field_name} cannot " - "follow default field(s) {default_names}" - .format(field_name=field_name, - default_names=', '.join(defaults_dict.keys()))) - nm_tpl.__new__.__defaults__ = tuple(defaults) - nm_tpl._field_defaults = defaults_dict - # update from user namespace without overriding special namedtuple attributes - for key in ns: - if key in _prohibited: - raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) - return nm_tpl - - -class NamedTuple(metaclass=NamedTupleMeta): - """Typed version of namedtuple. - - Usage in Python versions >= 3.6:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - The resulting class has extra __annotations__ and _field_types - attributes, giving an ordered dict mapping field names to types. - __annotations__ should be preferred, while _field_types - is kept to maintain pre PEP 526 compatibility. (The field names - are in the _fields attribute, which is part of the namedtuple - API.) Alternative equivalent keyword syntax is also accepted:: - - Employee = NamedTuple('Employee', name=str, id=int) - - In Python versions <= 3.5 use:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - """ - _root = True - - def __new__(self, typename, fields=None, **kwargs): - if kwargs and not _PY36: - raise TypeError("Keyword syntax for NamedTuple is only supported" - " in Python 3.6+") - if fields is None: - fields = kwargs.items() - elif kwargs: - raise TypeError("Either list of fields or keywords" - " can be provided to NamedTuple, not both") - return _make_nmtuple(typename, fields) - - -################################# -# Public functions (with helpers) -################################# - -def cast(typ, val): - """Cast a value to a type. - - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). - """ - return val - - -def _get_defaults(func): - """Internal helper to extract the default arguments, by name.""" - try: - code = func.__code__ - except AttributeError: - # Some built-in functions don't have __code__, __defaults__, etc. - return {} - pos_count = code.co_argcount - arg_names = code.co_varnames - arg_names = arg_names[:pos_count] - defaults = func.__defaults__ or () - kwdefaults = func.__kwdefaults__ - res = dict(kwdefaults) if kwdefaults else {} - pos_offset = pos_count - len(defaults) - for name, value in zip(arg_names[pos_offset:], defaults): - assert name not in res - res[name] = value - return res - - -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) - - -def get_type_hints(obj, globalns=None, localns=None): - """Return type hints for an object. - - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, and if necessary - adds Optional[t] if a default value equal to None is set. - - The argument may be a module, class, method, or function. The annotations - are returned as a dictionary. For classes, annotations include also - inherited members. - - TypeError is raised if the argument is not of a type that can contain - annotations, and an empty dictionary is returned if no annotations are - present. - - BEWARE -- the behavior of globalns and localns is counterintuitive - (unless you are familiar with how eval() and exec() work). The - search order is locals first, then globals. - - - If no dict arguments are passed, an attempt is made to use the - globals from obj, and these are also used as the locals. If the - object does not appear to have globals, an exception is raised. - - - If one dict argument is passed, it is used for both globals and - locals. - - - If two dict arguments are passed, they specify globals and - locals, respectively. - """ - - if getattr(obj, '__no_type_check__', None): - return {} - if globalns is None: - globalns = getattr(obj, '__globals__', {}) - if localns is None: - localns = globalns - elif localns is None: - localns = globalns - # Classes require a special treatment. - if isinstance(obj, type): - hints = {} - for base in reversed(obj.__mro__): - ann = base.__dict__.get('__annotations__', {}) - for name, value in ann.items(): - if value is None: - value = type(None) - if isinstance(value, str): - value = _ForwardRef(value) - value = _eval_type(value, globalns, localns) - hints[name] = value - return hints - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - defaults = _get_defaults(obj) - hints = dict(hints) - for name, value in hints.items(): - if value is None: - value = type(None) - if isinstance(value, str): - value = _ForwardRef(value) - value = _eval_type(value, globalns, localns) - if name in defaults and defaults[name] is None: - value = Optional[value] - hints[name] = value - return hints - - -def no_type_check(arg): - """Decorator to indicate that annotations are not type hints. - - The argument must be a class or function; if it is a class, it - applies recursively to all methods and classes defined in that class - (but not to methods defined in its superclasses or subclasses). - - This mutates the function(s) or class(es) in place. - """ - if isinstance(arg, type): - arg_attrs = arg.__dict__.copy() - for attr, val in arg.__dict__.items(): - if val in arg.__bases__ + (arg,): - arg_attrs.pop(attr) - for obj in arg_attrs.values(): - if isinstance(obj, types.FunctionType): - obj.__no_type_check__ = True - if isinstance(obj, type): - no_type_check(obj) - try: - arg.__no_type_check__ = True - except TypeError: # built-in classes - pass - return arg - - -def no_type_check_decorator(decorator): - """Decorator to give another decorator the @no_type_check effect. - - This wraps the decorator with something that wraps the decorated - function in @no_type_check. - """ - - @functools.wraps(decorator) - def wrapped_decorator(*args, **kwds): - func = decorator(*args, **kwds) - func = no_type_check(func) - return func - - return wrapped_decorator - - -def _overload_dummy(*args, **kwds): - """Helper for @overload to raise when called.""" - raise NotImplementedError( - "You should not call an overloaded function. " - "A series of @overload-decorated functions " - "outside a stub module should always be followed " - "by an implementation that is not @overload-ed.") - - -def overload(func): - """Decorator for overloaded functions/methods. - - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - def utf8(value): - # implementation goes here - """ - return _overload_dummy - - -def NewType(name, tp): - """NewType creates simple unique types with almost zero - runtime overhead. NewType(name, tp) is considered a subtype of tp - by static type checkers. At runtime, NewType(name, tp) returns - a dummy function that simply returns its argument. Usage:: - - UserId = NewType('UserId', int) - - def name_by_id(user_id: UserId) -> str: - ... - - UserId('user') # Fails type check - - name_by_id(42) # Fails type check - name_by_id(UserId(42)) # OK - - num = UserId(5) + 1 # type: int - """ - - def new_type(x): - return x - - new_type.__name__ = name - new_type.__supertype__ = tp - return new_type - - -###################################################################### -# Various generics mimicking those in collections.abc and -# additional protocols. A few are simply re-exported for completeness. -###################################################################### - -Hashable = collections.abc.Hashable # Not generic. - -Awaitable = GenericType(collections.abc.Awaitable, [T_co]) -Coroutine = GenericType(collections.abc.Coroutine, [T_co, T_contra, V_co]) -AsyncIterable = GenericType(collections.abc.AsyncIterable, [T_co]) -AsyncIterator = GenericType(collections.abc.AsyncIterator, [T_co]) - -Iterable = GenericType(collections.abc.Iterable, [T_co]) -Iterator = GenericType(collections.abc.Iterator, [T_co]) - -class SupportsInt(Protocol): - __slots__ = () - - @abstractmethod - def __int__(self) -> int: - pass - - -class SupportsFloat(Protocol): - __slots__ = () - - @abstractmethod - def __float__(self) -> float: - pass - - -class SupportsComplex(Protocol): - __slots__ = () - - @abstractmethod - def __complex__(self) -> complex: - pass - - -class SupportsBytes(Protocol): - __slots__ = () - - @abstractmethod - def __bytes__(self) -> bytes: - pass - - -class SupportsAbs(Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __abs__(self) -> T_co: - pass - - -class SupportsRound(Protocol[T_co]): - __slots__ = () - - @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: - pass - - -Reversible = GenericType(collections.abc.Reversible, [T_co]) -Sized = collections.abc.Sized # Not generic. -Container = GenericType(collections.abc.Container, [T_co]) -Collection = GenericType(collections.abc.Collection, [T_co]) - -# Callable was defined earlier. - -AbstractSet = GenericType(collections.abc.Set, [T_co]) -MutableSet = GenericType(collections.abc.MutableSet, [T]) -Set = GenericType(set, [T], name='Set') -FrozenSet = GenericType(frozenset, [T_co], name='FrozenSet') - -# NOTE: It is only covariant in the value type. -Mapping = GenericType(collections.abc.Mapping, [KT, VT_co]) -MutableMapping = GenericType(collections.abc.MutableMapping, [KT, VT]) -Dict = GenericType(dict, [KT, VT], call=False, name='Dict') -DefaultDict = GenericType(collections.defaultdict, [KT, VT], call=False, name='DefaultDict') -Counter = GenericType(collections.Counter, [T], call=False) -ChainMap = GenericType(collections.ChainMap, [KT, VT], call=False) - -Sequence = GenericType(collections.abc.Sequence, [T_co]) -MutableSequence = GenericType(collections.abc.MutableSequence, [T]) -ByteString = collections.abc.ByteString # Not generic. -List = GenericType(list, [T], name='List') -Deque = GenericType(collections.deque, [T], name='Deque') - -MappingView = GenericType(collections.abc.MappingView, [T_co]) -KeysView = GenericType(collections.abc.KeysView, [KT]) -ItemsView = GenericType(collections.abc.ItemsView, [KT, VT_co]) -ValuesView = GenericType(collections.abc.ValuesView, [VT_co]) - -ContextManager = GenericType(contextlib.AbstractContextManager, [T_co]) -if hasattr(contextlib, 'AbstractAsyncContextManager'): - AsyncContextManager = GenericType(contextlib.AbstractAsyncContextManager, [T_co]) - __all__.append('AsyncContextManager') - -Generator = GenericType(collections.abc.Generator, [T_co, T_contra, V_co]) -AsyncGenerator = GenericType(collections.abc.AsyncGenerator, [T_co, T_contra]) - -# Internal type variable used for Type[]. -CT_co = TypeVar('CT_co', covariant=True, bound=type) -# This is not a real generic class. Don't use outside annotations. -Type = GenericType(type, [CT_co], name='Type') -"""A special construct usable to annotate class objects. - -For example, suppose we have the following classes:: - - class User: ... # Abstract base for User classes - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - -And a function that takes a class argument that's a subclass of -User and returns an instance of the corresponding class:: - - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: - user = user_class() - # (Here we could write the user object to a database) - return user - - joe = new_user(BasicUser) - -At this point the type checker knows that joe has type BasicUser. -""" - -################## -# Public constants -################## - -# Python-version-specific alias (Python 2: unicode; Python 3: str) -Text = str - - -# Constant that's True when type checking, but False here. -TYPE_CHECKING = False - -######################################### -# Special io types and re generic aliases -######################################### - - -class IO(Generic[AnyStr]): - """Generic base class for TextIO and BinaryIO. - - This is an abstract, generic version of the return of open(). - - NOTE: This does not distinguish between the different possible - classes (text vs. binary, read vs. write vs. read/write, - append-only, unbuffered). The TextIO and BinaryIO subclasses - below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. - """ - - __slots__ = () - - @abstractproperty - def mode(self) -> str: - pass - - @abstractproperty - def name(self) -> str: - pass - - @abstractmethod - def close(self) -> None: - pass - - @abstractmethod - def closed(self) -> bool: - pass - - @abstractmethod - def fileno(self) -> int: - pass - - @abstractmethod - def flush(self) -> None: - pass - - @abstractmethod - def isatty(self) -> bool: - pass - - @abstractmethod - def read(self, n: int = -1) -> AnyStr: - pass - - @abstractmethod - def readable(self) -> bool: - pass - - @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: - pass - - @abstractmethod - def readlines(self, hint: int = -1) -> List[AnyStr]: - pass - - @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: - pass - - @abstractmethod - def seekable(self) -> bool: - pass - - @abstractmethod - def tell(self) -> int: - pass - - @abstractmethod - def truncate(self, size: int = None) -> int: - pass - - @abstractmethod - def writable(self) -> bool: - pass - - @abstractmethod - def write(self, s: AnyStr) -> int: - pass - - @abstractmethod - def writelines(self, lines: List[AnyStr]) -> None: - pass - - @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': - pass - - @abstractmethod - def __exit__(self, type, value, traceback) -> None: - pass - - -class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" - - __slots__ = () - - @abstractmethod - def write(self, s: Union[bytes, bytearray]) -> int: - pass - - @abstractmethod - def __enter__(self) -> 'BinaryIO': - pass - - -class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" - - __slots__ = () - - @abstractproperty - def buffer(self) -> BinaryIO: - pass - - @abstractproperty - def encoding(self) -> str: - pass - - @abstractproperty - def errors(self) -> Optional[str]: - pass - - @abstractproperty - def line_buffering(self) -> bool: - pass - - @abstractproperty - def newlines(self) -> Any: - pass - - @abstractmethod - def __enter__(self) -> 'TextIO': - pass - - -class io: - """Wrapper namespace for IO generic classes.""" - - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO - - -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io - - -Pattern = GenericType(type(stdlib_re.compile('')), [AnyStr], name='Pattern') -Match = GenericType(type(stdlib_re.match('', '')), [AnyStr], name='Match') - - -class re: - """Wrapper namespace for re type aliases.""" - - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match - - -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re From a8437fa175c437cede74aadfd3b60cbf0de3fcd6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2017 11:29:29 +0200 Subject: [PATCH 11/38] Rename base_subclass to subclass_base --- Python/bltinmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 215a2059b518f3..d710ded2199b45 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - new_base_meth = PyObject_GetAttrString(base, "__base_subclass__"); + new_base_meth = PyObject_GetAttrString(base, "__subclass_base__"); if (!new_base_meth) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -71,7 +71,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(new_base_meth)) { PyErr_SetString(PyExc_TypeError, - "__base_subclass__ must be callable"); + "__subclass_base__ must be callable"); return NULL; } new_base = _PyObject_FastCall(new_base_meth, stack, 1); From c2d8ac2f8dd8c53ba6c655544f10b33b957305cd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2017 12:11:18 +0200 Subject: [PATCH 12/38] Start adding tests --- Lib/test/test_genericclass.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Lib/test/test_genericclass.py diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py new file mode 100644 index 00000000000000..da26269bc848ea --- /dev/null +++ b/Lib/test/test_genericclass.py @@ -0,0 +1,42 @@ +import unittest + + +class TestSubclassBase(unittest.TestCase): + def test_subclass_base(self): + pass + + def test_subclass_base_with_builtins(self): + pass + + def test_subclass_base_errors(self): + pass + + def test_subclass_base_wrong(self): + pass + + def test_subclass_base_metaclass(self): + pass + + def test_subclass_base_type_call(self): + pass + + +class TestClassGetitem(unittest.TestCase): + def test_class_getitem(self): + pass + + def test_class_getitem_with_builtins(self): + pass + + def test_class_getitem_errors(self): + pass + + def test_class_getitem_inheritance(self): + pass + + def test_class_getitem_metaclass(self): + pass + + +if __name__ == "__main__": + unittest.main() From bc06c6b3fa22d2ec00c672b386ab808579b691bb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Sep 2017 11:57:53 +0200 Subject: [PATCH 13/38] Add tests for __subclass_base__ --- Lib/test/test_genericclass.py | 131 ++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index da26269bc848ea..020633fe37f8b8 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -2,23 +2,140 @@ class TestSubclassBase(unittest.TestCase): + def test_subclass_base_signature(self): + tested = [] + class B: ... + class C: + def __subclass_base__(self, *args, **kwargs): + tested.extend([args, kwargs]) + return C + c = C() + self.assertEqual(tested, []) + class D(B, c): ... + self.assertEqual(tested[0], ((B, c),)) + self.assertEqual(tested[1], {}) + def test_subclass_base(self): - pass + tested = [] + class A: ... + class B: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return self.__class__ + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, C, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, C, B, object)) + d = D() + class E(d): ... + self.assertEqual(tested[-1], (d,)) + self.assertEqual(E.__bases__, (D,)) + + def test_subclass_base_none(self): + tested = [] + class A: ... + class B: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return None + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + class E(c): ... + self.assertEqual(tested[-1], (c,)) + self.assertEqual(E.__bases__, (object,)) + self.assertEqual(E.__orig_bases__, (c,)) + self.assertEqual(E.__mro__, (E, object)) def test_subclass_base_with_builtins(self): - pass + tested = [] + class A: ... + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return dict + c = C() + self.assertEqual(tested, []) + class D(A, c): ... + self.assertEqual(tested[-1], (A, c)) + self.assertEqual(D.__bases__, (A, dict)) + self.assertEqual(D.__orig_bases__, (A, c)) + self.assertEqual(D.__mro__, (D, A, dict, object)) + + def test_subclass_base_with_builtins_2(self): + tested = [] + class C: + def __subclass_base__(self, bases): + tested.append(bases) + return C + c = C() + self.assertEqual(tested, []) + class D(c, dict): ... + self.assertEqual(tested[-1], (c, dict)) + self.assertEqual(D.__bases__, (C, dict)) + self.assertEqual(D.__orig_bases__, (c, dict)) + self.assertEqual(D.__mro__, (D, C, dict, object)) def test_subclass_base_errors(self): - pass + class C_too_many: + def __subclass_base__(self, bases, something, other): + return None + c = C_too_many() + with self.assertRaises(TypeError): + class D(c): ... + class C_too_few: + def __subclass_base__(self, bases, something, other): + return None + d = C_too_few() + with self.assertRaises(TypeError): + class D(d): ... - def test_subclass_base_wrong(self): - pass + def test_subclass_base_errors_2(self): + class C_not_callable: + __subclass_base__ = "Surprise!" + c = C_not_callable() + with self.assertRaises(TypeError): + class D(c): ... def test_subclass_base_metaclass(self): - pass + meta_args = [] + class Meta(type): + def __new__(mcls, name, bases, ns): + meta_args.extend([mcls, name, bases, ns]) + return super().__new__(mcls, name, bases, ns) + class A: ... + class C: + def __subclass_base__(self, bases): + return A + c = C() + class D(c, metaclass=Meta): + x = 1 + self.assertEqual(meta_args[0], Meta) + self.assertEqual(meta_args[1], 'D') + self.assertEqual(meta_args[2], (A,)) + self.assertEqual(meta_args[3]['x'], 1) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + self.assertEqual(D.__class__, Meta) def test_subclass_base_type_call(self): - pass + # Substitution should _not_ happen in direct type call + class C: + def __subclass_base__(self, bases): + return None + c = C() + with self.assertRaises(TypeError): + type('Bad', (c,), {}) class TestClassGetitem(unittest.TestCase): From 86c5d615fd1c1660a1325300f3e0199c6a276225 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Sep 2017 15:21:06 +0200 Subject: [PATCH 14/38] Add tests for __class_getitem__ --- Lib/test/test_genericclass.py | 103 +++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 020633fe37f8b8..76ec8000aa5b65 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -140,19 +140,104 @@ def __subclass_base__(self, bases): class TestClassGetitem(unittest.TestCase): def test_class_getitem(self): - pass - + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_patched(self): + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = __class_getitem__ + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + def test_class_getitem_with_builtins(self): - pass + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) def test_class_getitem_errors(self): - pass - - def test_class_getitem_inheritance(self): - pass - + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + def test_class_getitem_metaclass(self): - pass + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_metaclass_2(self): + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') if __name__ == "__main__": From 2cef78ae92353ca667730d2a1d7a81f36f9501a9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2017 22:38:44 +0200 Subject: [PATCH 15/38] Fix trailing whitespace --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 76ec8000aa5b65..a61b9e9ee9c3c4 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -195,7 +195,7 @@ class B(A): self.assertIs(B.called_with, None) B[int] self.assertIs(B.called_with, int) - + def test_class_getitem_errors(self): class C_too_few: def __class_getitem__(cls): From b9bbf7c569027cc9bd16d731683a08ea3838ecde Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 17:58:11 +0200 Subject: [PATCH 16/38] Alternative implementation of __class_getitem__ --- Objects/abstract.c | 17 +++++++++++++++++ Objects/typeobject.c | 32 +------------------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 3cb7a32b01ee5a..0d75067d36bf33 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,6 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; + PyObject *meth, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -168,6 +169,22 @@ PyObject_GetItem(PyObject *o, PyObject *key) "be integer, not '%.200s'", key); } + if (PyType_Check(o)) { + meth = PyObject_GetAttrString(o, "__class_getitem__"); + if (meth) { + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__class_getitem__ must be callable"); + return NULL; + } + return _PyObject_FastCall(meth, stack, 2); + } + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + } + return type_error("'%.200s' object is not subscriptable", o); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 66fb921c3c615e..2a8118b43c5a0a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3488,36 +3488,6 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } -static PyObject * -generic_subscript(PyTypeObject *tp, PyObject *key) -{ - PyObject* stack[2] = {(PyObject *)tp, key}; - const char* msg = "'%.200s' object is not subscriptable"; - PyObject *meth = PyObject_GetAttrString((PyObject *)tp, "__class_getitem__"); - if (meth) { - if (!PyCallable_Check(meth)) { - PyErr_SetString(PyExc_TypeError, - "__class_getitem__ must be callable"); - return NULL; - } - return _PyObject_FastCall(meth, stack, 2); - } - else { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, msg, ((PyObject *)tp)->ob_type->tp_name); - return NULL; - } - return NULL; - } -} - -static PyMappingMethods generic_as_mapping = { - NULL, /*mp_length*/ - (binaryfunc)generic_subscript, /*mp_subscript*/ - NULL, /*mp_ass_subscript*/ -}; - PyTypeObject PyType_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "type", /* tp_name */ @@ -3531,7 +3501,7 @@ PyTypeObject PyType_Type = { (reprfunc)type_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ - &generic_as_mapping, /* tp_as_mapping */ + 0, /* tp_as_mapping */ 0, /* tp_hash */ (ternaryfunc)type_call, /* tp_call */ 0, /* tp_str */ From 2495e942c7f90ff7b2292ef44ec5d82bf2281619 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 19:36:32 +0200 Subject: [PATCH 17/38] Simplify code and fix reference counting --- Objects/abstract.c | 7 +++++-- Python/bltinmodule.c | 49 ++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 0d75067d36bf33..ec11a1e5f3f789 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,7 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; - PyObject *meth, *stack[2] = {o, key}; + PyObject *meth, *result, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -175,9 +175,12 @@ PyObject_GetItem(PyObject *o, PyObject *key) if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, "__class_getitem__ must be callable"); + Py_DECREF(meth); return NULL; } - return _PyObject_FastCall(meth, stack, 2); + result = _PyObject_FastCall(meth, stack, 2); + Py_DECREF(meth); + return result; } else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d710ded2199b45..0341aee155311c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -49,9 +49,9 @@ _Py_IDENTIFIER(stderr); static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - int i, ind, tot_nones; - PyObject *base, *new_base, *new_base_meth, *new_bases; - PyObject* stack[1] = {bases}; + int i, ind, tot_nones = 0; + PyObject *base, *meth, *new_base, *new_bases; + PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); /* We have a separate cycle to calculate replacements with the idea that in @@ -61,45 +61,41 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - new_base_meth = PyObject_GetAttrString(base, "__subclass_base__"); - if (!new_base_meth) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - continue; + if (!(meth = PyObject_GetAttrString(base, "__subclass_base__"))) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; } - return NULL; + PyErr_Clear(); + continue; } - if (!PyCallable_Check(new_base_meth)) { + if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, "__subclass_base__ must be callable"); + Py_DECREF(meth); return NULL; } - new_base = _PyObject_FastCall(new_base_meth, stack, 1); - if (!new_base){ + if (!(new_base = _PyObject_FastCall(meth, stack, 1))){ + Py_DECREF(meth); return NULL; } - Py_INCREF(new_base); + if (new_base == Py_None) { + tot_nones++; + } + Py_DECREF(base); args[i] = new_base; *modified_bases = 1; + Py_DECREF(meth); } if (!*modified_bases){ return bases; } - /* Find out have many bases wants to be removed to pre-allocate - the tuple for new bases, then keep only non-None's in bases.*/ - tot_nones = 0; - for (i = 2; i < nargs; i++) { - if (args[i] == Py_None) { - tot_nones++; - } - } new_bases = PyTuple_New(nargs - 2 - tot_nones); ind = 0; for (i = 2; i < nargs; i++) { - base = args[i]; - if (base != Py_None) { - Py_INCREF(base); - PyTuple_SET_ITEM(new_bases, ind, base); + new_base = args[i]; + if (new_base != Py_None) { + Py_INCREF(new_base); + PyTuple_SET_ITEM(new_bases, ind, new_base); ind++; } } @@ -281,6 +277,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, Py_DECREF(meta); Py_XDECREF(mkw); Py_DECREF(bases); + if (modified_bases) { + Py_DECREF(old_bases); + } return cls; } From 9ca8dfca2357d5f1556966c23f346b6d609b98c1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:03:10 +0100 Subject: [PATCH 18/38] Rename __base_subclass__ to __mro_entry__ --- Lib/test/test_genericclass.py | 40 +++++++++++++++++------------------ Python/bltinmodule.c | 4 ++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index a61b9e9ee9c3c4..4e01a75f0d2a4e 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,12 +1,12 @@ import unittest -class TestSubclassBase(unittest.TestCase): - def test_subclass_base_signature(self): +class TestMROEntry(unittest.TestCase): + def test_mro_entry_signature(self): tested = [] class B: ... class C: - def __subclass_base__(self, *args, **kwargs): + def __mro_entry__(self, *args, **kwargs): tested.extend([args, kwargs]) return C c = C() @@ -15,12 +15,12 @@ class D(B, c): ... self.assertEqual(tested[0], ((B, c),)) self.assertEqual(tested[1], {}) - def test_subclass_base(self): + def test_mro_entry(self): tested = [] class A: ... class B: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return self.__class__ c = C() @@ -35,12 +35,12 @@ class E(d): ... self.assertEqual(tested[-1], (d,)) self.assertEqual(E.__bases__, (D,)) - def test_subclass_base_none(self): + def test_mro_entry_none(self): tested = [] class A: ... class B: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return None c = C() @@ -56,11 +56,11 @@ class E(c): ... self.assertEqual(E.__orig_bases__, (c,)) self.assertEqual(E.__mro__, (E, object)) - def test_subclass_base_with_builtins(self): + def test_mro_entry_with_builtins(self): tested = [] class A: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return dict c = C() @@ -71,10 +71,10 @@ class D(A, c): ... self.assertEqual(D.__orig_bases__, (A, c)) self.assertEqual(D.__mro__, (D, A, dict, object)) - def test_subclass_base_with_builtins_2(self): + def test_mro_entry_with_builtins_2(self): tested = [] class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): tested.append(bases) return C c = C() @@ -85,28 +85,28 @@ class D(c, dict): ... self.assertEqual(D.__orig_bases__, (c, dict)) self.assertEqual(D.__mro__, (D, C, dict, object)) - def test_subclass_base_errors(self): + def test_mro_entry_errors(self): class C_too_many: - def __subclass_base__(self, bases, something, other): + def __mro_entry__(self, bases, something, other): return None c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __subclass_base__(self, bases, something, other): + def __mro_entry__(self, bases, something, other): return None d = C_too_few() with self.assertRaises(TypeError): class D(d): ... - def test_subclass_base_errors_2(self): + def test_mro_entry_errors_2(self): class C_not_callable: - __subclass_base__ = "Surprise!" + __mro_entry__ = "Surprise!" c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... - def test_subclass_base_metaclass(self): + def test_mro_entry_metaclass(self): meta_args = [] class Meta(type): def __new__(mcls, name, bases, ns): @@ -114,7 +114,7 @@ def __new__(mcls, name, bases, ns): return super().__new__(mcls, name, bases, ns) class A: ... class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): return A c = C() class D(c, metaclass=Meta): @@ -128,10 +128,10 @@ class D(c, metaclass=Meta): self.assertEqual(D.__mro__, (D, A, object)) self.assertEqual(D.__class__, Meta) - def test_subclass_base_type_call(self): + def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: - def __subclass_base__(self, bases): + def __mro_entry__(self, bases): return None c = C() with self.assertRaises(TypeError): diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 0341aee155311c..26c112b0d999e5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - if (!(meth = PyObject_GetAttrString(base, "__subclass_base__"))) { + if (!(meth = PyObject_GetAttrString(base, "__mro_entry__"))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; } @@ -70,7 +70,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, - "__subclass_base__ must be callable"); + "__mro_entry__ must be callable"); Py_DECREF(meth); return NULL; } From 74bd36f8a4cbcb4bc822a56dab701c58fe77f2ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:27:18 +0100 Subject: [PATCH 19/38] Add types.resolve_bases --- Lib/test/test_types.py | 26 ++++++++++++++++++++++++++ Lib/types.py | 24 ++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 28133a3560f3c5..da82daa26a9200 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -844,6 +844,17 @@ def func(ns): self.assertEqual(C.y, 1) self.assertEqual(C.z, 2) + def test_new_class_with_mro_entry(self): + class A: pass + class C: + def __mro_entry__(self, bases): + return A + c = C() + D = types.new_class('D', (c,), {}) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -886,6 +897,21 @@ def __prepare__(*args): class Bar(metaclass=BadMeta()): pass + def test_resolve_bases(self): + class A: pass + class C: + def __mro_entry__(self, bases): + if A in bases: + return None + return A + c = C() + self.assertEqual(types.resolve_bases(()), ()) + self.assertEqual(types.resolve_bases((c,)), (A,)) + self.assertEqual(types.resolve_bases((C,)), (C,)) + self.assertEqual(types.resolve_bases((A, C)), (A, C)) + self.assertEqual(types.resolve_bases((c, A)), (A,)) + self.assertEqual(types.resolve_bases((A, c)), (A,)) + def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation new_calls = [] # to check the order of __new__ calls diff --git a/Lib/types.py b/Lib/types.py index 336918fea09d4a..40e58846a78efa 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -60,10 +60,30 @@ def _m(self): pass # Provide a PEP 3115 compliant mechanism for class creation def new_class(name, bases=(), kwds=None, exec_body=None): """Create a class object dynamically using the appropriate metaclass.""" - meta, ns, kwds = prepare_class(name, bases, kwds) + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) - return meta(name, bases, ns, **kwds) + cls = meta(name, resolved_bases, ns, **kwds) + if resolved_bases is not bases: + cls.__orig_bases__ = bases + return cls + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entry__"): + continue + new_base = base.__mro_entry__(bases) + updated = True + new_bases[i] = new_base + if not updated: + return bases + return tuple(b for b in new_bases if b is not None) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. From 6f20c4518bc4220f8f71ca9725e084ddbf976d2b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 01:31:18 +0100 Subject: [PATCH 20/38] Make Python and C versions as similar as possible --- Lib/types.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/types.py b/Lib/types.py index 40e58846a78efa..c1ed6bdc8b8066 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -64,10 +64,9 @@ def new_class(name, bases=(), kwds=None, exec_body=None): meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) - cls = meta(name, resolved_bases, ns, **kwds) if resolved_bases is not bases: - cls.__orig_bases__ = bases - return cls + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) def resolve_bases(bases): """Resolve MRO entries dynamically as specified by PEP 560.""" From bcfbcf60fc9db8cbf09e1c308aa64bc8126be727 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:13:50 +0100 Subject: [PATCH 21/38] More tests --- Lib/test/test_types.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index da82daa26a9200..f77a8bb363118e 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -855,6 +855,18 @@ def __mro_entry__(self, bases): self.assertEqual(D.__orig_bases__, (c,)) self.assertEqual(D.__mro__, (D, A, object)) + def test_new_class_with_mro_entry_none(self): + class A: pass + class B: pass + class C: + def __mro_entry__(self, bases): + return None + c = C() + D = types.new_class('D', (A, c, B), {}) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -899,6 +911,7 @@ class Bar(metaclass=BadMeta()): def test_resolve_bases(self): class A: pass + class B: pass class C: def __mro_entry__(self, bases): if A in bases: @@ -911,6 +924,12 @@ def __mro_entry__(self, bases): self.assertEqual(types.resolve_bases((A, C)), (A, C)) self.assertEqual(types.resolve_bases((c, A)), (A,)) self.assertEqual(types.resolve_bases((A, c)), (A,)) + x = (A,) + y = (C,) + z = (A, C) + t = (A, C, B) + for bases in [x, y, z, t]: + self.assertIs(types.resolve_bases(bases), bases) def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation From d7bd63080d25ca50fcd1101839dea0202ad7d0c2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:51:06 +0100 Subject: [PATCH 22/38] Fail fast in type.__new__ --- Objects/typeobject.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2a8118b43c5a0a..8d48bd845df5f8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2377,6 +2377,15 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) nbases = 1; } else { + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entry__")) { + PyErr_SetString(PyExc_TypeError, + "type() doesn't support MRO entry resolution; " + "use types.new_class()"); + return NULL; + } + } /* Search the bases for the proper metatype to deal with this: */ winner = _PyType_CalculateMetaclass(metatype, bases); if (winner == NULL) { From 29e54c334dc00427644db08fdad2a73bcb2dc8bd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 10:57:52 +0100 Subject: [PATCH 23/38] Fix a test --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 4e01a75f0d2a4e..691ee0b6f775d7 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -93,7 +93,7 @@ def __mro_entry__(self, bases, something, other): with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __mro_entry__(self, bases, something, other): + def __mro_entry__(self): return None d = C_too_few() with self.assertRaises(TypeError): From cba2a58f8417fa44993c5a5f6113b6476875cb4f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:12:36 +0100 Subject: [PATCH 24/38] Test error message --- Lib/test/test_genericclass.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 691ee0b6f775d7..fde788008cc664 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -134,7 +134,9 @@ class C: def __mro_entry__(self, bases): return None c = C() - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, + "MRO entry resolution; " + "use types.new_class()"): type('Bad', (c,), {}) From 966a9ea754e38b34a97b55a7e4b7b46d75cccbcd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:12:48 +0100 Subject: [PATCH 25/38] Test error message --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index fde788008cc664..556c2d2f4fe9ed 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -136,7 +136,7 @@ def __mro_entry__(self, bases): c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " - "use types.new_class()"): + "use type.new_class()"): type('Bad', (c,), {}) From 734c7e9a79796c6f2f685bde1de188ff0b439446 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 11 Nov 2017 11:13:13 +0100 Subject: [PATCH 26/38] Test error message --- Lib/test/test_genericclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 556c2d2f4fe9ed..fde788008cc664 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -136,7 +136,7 @@ def __mro_entry__(self, bases): c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " - "use type.new_class()"): + "use types.new_class()"): type('Bad', (c,), {}) From 152d2f6352d3eb70b3b364d4ee020b1bd126663e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 13 Nov 2017 12:43:26 +0100 Subject: [PATCH 27/38] Allow expansion to multiple bases --- Lib/test/test_genericclass.py | 8 ++++---- Lib/test/test_types.py | 4 ++-- Lib/types.py | 7 +++++-- Python/bltinmodule.c | 20 ++++++++++++++------ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index fde788008cc664..8ad4d8f79f1834 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -42,7 +42,7 @@ class B: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return None + return () c = C() self.assertEqual(tested, []) class D(A, c, B): ... @@ -88,13 +88,13 @@ class D(c, dict): ... def test_mro_entry_errors(self): class C_too_many: def __mro_entry__(self, bases, something, other): - return None + return () c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: def __mro_entry__(self): - return None + return () d = C_too_few() with self.assertRaises(TypeError): class D(d): ... @@ -132,7 +132,7 @@ def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: def __mro_entry__(self, bases): - return None + return () c = C() with self.assertRaisesRegex(TypeError, "MRO entry resolution; " diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index f77a8bb363118e..b7dd83f50de44d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -860,7 +860,7 @@ class A: pass class B: pass class C: def __mro_entry__(self, bases): - return None + return () c = C() D = types.new_class('D', (A, c, B), {}) self.assertEqual(D.__bases__, (A, B)) @@ -915,7 +915,7 @@ class B: pass class C: def __mro_entry__(self, bases): if A in bases: - return None + return () return A c = C() self.assertEqual(types.resolve_bases(()), ()) diff --git a/Lib/types.py b/Lib/types.py index c1ed6bdc8b8066..ef97da2df02cd1 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -79,10 +79,13 @@ def resolve_bases(bases): continue new_base = base.__mro_entry__(bases) updated = True - new_bases[i] = new_base + if not isinstance(new_base, tuple): + new_bases[i] = new_base + else: + new_bases[i:i+1] = new_base if not updated: return bases - return tuple(b for b in new_bases if b is not None) + return tuple(new_bases) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 26c112b0d999e5..ce209c540a9987 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -49,8 +49,8 @@ _Py_IDENTIFIER(stderr); static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { - int i, ind, tot_nones = 0; - PyObject *base, *meth, *new_base, *new_bases; + int i, j, ind, tot_extra = 0; + PyObject *base, *meth, *new_base, *new_sub_base, *new_bases; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); @@ -78,8 +78,8 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) Py_DECREF(meth); return NULL; } - if (new_base == Py_None) { - tot_nones++; + if (PyTuple_Check(new_base)) { + tot_extra += PyTuple_Size(new_base) - 1; } Py_DECREF(base); args[i] = new_base; @@ -89,15 +89,23 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (!*modified_bases){ return bases; } - new_bases = PyTuple_New(nargs - 2 - tot_nones); + new_bases = PyTuple_New(nargs - 2 + tot_extra); ind = 0; for (i = 2; i < nargs; i++) { new_base = args[i]; - if (new_base != Py_None) { + if (!PyTuple_Check(new_base)) { Py_INCREF(new_base); PyTuple_SET_ITEM(new_bases, ind, new_base); ind++; } + else { + for (j = 0; j < PyTuple_Size(new_base); j++) { + new_sub_base = PyTuple_GET_ITEM(new_base, j); + Py_INCREF(new_sub_base); + PyTuple_SET_ITEM(new_bases, ind, new_sub_base); + ind++; + } + } } return new_bases; } From 1a218e0c42faa7e8d9b34795dc6e3f2f4ccd5fb0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:31:07 +0100 Subject: [PATCH 28/38] Prohibit returning non-tuple --- Lib/test/test_genericclass.py | 16 +++++++++++----- Python/bltinmodule.c | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 8ad4d8f79f1834..e810887b744094 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -8,7 +8,7 @@ class B: ... class C: def __mro_entry__(self, *args, **kwargs): tested.extend([args, kwargs]) - return C + return (C,) c = C() self.assertEqual(tested, []) class D(B, c): ... @@ -22,7 +22,7 @@ class B: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return self.__class__ + return (self.__class__,) c = C() self.assertEqual(tested, []) class D(A, c, B): ... @@ -62,7 +62,7 @@ class A: ... class C: def __mro_entry__(self, bases): tested.append(bases) - return dict + return (dict,) c = C() self.assertEqual(tested, []) class D(A, c): ... @@ -76,7 +76,7 @@ def test_mro_entry_with_builtins_2(self): class C: def __mro_entry__(self, bases): tested.append(bases) - return C + return (C,) c = C() self.assertEqual(tested, []) class D(c, dict): ... @@ -105,6 +105,12 @@ class C_not_callable: c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... + class C_not_tuple: + def __mro_entry__(self): + return object + c = C_not_tuple() + with self.assertRaises(TypeError): + class D(c): ... def test_mro_entry_metaclass(self): meta_args = [] @@ -115,7 +121,7 @@ def __new__(mcls, name, bases, ns): class A: ... class C: def __mro_entry__(self, bases): - return A + return (A,) c = C() class D(c, metaclass=Meta): x = 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ce209c540a9987..6b3f28f7fe1256 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -81,6 +81,12 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyTuple_Check(new_base)) { tot_extra += PyTuple_Size(new_base) - 1; } + else { + PyErr_SetString(PyExc_TypeError, + "__mro_entry__ must return a tuple"); + Py_DECREF(meth); + return NULL; + } Py_DECREF(base); args[i] = new_base; *modified_bases = 1; From 7cc8d8ff0b1f54bbb4ecb131098e74afedf95032 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 10:41:30 +0100 Subject: [PATCH 29/38] Rename to __mro_entries__ --- Lib/test/test_genericclass.py | 22 +++++++++++----------- Lib/test/test_types.py | 19 ++++++++++++++----- Lib/types.py | 6 +++--- Objects/typeobject.c | 2 +- Python/bltinmodule.c | 6 +++--- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index e810887b744094..214527b01fa871 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -6,7 +6,7 @@ def test_mro_entry_signature(self): tested = [] class B: ... class C: - def __mro_entry__(self, *args, **kwargs): + def __mro_entries__(self, *args, **kwargs): tested.extend([args, kwargs]) return (C,) c = C() @@ -20,7 +20,7 @@ def test_mro_entry(self): class A: ... class B: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (self.__class__,) c = C() @@ -40,7 +40,7 @@ def test_mro_entry_none(self): class A: ... class B: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return () c = C() @@ -60,7 +60,7 @@ def test_mro_entry_with_builtins(self): tested = [] class A: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (dict,) c = C() @@ -74,7 +74,7 @@ class D(A, c): ... def test_mro_entry_with_builtins_2(self): tested = [] class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): tested.append(bases) return (C,) c = C() @@ -87,13 +87,13 @@ class D(c, dict): ... def test_mro_entry_errors(self): class C_too_many: - def __mro_entry__(self, bases, something, other): + def __mro_entries__(self, bases, something, other): return () c = C_too_many() with self.assertRaises(TypeError): class D(c): ... class C_too_few: - def __mro_entry__(self): + def __mro_entries__(self): return () d = C_too_few() with self.assertRaises(TypeError): @@ -101,12 +101,12 @@ class D(d): ... def test_mro_entry_errors_2(self): class C_not_callable: - __mro_entry__ = "Surprise!" + __mro_entries__ = "Surprise!" c = C_not_callable() with self.assertRaises(TypeError): class D(c): ... class C_not_tuple: - def __mro_entry__(self): + def __mro_entries__(self): return object c = C_not_tuple() with self.assertRaises(TypeError): @@ -120,7 +120,7 @@ def __new__(mcls, name, bases, ns): return super().__new__(mcls, name, bases, ns) class A: ... class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return (A,) c = C() class D(c, metaclass=Meta): @@ -137,7 +137,7 @@ class D(c, metaclass=Meta): def test_mro_entry_type_call(self): # Substitution should _not_ happen in direct type call class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return () c = C() with self.assertRaisesRegex(TypeError, diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index b7dd83f50de44d..e822ff7bb5eeb1 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -847,8 +847,8 @@ def func(ns): def test_new_class_with_mro_entry(self): class A: pass class C: - def __mro_entry__(self, bases): - return A + def __mro_entries__(self, bases): + return (A,) c = C() D = types.new_class('D', (c,), {}) self.assertEqual(D.__bases__, (A,)) @@ -859,7 +859,7 @@ def test_new_class_with_mro_entry_none(self): class A: pass class B: pass class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): return () c = C() D = types.new_class('D', (A, c, B), {}) @@ -867,6 +867,15 @@ def __mro_entry__(self, bases): self.assertEqual(D.__orig_bases__, (A, c, B)) self.assertEqual(D.__mro__, (D, A, B, object)) + def test_new_class_with_mro_entry_error(self): + class A: pass + class C: + def __mro_entries__(self, bases): + return A + c = C() + with self.assertRaises(TypeError): + types.new_class('D', (c,), {}) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -913,10 +922,10 @@ def test_resolve_bases(self): class A: pass class B: pass class C: - def __mro_entry__(self, bases): + def __mro_entries__(self, bases): if A in bases: return () - return A + return (A,) c = C() self.assertEqual(types.resolve_bases(()), ()) self.assertEqual(types.resolve_bases((c,)), (A,)) diff --git a/Lib/types.py b/Lib/types.py index ef97da2df02cd1..08aabd6343db60 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -75,12 +75,12 @@ def resolve_bases(bases): for i, base in enumerate(bases): if isinstance(base, type): continue - if not hasattr(base, "__mro_entry__"): + if not hasattr(base, "__mro_entries__"): continue - new_base = base.__mro_entry__(bases) + new_base = base.__mro_entries__(bases) updated = True if not isinstance(new_base, tuple): - new_bases[i] = new_base + raise TypeError("__mro_entries__ must return a tuple") else: new_bases[i:i+1] = new_base if not updated: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8d48bd845df5f8..4713082e19dac4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2379,7 +2379,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) else { for (i = 0; i < nbases; i++) { tmp = PyTuple_GET_ITEM(bases, i); - if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entry__")) { + if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entries__")) { PyErr_SetString(PyExc_TypeError, "type() doesn't support MRO entry resolution; " "use types.new_class()"); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6b3f28f7fe1256..f282abee773ca1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -61,7 +61,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - if (!(meth = PyObject_GetAttrString(base, "__mro_entry__"))) { + if (!(meth = PyObject_GetAttrString(base, "__mro_entries__"))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; } @@ -70,7 +70,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, - "__mro_entry__ must be callable"); + "__mro_entries__ must be callable"); Py_DECREF(meth); return NULL; } @@ -83,7 +83,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } else { PyErr_SetString(PyExc_TypeError, - "__mro_entry__ must return a tuple"); + "__mro_entries__ must return a tuple"); Py_DECREF(meth); return NULL; } From d0c088950096ad7843d976d99142804fc5214314 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 5 Dec 2017 21:43:09 +0100 Subject: [PATCH 30/38] Add news entry --- .../Core and Builtins/2017-12-05-21-42-58.bpo-32226.G8fqb6.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-42-58.bpo-32226.G8fqb6.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-42-58.bpo-32226.G8fqb6.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-42-58.bpo-32226.G8fqb6.rst new file mode 100644 index 00000000000000..97954fd1bb0097 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-42-58.bpo-32226.G8fqb6.rst @@ -0,0 +1,2 @@ +PEP 560: Add support for __mro_entries__ and __class_getitem__. Implemented +by Ivan Levkivskyi. From bfea2f02fa3146f76e3b16a1f45fa2111e2e2f21 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Dec 2017 00:00:01 +0100 Subject: [PATCH 31/38] Fix indentation and refleak --- Objects/typeobject.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4713082e19dac4..2e56a54a508bd0 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2379,11 +2379,12 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) else { for (i = 0; i < nbases; i++) { tmp = PyTuple_GET_ITEM(bases, i); - if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entries__")) { - PyErr_SetString(PyExc_TypeError, - "type() doesn't support MRO entry resolution; " - "use types.new_class()"); - return NULL; + if (!PyType_Check(tmp) && (tmp = PyObject_GetAttrString(tmp, "__mro_entries__"))) { + PyErr_SetString(PyExc_TypeError, + "type() doesn't support MRO entry resolution; " + "use types.new_class()"); + Py_DECREF(tmp); + return NULL; } } /* Search the bases for the proper metatype to deal with this: */ From 978588ca1d79ea5941f17c2bc3c71477ca59f979 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Dec 2017 01:27:54 +0100 Subject: [PATCH 32/38] Address CR (part 1) --- Lib/test/test_types.py | 30 ++++++++++++++++++++++++++++++ Lib/types.py | 4 +++- Objects/abstract.c | 5 +++-- Objects/typeobject.c | 13 ++++++++++++- Python/bltinmodule.c | 19 ++++++++++--------- 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index e822ff7bb5eeb1..47488a615b1ebe 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -876,6 +876,36 @@ def __mro_entries__(self, bases): with self.assertRaises(TypeError): types.new_class('D', (c,), {}) + def test_new_class_with_mro_entry_multiple(self): + class A1: pass + class A2: pass + class B1: pass + class B2: pass + class A: + def __mro_entries__(self, bases): + return (A1, A2) + class B: + def __mro_entries__(self, bases): + return (B1, B2) + D = types.new_class('D', (A(), B()), {}) + self.assertEqual(D.__bases__, (A1, A2, B1, B2)) + + def test_new_class_with_mro_entry_multiple_2(self): + class A1: pass + class A2: pass + class A3: pass + class B1: pass + class B2: pass + class A: + def __mro_entries__(self, bases): + return (A1, A2, A3) + class B: + def __mro_entries__(self, bases): + return (B1, B2) + class C: pass + D = types.new_class('D', (A(), C, B()), {}) + self.assertEqual(D.__bases__, (A1, A2, A3, C, B1, B2)) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation diff --git a/Lib/types.py b/Lib/types.py index 08aabd6343db60..c5976f3057ffc5 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -72,6 +72,7 @@ def resolve_bases(bases): """Resolve MRO entries dynamically as specified by PEP 560.""" new_bases = list(bases) updated = False + shift = 0 for i, base in enumerate(bases): if isinstance(base, type): continue @@ -82,7 +83,8 @@ def resolve_bases(bases): if not isinstance(new_base, tuple): raise TypeError("__mro_entries__ must return a tuple") else: - new_bases[i:i+1] = new_base + new_bases[i+shift:i+shift+1] = new_base + shift += len(new_base) - 1 if not updated: return bases return tuple(new_bases) diff --git a/Objects/abstract.c b/Objects/abstract.c index ec11a1e5f3f789..52e2a3d0a01c6d 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,7 +143,6 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; - PyObject *meth, *result, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -170,7 +169,9 @@ PyObject_GetItem(PyObject *o, PyObject *key) } if (PyType_Check(o)) { - meth = PyObject_GetAttrString(o, "__class_getitem__"); + PyObject *meth, *result, *stack[2] = {o, key}; + _Py_IDENTIFIER(__class_getitem__); + meth = _PyObject_GetAttrId(o, &PyId___class_getitem__); if (meth) { if (!PyCallable_Check(meth)) { PyErr_SetString(PyExc_TypeError, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2e56a54a508bd0..39ea74b65d7543 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2377,15 +2377,26 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) nbases = 1; } else { + _Py_IDENTIFIER(__mro_entries__); for (i = 0; i < nbases; i++) { tmp = PyTuple_GET_ITEM(bases, i); - if (!PyType_Check(tmp) && (tmp = PyObject_GetAttrString(tmp, "__mro_entries__"))) { + if (PyType_Check(tmp)) { + continue; + } + tmp = _PyObject_GetAttrId(tmp, &PyId___mro_entries__); + if (tmp != NULL) { PyErr_SetString(PyExc_TypeError, "type() doesn't support MRO entry resolution; " "use types.new_class()"); Py_DECREF(tmp); return NULL; } + else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + return NULL; + } } /* Search the bases for the proper metatype to deal with this: */ winner = _PyType_CalculateMetaclass(metatype, bases); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 0bda04344cc1a5..80241c94bd4e33 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -34,6 +34,7 @@ _Py_IDENTIFIER(__builtins__); _Py_IDENTIFIER(__dict__); _Py_IDENTIFIER(__prepare__); _Py_IDENTIFIER(__round__); +_Py_IDENTIFIER(__mro_entries__); _Py_IDENTIFIER(encoding); _Py_IDENTIFIER(errors); _Py_IDENTIFIER(fileno); @@ -61,7 +62,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) if (PyType_Check(base)) { continue; } - if (!(meth = PyObject_GetAttrString(base, "__mro_entries__"))) { + if (!(meth = _PyObject_GetAttrId(base, &PyId___mro_entries__))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return NULL; } @@ -74,25 +75,25 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) Py_DECREF(meth); return NULL; } - if (!(new_base = _PyObject_FastCall(meth, stack, 1))){ - Py_DECREF(meth); + new_base = _PyObject_FastCall(meth, stack, 1); + Py_DECREF(meth); + if (!new_base) { return NULL; } if (PyTuple_Check(new_base)) { - tot_extra += PyTuple_Size(new_base) - 1; + tot_extra += PyTuple_GET_SIZE(new_base) - 1; } else { PyErr_SetString(PyExc_TypeError, "__mro_entries__ must return a tuple"); - Py_DECREF(meth); + Py_DECREF(new_base); return NULL; } Py_DECREF(base); args[i] = new_base; *modified_bases = 1; - Py_DECREF(meth); } - if (!*modified_bases){ + if (!*modified_bases) { return bases; } new_bases = PyTuple_New(nargs - 2 + tot_extra); @@ -105,7 +106,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) ind++; } else { - for (j = 0; j < PyTuple_Size(new_base); j++) { + for (j = 0; j < PyTuple_GET_SIZE(new_base); j++) { new_sub_base = PyTuple_GET_ITEM(new_base, j); Py_INCREF(new_sub_base); PyTuple_SET_ITEM(new_bases, ind, new_sub_base); @@ -250,7 +251,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { - if (modified_bases){ + if (modified_bases) { PyMapping_SetItemString(ns, "__orig_bases__", old_bases); } PyObject *margs[3] = {name, bases, ns}; From 7bff68d365889a83eb415d590ef09102a6666ac4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 8 Dec 2017 02:10:32 +0100 Subject: [PATCH 33/38] Address CR (part 2) use a temporary tuple instead of modifying args in place --- Python/bltinmodule.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 80241c94bd4e33..661cc02251d554 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -51,10 +51,11 @@ static PyObject* update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) { int i, j, ind, tot_extra = 0; - PyObject *base, *meth, *new_base, *new_sub_base, *new_bases; + PyObject *base, *meth, *new_base, *new_sub_base, *new_bases, *replacements; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); + replacements = NULL; /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ for (i = 2; i < nargs; i++){ @@ -64,7 +65,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } if (!(meth = _PyObject_GetAttrId(base, &PyId___mro_entries__))) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - return NULL; + goto error; } PyErr_Clear(); continue; @@ -73,12 +74,12 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyErr_SetString(PyExc_TypeError, "__mro_entries__ must be callable"); Py_DECREF(meth); - return NULL; + goto error; } new_base = _PyObject_FastCall(meth, stack, 1); Py_DECREF(meth); if (!new_base) { - return NULL; + goto error; } if (PyTuple_Check(new_base)) { tot_extra += PyTuple_GET_SIZE(new_base) - 1; @@ -87,10 +88,13 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyErr_SetString(PyExc_TypeError, "__mro_entries__ must return a tuple"); Py_DECREF(new_base); - return NULL; + goto error; + } + if (!replacements) { + replacements = PyTuple_New(nargs - 2); } - Py_DECREF(base); - args[i] = new_base; + Py_INCREF(new_base); + PyTuple_SET_ITEM(replacements, i - 2, new_base); *modified_bases = 1; } if (!*modified_bases) { @@ -99,10 +103,11 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) new_bases = PyTuple_New(nargs - 2 + tot_extra); ind = 0; for (i = 2; i < nargs; i++) { - new_base = args[i]; - if (!PyTuple_Check(new_base)) { - Py_INCREF(new_base); - PyTuple_SET_ITEM(new_bases, ind, new_base); + base = args[i]; + new_base = PyTuple_GET_ITEM(replacements, i - 2); + if (!new_base) { + Py_INCREF(base); + PyTuple_SET_ITEM(new_bases, ind, base); ind++; } else { @@ -112,9 +117,14 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyTuple_SET_ITEM(new_bases, ind, new_sub_base); ind++; } + Py_DECREF(new_base); } } + Py_DECREF(replacements); return new_bases; +error: + Py_XDECREF(replacements); + return NULL; } /* AC: cannot convert yet, waiting for *args support */ @@ -252,7 +262,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { if (modified_bases) { - PyMapping_SetItemString(ns, "__orig_bases__", old_bases); + if (PyMapping_SetItemString(ns, "__orig_bases__", old_bases) < 0) { + goto error; + } } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); From b3d85ff6ebf117ce0008cedb2813da3d1203c4a0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Dec 2017 22:01:03 +0100 Subject: [PATCH 34/38] Remove checks for callability of special methods --- Objects/abstract.c | 6 ------ Python/bltinmodule.c | 6 ------ 2 files changed, 12 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 52e2a3d0a01c6d..0105c5d16961e0 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -173,12 +173,6 @@ PyObject_GetItem(PyObject *o, PyObject *key) _Py_IDENTIFIER(__class_getitem__); meth = _PyObject_GetAttrId(o, &PyId___class_getitem__); if (meth) { - if (!PyCallable_Check(meth)) { - PyErr_SetString(PyExc_TypeError, - "__class_getitem__ must be callable"); - Py_DECREF(meth); - return NULL; - } result = _PyObject_FastCall(meth, stack, 2); Py_DECREF(meth); return result; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 661cc02251d554..0aafc644888a43 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -70,12 +70,6 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) PyErr_Clear(); continue; } - if (!PyCallable_Check(meth)) { - PyErr_SetString(PyExc_TypeError, - "__mro_entries__ must be callable"); - Py_DECREF(meth); - goto error; - } new_base = _PyObject_FastCall(meth, stack, 1); Py_DECREF(meth); if (!new_base) { From 687e1e7ca29134141fcac758dd487384015a0d90 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Dec 2017 00:06:50 +0100 Subject: [PATCH 35/38] Some simplifications --- Python/bltinmodule.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 0aafc644888a43..e0e15f25ffec68 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -48,9 +48,9 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" static PyObject* -update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) +update_bases(PyObject* bases, PyObject** args, int nargs) { - int i, j, ind, tot_extra = 0; + int i, j, ind, modified_bases = 0, tot_extra = 0; PyObject *base, *meth, *new_base, *new_sub_base, *new_bases, *replacements; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); @@ -89,9 +89,9 @@ update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) } Py_INCREF(new_base); PyTuple_SET_ITEM(replacements, i - 2, new_base); - *modified_bases = 1; + modified_bases = 1; } - if (!*modified_bases) { + if (!modified_bases) { return bases; } new_bases = PyTuple_New(nargs - 2 + tot_extra); @@ -130,7 +130,6 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *new_bases, *old_bases = NULL; PyObject *cls = NULL, *cell = NULL; int isclass = 0; /* initialize to prevent gcc warning */ - int modified_bases = 0; if (nargs < 2) { PyErr_SetString(PyExc_TypeError, @@ -153,7 +152,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, if (bases == NULL) return NULL; - new_bases = update_bases(bases, args, nargs, &modified_bases); + new_bases = update_bases(bases, args, nargs); if (new_bases == NULL) { Py_DECREF(bases); return NULL; @@ -255,7 +254,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { - if (modified_bases) { + if (bases != old_bases) { if (PyMapping_SetItemString(ns, "__orig_bases__", old_bases) < 0) { goto error; } @@ -298,7 +297,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, Py_DECREF(meta); Py_XDECREF(mkw); Py_DECREF(bases); - if (modified_bases) { + if (bases != old_bases) { Py_DECREF(old_bases); } return cls; From 4426e8cfbdf36ed01c1704494fbdde33e052402b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Dec 2017 01:04:59 +0100 Subject: [PATCH 36/38] More simplifications: use single pass --- Python/bltinmodule.c | 97 ++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 52 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e0e15f25ffec68..a8543c6f06a297 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -48,19 +48,23 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" static PyObject* -update_bases(PyObject* bases, PyObject** args, int nargs) +update_bases(PyObject* bases) { - int i, j, ind, modified_bases = 0, tot_extra = 0; - PyObject *base, *meth, *new_base, *new_sub_base, *new_bases, *replacements; + int i, j; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); - replacements = NULL; /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ - for (i = 2; i < nargs; i++){ - base = args[i]; + for (i = 0; i < PyTuple_GET_SIZE(bases); i++) { + base = PyTuple_GET_ITEM(bases, i); if (PyType_Check(base)) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } continue; } if (!(meth = _PyObject_GetAttrId(base, &PyId___mro_entries__))) { @@ -68,6 +72,11 @@ update_bases(PyObject* bases, PyObject** args, int nargs) goto error; } PyErr_Clear(); + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } continue; } new_base = _PyObject_FastCall(meth, stack, 1); @@ -76,7 +85,21 @@ update_bases(PyObject* bases, PyObject** args, int nargs) goto error; } if (PyTuple_Check(new_base)) { - tot_extra += PyTuple_GET_SIZE(new_base) - 1; + if (!new_bases) { + /* If this is a first successful conversion, create new_bases list and + copy previously encountered bases. */ + new_bases = PyList_New(i); + for (j = 0; j < i; j++) { + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + if (!_PyList_Extend((PyListObject *)new_bases, new_base)) { + goto error; + } + Py_DECREF(Py_None); + Py_DECREF(new_base); } else { PyErr_SetString(PyExc_TypeError, @@ -84,40 +107,15 @@ update_bases(PyObject* bases, PyObject** args, int nargs) Py_DECREF(new_base); goto error; } - if (!replacements) { - replacements = PyTuple_New(nargs - 2); - } - Py_INCREF(new_base); - PyTuple_SET_ITEM(replacements, i - 2, new_base); - modified_bases = 1; } - if (!modified_bases) { + if (!new_bases) { return bases; } - new_bases = PyTuple_New(nargs - 2 + tot_extra); - ind = 0; - for (i = 2; i < nargs; i++) { - base = args[i]; - new_base = PyTuple_GET_ITEM(replacements, i - 2); - if (!new_base) { - Py_INCREF(base); - PyTuple_SET_ITEM(new_bases, ind, base); - ind++; - } - else { - for (j = 0; j < PyTuple_GET_SIZE(new_base); j++) { - new_sub_base = PyTuple_GET_ITEM(new_base, j); - Py_INCREF(new_sub_base); - PyTuple_SET_ITEM(new_bases, ind, new_sub_base); - ind++; - } - Py_DECREF(new_base); - } - } - Py_DECREF(replacements); - return new_bases; + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; error: - Py_XDECREF(replacements); + Py_XDECREF(new_bases); return NULL; } @@ -126,8 +124,7 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; - PyObject *new_bases, *old_bases = NULL; + PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *orig_bases; PyObject *cls = NULL, *cell = NULL; int isclass = 0; /* initialize to prevent gcc warning */ @@ -148,19 +145,15 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } - bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); - if (bases == NULL) + orig_bases = _PyStack_AsTupleSlice(args, nargs, 2, nargs); + if (orig_bases == NULL) return NULL; - new_bases = update_bases(bases, args, nargs); - if (new_bases == NULL) { - Py_DECREF(bases); + bases = update_bases(orig_bases); + if (bases == NULL) { + Py_DECREF(orig_bases); return NULL; } - else { - old_bases = bases; - bases = new_bases; - } if (kwnames == NULL) { meta = NULL; @@ -254,8 +247,8 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { - if (bases != old_bases) { - if (PyMapping_SetItemString(ns, "__orig_bases__", old_bases) < 0) { + if (bases != orig_bases) { + if (PyMapping_SetItemString(ns, "__orig_bases__", orig_bases) < 0) { goto error; } } @@ -297,8 +290,8 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, Py_DECREF(meta); Py_XDECREF(mkw); Py_DECREF(bases); - if (bases != old_bases) { - Py_DECREF(old_bases); + if (bases != orig_bases) { + Py_DECREF(orig_bases); } return cls; } From b7d28d839540d5bc3ff095d80b9627e6cb8ec6ff Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Dec 2017 01:24:22 +0100 Subject: [PATCH 37/38] Some fixes + performance --- Python/bltinmodule.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a8543c6f06a297..e473dd48aa43a3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -48,7 +48,7 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" static PyObject* -update_bases(PyObject* bases) +update_bases(PyObject* bases, PyObject** args, int nargs) { int i, j; PyObject *base, *meth, *new_base, *result, *new_bases = NULL; @@ -57,8 +57,8 @@ update_bases(PyObject* bases) /* We have a separate cycle to calculate replacements with the idea that in most cases we just scroll quickly though it and return original bases */ - for (i = 0; i < PyTuple_GET_SIZE(bases); i++) { - base = PyTuple_GET_ITEM(bases, i); + for (i = 0; i < nargs; i++) { + base = args[i]; if (PyType_Check(base)) { if (new_bases) { if (PyList_Append(new_bases, base) < 0) { @@ -74,6 +74,7 @@ update_bases(PyObject* bases) PyErr_Clear(); if (new_bases) { if (PyList_Append(new_bases, base) < 0) { + Py_DECREF(meth); goto error; } } @@ -90,7 +91,7 @@ update_bases(PyObject* bases) copy previously encountered bases. */ new_bases = PyList_New(i); for (j = 0; j < i; j++) { - base = PyTuple_GET_ITEM(bases, j); + base = args[j]; PyList_SET_ITEM(new_bases, j, base); Py_INCREF(base); } @@ -149,7 +150,7 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, if (orig_bases == NULL) return NULL; - bases = update_bases(orig_bases); + bases = update_bases(orig_bases, args + 2, nargs - 2); if (bases == NULL) { Py_DECREF(orig_bases); return NULL; From caf923898257a51b2c3c6575c98ae4a1e25738b5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Dec 2017 14:24:38 +0100 Subject: [PATCH 38/38] Re-organize code and fix formatting --- Python/bltinmodule.c | 47 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e473dd48aa43a3..f673a3506355d3 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -48,33 +48,33 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" static PyObject* -update_bases(PyObject* bases, PyObject** args, int nargs) +update_bases(PyObject *bases, PyObject *const *args, int nargs) { int i, j; PyObject *base, *meth, *new_base, *result, *new_bases = NULL; PyObject *stack[1] = {bases}; assert(PyTuple_Check(bases)); - /* We have a separate cycle to calculate replacements with the idea that in - most cases we just scroll quickly though it and return original bases */ for (i = 0; i < nargs; i++) { base = args[i]; if (PyType_Check(base)) { if (new_bases) { + /* If we already have made a replacement, then we append every normal base, + otherwise just skip it. */ if (PyList_Append(new_bases, base) < 0) { goto error; } } continue; } - if (!(meth = _PyObject_GetAttrId(base, &PyId___mro_entries__))) { + meth = _PyObject_GetAttrId(base, &PyId___mro_entries__); + if (!meth) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { goto error; } PyErr_Clear(); if (new_bases) { if (PyList_Append(new_bases, base) < 0) { - Py_DECREF(meth); goto error; } } @@ -85,29 +85,29 @@ update_bases(PyObject* bases, PyObject** args, int nargs) if (!new_base) { goto error; } - if (PyTuple_Check(new_base)) { - if (!new_bases) { - /* If this is a first successful conversion, create new_bases list and - copy previously encountered bases. */ - new_bases = PyList_New(i); - for (j = 0; j < i; j++) { - base = args[j]; - PyList_SET_ITEM(new_bases, j, base); - Py_INCREF(base); - } - } - if (!_PyList_Extend((PyListObject *)new_bases, new_base)) { - goto error; - } - Py_DECREF(Py_None); - Py_DECREF(new_base); - } - else { + if (!PyTuple_Check(new_base)) { PyErr_SetString(PyExc_TypeError, "__mro_entries__ must return a tuple"); Py_DECREF(new_base); goto error; } + if (!new_bases) { + /* If this is a first successful replacement, create new_bases list and + copy previously encountered bases. */ + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + base = args[j]; + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); } if (!new_bases) { return bases; @@ -115,6 +115,7 @@ update_bases(PyObject* bases, PyObject** args, int nargs) result = PyList_AsTuple(new_bases); Py_DECREF(new_bases); return result; + error: Py_XDECREF(new_bases); return NULL;