From 119edbe46e87d23d234909d9e7eeb70cd9c85c5a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2017 22:37:09 +0200 Subject: [PATCH 01/12] Implement module __getattr__ --- Objects/moduleobject.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 2be49fbda38908..8fb7cf770419f5 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -679,12 +679,27 @@ module_repr(PyModuleObject *m) static PyObject* module_getattro(PyModuleObject *m, PyObject *name) { - PyObject *attr, *mod_name; + PyObject *attr, *mod_name, *getattr; + PyObject* stack[1] = {name}; attr = PyObject_GenericGetAttr((PyObject *)m, name); - if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) + if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) { return attr; + } PyErr_Clear(); if (m->md_dict) { + _Py_IDENTIFIER(__getattr__); + getattr = _PyDict_GetItemId(m->md_dict, &PyId___getattr__); + if (getattr) { + if (!PyCallable_Check(getattr)) { + PyErr_SetString(PyExc_TypeError, + "module __getattr__ must be callable"); + return NULL; + } + return _PyObject_FastCall(getattr, stack, 1); + } + else if (PyErr_Occurred()) { + PyErr_Clear(); + } _Py_IDENTIFIER(__name__); mod_name = _PyDict_GetItemId(m->md_dict, &PyId___name__); if (mod_name) { From c2ec3c1ec400687e1390d2250e5fdf075c1a76a5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2017 23:33:27 +0200 Subject: [PATCH 02/12] Add tests --- Lib/test/bad_getattr.py | 3 +++ Lib/test/bad_getattr2.py | 4 ++++ Lib/test/good_getattr.py | 8 ++++++++ Lib/test/test_module.py | 20 ++++++++++++++++++++ 4 files changed, 35 insertions(+) create mode 100644 Lib/test/bad_getattr.py create mode 100644 Lib/test/bad_getattr2.py create mode 100644 Lib/test/good_getattr.py diff --git a/Lib/test/bad_getattr.py b/Lib/test/bad_getattr.py new file mode 100644 index 00000000000000..b4f790f3bf166e --- /dev/null +++ b/Lib/test/bad_getattr.py @@ -0,0 +1,3 @@ +x = 1 + +__getattr__ = "Surprise!" diff --git a/Lib/test/bad_getattr2.py b/Lib/test/bad_getattr2.py new file mode 100644 index 00000000000000..21fa74f0baca3e --- /dev/null +++ b/Lib/test/bad_getattr2.py @@ -0,0 +1,4 @@ +def __getattr__(): + "Bad one" + +x = 1 diff --git a/Lib/test/good_getattr.py b/Lib/test/good_getattr.py new file mode 100644 index 00000000000000..a100ae943862fa --- /dev/null +++ b/Lib/test/good_getattr.py @@ -0,0 +1,8 @@ +x = 1 + +def __getattr__(name): + if name == "yolo": + raise AttributeError("Deprecated, use whatever instead") + return f"There is {name}" + +y = 2 diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 6d0d59407ed240..d657df9a5e0e28 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -125,6 +125,26 @@ def test_weakref(self): gc_collect() self.assertIs(wr(), None) + def test_module_getattr(self): + import test.good_getattr as gga + from test.good_getattr import test + self.assertEqual(test, "There is test") + self.assertEqual(gga.x, 1) + self.assertEqual(gga.y, 2) + with self.assertRaises(AttributeError): + gga.yolo + self.assertEqual(gga.whatever, "There is whatever") + + def test_module_getattr_errors(self): + import test.bad_getattr as bga + from test import bad_getattr2 + self.assertEqual(bga.x, 1) + self.assertEqual(bad_getattr2.x, 1) + with self.assertRaises(TypeError): + bga.nope + with self.assertRaises(TypeError): + bad_getattr2.nope + def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ m = ModuleType('foo') From f657a7e9bfe5cf88b3d98d8acc4cbbe48fd261dc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 00:26:15 +0200 Subject: [PATCH 03/12] Clean-up after tests --- Lib/test/test_module.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index d657df9a5e0e28..2566603123013d 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -134,6 +134,7 @@ def test_module_getattr(self): with self.assertRaises(AttributeError): gga.yolo self.assertEqual(gga.whatever, "There is whatever") + del sys.modules['test.good_getattr'] def test_module_getattr_errors(self): import test.bad_getattr as bga @@ -144,6 +145,8 @@ def test_module_getattr_errors(self): bga.nope with self.assertRaises(TypeError): bad_getattr2.nope + del sys.modules['test.bad_getattr'] + del sys.modules['test.bad_getattr2'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ From 8aa052f6174f6307ed9c33508dbf3c9b7d90bfe9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2017 19:46:31 +0200 Subject: [PATCH 04/12] Fix test clean-up in refleak hunting mode --- Lib/test/test_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 2566603123013d..2a15f1a206cebd 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -146,7 +146,8 @@ def test_module_getattr_errors(self): with self.assertRaises(TypeError): bad_getattr2.nope del sys.modules['test.bad_getattr'] - del sys.modules['test.bad_getattr2'] + if 'test.bad_getattr2' in sys.modules: + del sys.modules['test.bad_getattr2'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ From 32d092f49b0abf60b887b047560caba73690c785 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 14 Nov 2017 21:11:56 +0100 Subject: [PATCH 05/12] Add module __dir__ (with tests) --- Lib/test/bad_getattr.py | 1 + Lib/test/bad_getattr2.py | 3 +++ Lib/test/good_getattr.py | 3 +++ Lib/test/test_module.py | 17 +++++++++++++++++ Objects/moduleobject.c | 20 ++++++++++++++++++-- 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/Lib/test/bad_getattr.py b/Lib/test/bad_getattr.py index b4f790f3bf166e..16f901b13b822a 100644 --- a/Lib/test/bad_getattr.py +++ b/Lib/test/bad_getattr.py @@ -1,3 +1,4 @@ x = 1 __getattr__ = "Surprise!" +__dir__ = "Surprise again!" diff --git a/Lib/test/bad_getattr2.py b/Lib/test/bad_getattr2.py index 21fa74f0baca3e..0a52a53b54bb02 100644 --- a/Lib/test/bad_getattr2.py +++ b/Lib/test/bad_getattr2.py @@ -2,3 +2,6 @@ def __getattr__(): "Bad one" x = 1 + +def __dir__(bad_sig): + return [] diff --git a/Lib/test/good_getattr.py b/Lib/test/good_getattr.py index a100ae943862fa..7d27de6262ad74 100644 --- a/Lib/test/good_getattr.py +++ b/Lib/test/good_getattr.py @@ -1,5 +1,8 @@ x = 1 +def __dir__(): + return ['a', 'b', 'c'] + def __getattr__(name): if name == "yolo": raise AttributeError("Deprecated, use whatever instead") diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 2a15f1a206cebd..1157ea1029b7ca 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -149,6 +149,23 @@ def test_module_getattr_errors(self): if 'test.bad_getattr2' in sys.modules: del sys.modules['test.bad_getattr2'] + def test_module_dir(self): + import test.good_getattr as gga + self.assertEqual(dir(gga), ['a', 'b', 'c']) + del sys.modules['test.good_getattr'] + + def test_module_dir_errors(self): + import test.bad_getattr as bga + from test import bad_getattr2 + with self.assertRaises(TypeError): + dir(bga) + with self.assertRaises(TypeError): + dir(bad_getattr2) + del sys.modules['test.bad_getattr'] + if 'test.bad_getattr2' in sys.modules: + del sys.modules['test.bad_getattr2'] + + def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ m = ModuleType('foo') diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8fb7cf770419f5..ea847ee7d05f4a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -748,8 +748,24 @@ module_dir(PyObject *self, PyObject *args) PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__); if (dict != NULL) { - if (PyDict_Check(dict)) - result = PyDict_Keys(dict); + if (PyDict_Check(dict)) { + PyObject *dirfunc = PyDict_GetItemString(dict, "__dir__"); + if (dirfunc) { + if (!PyCallable_Check(dirfunc)) { + PyErr_SetString(PyExc_TypeError, + "module __dir__ must be callable"); + result = NULL; + } + else { + result = _PyObject_FastCall(dirfunc, NULL, 0); + } + } + else { + if (PyErr_Occurred()) + PyErr_Clear(); + result = PyDict_Keys(dict); + } + } else { const char *name = PyModule_GetName(self); if (name) From 4fa4b1be6ca254cd0d9331f87087777d25663009 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 5 Dec 2017 21:33:58 +0100 Subject: [PATCH 06/12] Add news entry --- .../Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst new file mode 100644 index 00000000000000..370d3dc0e13bd0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst @@ -0,0 +1,2 @@ +PEP 562: Add support for module __getattr__ and __dir__. Implemented by Ivan +Levkivskyi. From d1ada1172972a26ac837598a1461e916a4134cd2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Dec 2017 01:06:33 +0100 Subject: [PATCH 07/12] Address CR --- Lib/test/bad_getattr3.py | 6 ++++++ Lib/test/test_module.py | 13 ++++++++++++- Objects/moduleobject.c | 14 +------------- 3 files changed, 19 insertions(+), 14 deletions(-) create mode 100644 Lib/test/bad_getattr3.py diff --git a/Lib/test/bad_getattr3.py b/Lib/test/bad_getattr3.py new file mode 100644 index 00000000000000..d84f542c050af3 --- /dev/null +++ b/Lib/test/bad_getattr3.py @@ -0,0 +1,6 @@ +def __getattr__(name): + if name != 'delgetattr': + raise AttributeError('Only deletion') + del globals()['__getattr__'] + return 'OK, deleted' + diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 1157ea1029b7ca..493ba9b8842b6b 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -131,7 +131,8 @@ def test_module_getattr(self): self.assertEqual(test, "There is test") self.assertEqual(gga.x, 1) self.assertEqual(gga.y, 2) - with self.assertRaises(AttributeError): + with self.assertRaisesRegex(AttributeError, + "Deprecated, use whatever instead"): gga.yolo self.assertEqual(gga.whatever, "There is whatever") del sys.modules['test.good_getattr'] @@ -165,6 +166,16 @@ def test_module_dir_errors(self): if 'test.bad_getattr2' in sys.modules: del sys.modules['test.bad_getattr2'] + def test_module_getattr_tricky(self): + from test import bad_getattr3 + with self.assertRaises(AttributeError): + bad_getattr3.one + self.assertEqual(bad_getattr3.delgetattr, "OK, deleted") + with self.assertRaises(AttributeError): + # this will not work second time + bad_getattr3.delgetattr + if 'test.bad_getattr3' in sys.modules: + del sys.modules['test.bad_getattr3'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8083062eb31f57..7d8dad6af6e7ca 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -697,9 +697,6 @@ module_getattro(PyModuleObject *m, PyObject *name) } return _PyObject_FastCall(getattr, stack, 1); } - else if (PyErr_Occurred()) { - PyErr_Clear(); - } _Py_IDENTIFIER(__name__); mod_name = _PyDict_GetItemId(m->md_dict, &PyId___name__); if (mod_name && PyUnicode_Check(mod_name)) { @@ -748,18 +745,9 @@ module_dir(PyObject *self, PyObject *args) if (PyDict_Check(dict)) { PyObject *dirfunc = PyDict_GetItemString(dict, "__dir__"); if (dirfunc) { - if (!PyCallable_Check(dirfunc)) { - PyErr_SetString(PyExc_TypeError, - "module __dir__ must be callable"); - result = NULL; - } - else { - result = _PyObject_FastCall(dirfunc, NULL, 0); - } + result = _PyObject_CallNoArg(dirfunc); } else { - if (PyErr_Occurred()) - PyErr_Clear(); result = PyDict_Keys(dict); } } From a9f7d4aa56b2f0d8682241238e2363f637c16100 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Dec 2017 01:18:15 +0100 Subject: [PATCH 08/12] Fix whitespace --- Lib/test/bad_getattr3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/bad_getattr3.py b/Lib/test/bad_getattr3.py index d84f542c050af3..22793f554d776b 100644 --- a/Lib/test/bad_getattr3.py +++ b/Lib/test/bad_getattr3.py @@ -3,4 +3,3 @@ def __getattr__(name): raise AttributeError('Only deletion') del globals()['__getattr__'] return 'OK, deleted' - From f93b16812ac1e7ff85c70ab8f0c19c7d05cabebf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Dec 2017 01:52:17 +0100 Subject: [PATCH 09/12] Simplify the tricky test --- Lib/test/bad_getattr3.py | 4 ++-- Lib/test/test_module.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/test/bad_getattr3.py b/Lib/test/bad_getattr3.py index 22793f554d776b..0d5f9266c7183d 100644 --- a/Lib/test/bad_getattr3.py +++ b/Lib/test/bad_getattr3.py @@ -1,5 +1,5 @@ def __getattr__(name): if name != 'delgetattr': - raise AttributeError('Only deletion') + raise AttributeError del globals()['__getattr__'] - return 'OK, deleted' + raise AttributeError diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 493ba9b8842b6b..efe9a8ed5e5717 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -168,11 +168,10 @@ def test_module_dir_errors(self): def test_module_getattr_tricky(self): from test import bad_getattr3 + # these lookups should not crash with self.assertRaises(AttributeError): bad_getattr3.one - self.assertEqual(bad_getattr3.delgetattr, "OK, deleted") with self.assertRaises(AttributeError): - # this will not work second time bad_getattr3.delgetattr if 'test.bad_getattr3' in sys.modules: del sys.modules['test.bad_getattr3'] From c720cfd40454ae8daa6d7a804c0e294e79c7e621 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Dec 2017 18:00:51 +0100 Subject: [PATCH 10/12] Fix rst formatting --- .../Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst index 370d3dc0e13bd0..5cde073dd63b6c 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-05-21-33-47.bpo-32225.ucKjvw.rst @@ -1,2 +1,2 @@ -PEP 562: Add support for module __getattr__ and __dir__. Implemented by Ivan +PEP 562: Add support for module ``__getattr__`` and ``__dir__``. Implemented by Ivan Levkivskyi. From fd2f02f22003b5d2cdefbf5d3452f660e298b67a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Dec 2017 21:51:59 +0100 Subject: [PATCH 11/12] Add docs; minor comment by Serhiy --- Doc/reference/datamodel.rst | 40 +++++++++++++++++++++++++++++++++++++ Objects/moduleobject.c | 7 +------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 153b58b4fbfbab..e31f8cf3a2d2e5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1512,6 +1512,46 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. returned. :func:`dir` converts the returned sequence to a list and sorts it. +Customizing module attribute access +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Special names ``__getattr__`` and ``__dir__`` can be also used to customize +access to module attributes. The ``__getattr__`` function at the module level +should accept one argument which is the name of an attribute and return the +computed value or raise an :exc:`AttributeError`. If an attribute is +not found on a module object through the normal lookup, i.e. +:meth:`object.__getattribute__`, then ``__getattr__`` is searched in +the module ``__dict__`` before raising an :exc:`AttributeError`. If found, +it is called with the attribute name and the result is returned. + +The ``__dir__`` function should accept no arguments, and return a list of +strings that represents the names accessible on module. If present, this +function overrides the standard :func:`dir` search on a module. + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the ``__class__`` attribute of +a module object to a subclass of :class:`types.ModuleType`. For example:: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + setattr(self, attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +.. note:: + Defining module ``__getattr__`` and setting module ``__class__`` only + affect lookups made using the attribute access syntax -- directly accessing + the module globals (whether by code within the module, or via a reference + to the module's globals dictionary) is unaffected. + + .. _descriptors: Implementing Descriptors diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 7d8dad6af6e7ca..d6cde4024336c5 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -680,7 +680,6 @@ static PyObject* module_getattro(PyModuleObject *m, PyObject *name) { PyObject *attr, *mod_name, *getattr; - PyObject* stack[1] = {name}; attr = PyObject_GenericGetAttr((PyObject *)m, name); if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) { return attr; @@ -690,11 +689,7 @@ module_getattro(PyModuleObject *m, PyObject *name) _Py_IDENTIFIER(__getattr__); getattr = _PyDict_GetItemId(m->md_dict, &PyId___getattr__); if (getattr) { - if (!PyCallable_Check(getattr)) { - PyErr_SetString(PyExc_TypeError, - "module __getattr__ must be callable"); - return NULL; - } + PyObject* stack[1] = {name}; return _PyObject_FastCall(getattr, stack, 1); } _Py_IDENTIFIER(__name__); From e4b53d9c22e5adc01200f34ce091d8531bc1ca9a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Dec 2017 00:16:10 +0100 Subject: [PATCH 12/12] Add what's new entry, and index items --- Doc/reference/datamodel.rst | 5 +++++ Doc/whatsnew/3.7.rst | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index e31f8cf3a2d2e5..790339cd47b286 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1515,6 +1515,11 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. Customizing module attribute access ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. index:: + single: __getattr__ (module attribute) + single: __dir__ (module attribute) + single: __class__ (module attribute) + Special names ``__getattr__`` and ``__dir__`` can be also used to customize access to module attributes. The ``__getattr__`` function at the module level should accept one argument which is the name of an attribute and return the diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9363730bf1fd64..c04aaea0164f21 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -159,6 +159,24 @@ effort will be made to add such support. PEP written by Erik M. Bray; implementation by Masayuki Yamamoto. +PEP 562: Customization of access to module attributes +----------------------------------------------------- + +It is sometimes convenient to customize or otherwise have control over access +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning ``__class__`` of a module object to +a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` +item with a custom wrapper instance. This procedure is now simplified by +recognizing ``__getattr__`` defined directly in a module that would act like +a normal ``__getattr__`` method, except that it will be defined on module +*instances*. + +.. seealso:: + + :pep:`562` -- Module ``__getattr__`` and ``__dir__`` + PEP written and implemented by Ivan Levkivskyi + + PEP 564: Add new time functions with nanosecond resolution ----------------------------------------------------------