Skip to content

gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__ (alternative2) #150540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dolfinus wants to merge 4 commits into
base: main
Choose a base branch
from
Open

gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__ (alternative2) #150540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
38162ca
gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__
dolfinus May 27, 2026
091e2b7
📜🤖 Added by blurb_it.
blurb-it[bot] May 27, 2026
dc63fcc
gh-92810: Fix blurb message
dolfinus May 27, 2026
49e8b49
gh-92810: Fix compiler warning
dolfinus May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions Doc/whatsnew/3.16.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,31 @@ New modules
Improved modules
================

abc
---

* Reduce memory usage of :func:`issubclass` checks for :class:`abc.ABCMeta` subclasses.

:class:`abc.ABCMeta` subclasses can trigger downstream checks:

``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)``
(nothing found) -> ``issubclass(Some, Class.__subclasses__)`` -> ``issubclass(Some, SubClass)``
-> ``issubclass(Some, SubClass.__subclasses__)`` -> ...

Due to caching of ``issubclass`` result within each ABC class,
this could lead to memory bloat in large class trees, e.g. thousands of subclasses.

Now :meth:`!abc.ABCMeta.register` recursively calls ``Parent.register(subclass)``,
``Top.register(subclass)`` and so on for all base classes, so downstream checks are not needed.
Also :func:`issubclass` does not recursively checks all ``__subclasses__`` of current ABC class.

This reduces both the number of checks, and the RAM usage by internal caches:

``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)``
(nothing found, stops here)

(Contributed by Maxim Martynov in :gh:`92810`.)

gzip
----

Expand Down
21 changes: 16 additions & 5 deletions Lib/_py_abc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,20 @@ def register(cls, subclass):
if issubclass(cls, subclass):
# This would create a cycle, which is bad for the algorithm below
raise RuntimeError("Refusing to create an inheritance cycle")
# Add registry entry
cls._abc_registry.add(subclass)
ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache
# Recursively register the subclass in all ABC bases,
# to avoid recursive lookups down the class tree.
# >>> class Ancestor1(ABC): pass
# >>> class Ancestor2(Ancestor1): pass
# >>> class Other: pass
# >>> Ancestor2.register(Other) # calls Ancestor1.register(Other)
# >>> issubclass(Other, Ancestor2) is True
# >>> issubclass(Other, Ancestor1) is True # already in registry
for base in cls.__bases__:
if hasattr(base, "_abc_registry"):
base.register(subclass)
return subclass

def _dump_registry(cls, file=None):
Expand Down Expand Up @@ -132,16 +144,15 @@ def __subclasscheck__(cls, subclass):
if cls in getattr(subclass, '__mro__', ()):
cls._abc_cache.add(subclass)
return True
# Fast path: check subclass is in weakset directly.
if subclass in cls._abc_registry:
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a registered class (recursive)
for rcls in cls._abc_registry:
if issubclass(subclass, rcls):
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a subclass (recursive)
for scls in cls.__subclasses__():
if issubclass(subclass, scls):
cls._abc_cache.add(subclass)
return True
# No dice; update negative cache
cls._abc_negative_cache.add(subclass)
return False
225 changes: 167 additions & 58 deletions Lib/test/test_abc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ def foo(): return 4


class TestABC(unittest.TestCase):
def check_isinstance(self, obj, target_class):
self.assertIsInstance(obj, target_class)
self.assertIsInstance(obj, (target_class,))
self.assertIsInstance(obj, target_class | int)

def check_not_isinstance(self, obj, target_class):
self.assertNotIsInstance(obj, target_class)
self.assertNotIsInstance(obj, (target_class,))
self.assertNotIsInstance(obj, target_class | int)

def check_issubclass(self, klass, target_class):
self.assertIsSubclass(klass, target_class)
self.assertIsSubclass(klass, (target_class,))
self.assertIsSubclass(klass, target_class | int)

def check_not_issubclass(self, klass, target_class):
self.assertNotIsSubclass(klass, target_class)
self.assertNotIsSubclass(klass, (target_class,))
self.assertNotIsSubclass(klass, target_class | int)

def test_ABC_helper(self):
# create an ABC using the helper class and perform basic checks
Expand Down Expand Up @@ -270,29 +289,75 @@ def x(self):
class C(metaclass=meta):
pass

def test_isinstance_direct_inheritance(self):
class A(metaclass=abc_ABCMeta):
pass
class B(A):
pass
class C(A):
pass

a = A()
b = B()
c = C()
# trigger caching
for _ in range(2):
self.check_isinstance(a, A)
self.check_not_isinstance(a, B)
self.check_not_isinstance(a, C)

self.check_isinstance(b, B)
self.check_isinstance(b, A)
self.check_not_isinstance(b, C)

self.check_isinstance(c, C)
self.check_isinstance(c, A)
self.check_not_isinstance(c, B)

self.check_issubclass(B, A)
self.check_issubclass(C, A)
self.check_not_issubclass(B, C)
self.check_not_issubclass(C, B)
self.check_not_issubclass(A, B)
self.check_not_issubclass(A, C)

def test_registration_basics(self):
class A(metaclass=abc_ABCMeta):
pass
class B(object):
pass

a = A()
b = B()
self.assertNotIsSubclass(B, A)
self.assertNotIsSubclass(B, (A,))
self.assertNotIsInstance(b, A)
self.assertNotIsInstance(b, (A,))
# trigger caching
for _ in range(2):
self.check_not_issubclass(B, A)
self.check_not_isinstance(b, A)

self.check_not_issubclass(A, B)
self.check_not_isinstance(a, B)

B1 = A.register(B)
self.assertIsSubclass(B, A)
self.assertIsSubclass(B, (A,))
self.assertIsInstance(b, A)
self.assertIsInstance(b, (A,))
self.assertIs(B1, B)
# trigger caching
for _ in range(2):
self.check_issubclass(B, A)
self.check_isinstance(b, A)
self.assertIs(B1, B)

self.check_not_issubclass(A, B)
self.check_not_isinstance(a, B)

class C(B):
pass

c = C()
self.assertIsSubclass(C, A)
self.assertIsSubclass(C, (A,))
self.assertIsInstance(c, A)
self.assertIsInstance(c, (A,))
# trigger caching
for _ in range(2):
self.check_issubclass(C, A)
self.check_isinstance(c, A)

self.check_not_issubclass(A, C)
self.check_not_isinstance(a, C)

def test_register_as_class_deco(self):
class A(metaclass=abc_ABCMeta):
Expand Down Expand Up @@ -377,39 +442,49 @@ class A(metaclass=abc_ABCMeta):
pass
self.assertIsSubclass(A, A)
self.assertIsSubclass(A, (A,))

class B(metaclass=abc_ABCMeta):
pass
self.assertNotIsSubclass(A, B)
self.assertNotIsSubclass(A, (B,))
self.assertNotIsSubclass(B, A)
self.assertNotIsSubclass(B, (A,))

class C(metaclass=abc_ABCMeta):
pass
A.register(B)
class B1(B):
pass
self.assertIsSubclass(B1, A)
self.assertIsSubclass(B1, (A,))
# trigger caching
for _ in range(2):
self.assertIsSubclass(B1, A)
self.assertIsSubclass(B1, (A,))

class C1(C):
pass
B1.register(C1)
self.assertNotIsSubclass(C, B)
self.assertNotIsSubclass(C, (B,))
self.assertNotIsSubclass(C, B1)
self.assertNotIsSubclass(C, (B1,))
self.assertIsSubclass(C1, A)
self.assertIsSubclass(C1, (A,))
self.assertIsSubclass(C1, B)
self.assertIsSubclass(C1, (B,))
self.assertIsSubclass(C1, B1)
self.assertIsSubclass(C1, (B1,))
# trigger caching
for _ in range(2):
self.assertNotIsSubclass(C, B)
self.assertNotIsSubclass(C, (B,))
self.assertNotIsSubclass(C, B1)
self.assertNotIsSubclass(C, (B1,))
self.assertIsSubclass(C1, A)
self.assertIsSubclass(C1, (A,))
self.assertIsSubclass(C1, B)
self.assertIsSubclass(C1, (B,))
self.assertIsSubclass(C1, B1)
self.assertIsSubclass(C1, (B1,))

C1.register(int)
class MyInt(int):
pass
self.assertIsSubclass(MyInt, A)
self.assertIsSubclass(MyInt, (A,))
self.assertIsInstance(42, A)
self.assertIsInstance(42, (A,))
# trigger caching
for _ in range(2):
self.assertIsSubclass(MyInt, A)
self.assertIsSubclass(MyInt, (A,))
self.assertIsInstance(42, A)
self.assertIsInstance(42, (A,))

def test_issubclass_bad_arguments(self):
class A(metaclass=abc_ABCMeta):
Expand All @@ -429,39 +504,54 @@ class C:
with self.assertRaises(TypeError):
issubclass(C(), A)

# bpo-34441: Check that issubclass() doesn't crash on bogus
# classes.
bogus_subclasses = [
None,
lambda x: [],
lambda: 42,
lambda: [42],
]

for i, func in enumerate(bogus_subclasses):
class S(metaclass=abc_ABCMeta):
__subclasses__ = func

with self.subTest(i=i):
with self.assertRaises(TypeError):
issubclass(int, S)

# Also check that issubclass() propagates exceptions raised by
# __subclasses__.
class CustomError(Exception): ...
exc_msg = "exception from __subclasses__"
def test_issubclass_bad_class(self):
class A(metaclass=abc.ABCMeta):
pass

def raise_exc():
raise CustomError(exc_msg)
A._abc_impl = 1
error_msg = "_abc_impl is set to a wrong type"
with self.assertRaisesRegex(TypeError, error_msg):
issubclass(A, A)

class S(metaclass=abc_ABCMeta):
__subclasses__ = raise_exc
class B(metaclass=_py_abc.ABCMeta):
pass

with self.assertRaisesRegex(CustomError, exc_msg):
issubclass(int, S)
B._abc_cache = 1
error_msg = "argument of type 'int' is not a container or iterable"
with self.assertRaisesRegex(TypeError, error_msg):
issubclass(B, B)

class C(metaclass=_py_abc.ABCMeta):
pass

C._abc_negative_cache = 1
with self.assertRaisesRegex(TypeError, error_msg):
issubclass(C, C)

def test_custom_subclasses_are_ignored(self):
class A: pass
class B: pass

class Parent1(metaclass=abc_ABCMeta):
@classmethod
def __subclasses__(cls):
return [A, B]

class Parent2(metaclass=abc_ABCMeta):
__subclasses__ = lambda: [A, B]

self.assertNotIsInstance(A(), Parent1)
self.assertNotIsInstance(B(), Parent1)
self.assertNotIsSubclass(A, Parent1)
self.assertNotIsSubclass(B, Parent1)

self.assertNotIsInstance(A(), Parent2)
self.assertNotIsInstance(B(), Parent2)
self.assertNotIsSubclass(A, Parent2)
self.assertNotIsSubclass(B, Parent2)

def test_subclasshook(self):
class A(metaclass=abc.ABCMeta):
class A(metaclass=abc_ABCMeta):
@classmethod
def __subclasshook__(cls, C):
if cls is A:
Expand All @@ -478,6 +568,26 @@ class C:
self.assertNotIsSubclass(C, A)
self.assertNotIsSubclass(C, (A,))

def test_subclasshook_exception(self):
# Check that issubclass() propagates exceptions raised by
# __subclasshook__.
class CustomError(Exception): ...
exc_msg = "exception from __subclasshook__"
class A(metaclass=abc_ABCMeta):
@classmethod
def __subclasshook__(cls, C):
raise CustomError(exc_msg)
with self.assertRaisesRegex(CustomError, exc_msg):
issubclass(A, A)
class B(A):
pass
with self.assertRaisesRegex(CustomError, exc_msg):
issubclass(B, A)
class C:
pass
with self.assertRaisesRegex(CustomError, exc_msg):
issubclass(C, A)

def test_all_new_methods_are_called(self):
class A(metaclass=abc_ABCMeta):
pass
Expand Down Expand Up @@ -522,7 +632,6 @@ def foo(self):
self.assertEqual(A.__abstractmethods__, set())
A()


def test_update_new_abstractmethods(self):
class A(metaclass=abc_ABCMeta):
@abc.abstractmethod
Expand Down
Loading
Loading