bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when …

…two threads attempt to commit the last pending removal (GH-27921) Fixes: Traceback (most recent call last): File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in <module> sys.exit(main()) File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main test_all_tasks_threading() File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading results.append(f.result()) File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result return self.__get_result() File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result raise self._exception File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run result = self.fn(*self.args, **self.kwargs) File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run _cancel_all_tasks(loop) File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks to_cancel = tasks.all_tasks(loop) File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks tasks = list(_all_tasks) File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__ with _IterationGuard(self): File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__ w._commit_removals() File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals discard(l.pop()) IndexError: pop from empty list Also fixes: Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: <weakref at 0x00007fe76e8d8180; dead> Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: <weakref at 0x00007fe76e8d81a0; dead> Exception ignored in: weakref callback <function WeakKeyDictionary.__init__.<locals>.remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: <weakref at 0x000056548f1e24a0; dead> See: https://github.com/agronholm/anyio/issues/362GH-issuecomment-904424310 See also: https://bugs.python.org/issue29519 Co-authored-by: Łukasz Langa <lukasz@langa.pl> (cherry picked from commit 206b21e) Co-authored-by: Thomas Grainger <tagrain@gmail.com>
python · ambv · Aug 28, 2021 · Aug 28, 2021 · Aug 28, 2021 · e978d6afc225a750c80f28fbd82af63eecd4432b
commit e978d6afc225a750c80f28fbd82af63eecd4432b
diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py
@@ -51,10 +51,14 @@ def _remove(item, selfref=ref(self)):
             self.update(data)
 
     def _commit_removals(self):
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         discard = self.data.discard
-        while l:
-            discard(l.pop())
+        while True:
+            try:
+                item = pop()
+            except IndexError:
+                return
+            discard(item)
 
     def __iter__(self):
         with _IterationGuard(self):

diff --git a/Lib/weakref.py b/Lib/weakref.py
@@ -119,14 +119,17 @@ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
         self.data = {}
         self.update(other, **kw)
 
-    def _commit_removals(self):
-        l = self._pending_removals
+    def _commit_removals(self, _atomic_removal=_remove_dead_weakref):
+        pop = self._pending_removals.pop
         d = self.data
         # We shouldn't encounter any KeyError, because this method should
         # always be called *before* mutating the dict.
-        while l:
-            key = l.pop()
-            _remove_dead_weakref(d, key)
+        while True:
+            try:
+                key = pop()
+            except IndexError:
+                return
+            _atomic_removal(d, key)
 
     def __getitem__(self, key):
         if self._pending_removals:
@@ -370,7 +373,10 @@ def remove(k, selfref=ref(self)):
                 if self._iterating:
                     self._pending_removals.append(k)
                 else:
-                    del self.data[k]
+                    try:
+                        del self.data[k]
+                    except KeyError:
+                        pass
         self._remove = remove
         # A list of dead weakrefs (keys to be removed)
         self._pending_removals = []
@@ -384,11 +390,16 @@ def _commit_removals(self):
         # because a dead weakref never compares equal to a live weakref,
         # even if they happened to refer to equal objects.
         # However, it means keys may already have been removed.
-        l = self._pending_removals
+        pop = self._pending_removals.pop
         d = self.data
-        while l:
+        while True:
+            try:
+                key = pop()
+            except IndexError:
+                return
+
             try:
-                del d[l.pop()]
+                del d[key]
             except KeyError:
                 pass
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst
@@ -0,0 +1 @@
+Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run