Skip to content

Draft: gh-123471: Make concurrent iteration over itertools.pairwise safe under free-threading #125417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
eendebakpt wants to merge 5 commits into from
Closed

Draft: gh-123471: Make concurrent iteration over itertools.pairwise safe under free-threading #125417

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d7cbe16
Make pairwise_next safe for free-threading
eendebakpt Oct 13, 2024
c7ffc4f
refactor
eendebakpt Oct 14, 2024
fdc7b01
📜🤖 Added by blurb_it.
blurb-it[bot] Oct 14, 2024
e111e10
Merge branch 'main' into pairwise_v2
eendebakpt Oct 14, 2024
d6c48fc
Merge branch 'main' into pairwise_v2
eendebakpt Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor
  • Loading branch information
@eendebakpt
eendebakpt committed Oct 14, 2024
commit c7ffc4f7b3c064ded698a5852445be289c75ecaa
42 changes: 42 additions & 0 deletions Lib/test/test_free_threading/test_itertools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import unittest
from threading import Thread

from test.support import threading_helper

from itertools import zip_longest

class PairwiseThreading(unittest.TestCase):
@staticmethod
def work(enum):
while True:
try:
next(enum)
except StopIteration:
break

@threading_helper.reap_threads
@threading_helper.requires_working_threading()
def test_zip_longest(self):
number_of_threads = 8
number_of_iterations = 40
n = 200
enum = zip_longest(range(n), range(2*n))
for _ in range(number_of_iterations):
worker_threads = []
for ii in range(number_of_threads):
worker_threads.append(
Thread(
target=self.work,
args=[
enum,
],
)
)
for t in worker_threads:
t.start()
for t in worker_threads:
t.join()


if __name__ == "__main__":
unittest.main()
17 changes: 15 additions & 2 deletions Modules/itertoolsmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,11 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg)
static PyObject *
pairwise_next(pairwiseobject *po)
{
#ifdef Py_GIL_DISABLED
PyObject *it = Py_XNewRef(po->it);
#else
PyObject *it = po->it;
#endif
if (it == NULL) {
return NULL;
}
Expand All @@ -339,16 +343,21 @@ pairwise_next(pairwiseobject *po)
old = (*Py_TYPE(it)->tp_iternext)(it);
if (old == NULL) {
Py_CLEAR(po->it);
#ifdef Py_GIL_DISABLED
Py_DECREF(it);
#endif
return NULL;
}
Py_XSETREF(po->old, Py_NewRef(old));
if (po->it == NULL) {
// for re-entrant calls to pairwise next. the actual behavior is not important and this does not avoid any bugs (or does it?)
// the reason for having it is to make the behaviour equal to the python implementation behavior
// gh-109786: special case for re-entrant calls to pairwise next. the actual behavior is not
// important and this does not avoid any bugs (or does it?)
// the reason for having it is to make the behaviour equal to the python implementation
Py_CLEAR(po->old);
Py_DECREF(old);
#ifdef Py_GIL_DISABLED
Py_DECREF(it);
#endif
return NULL;
}
}
Expand All @@ -357,7 +366,9 @@ pairwise_next(pairwiseobject *po)
if (new == NULL) {
Py_CLEAR(po->it);
Py_CLEAR(po->old);
#ifdef Py_GIL_DISABLED
Py_DECREF(it);
#endif
Py_DECREF(old);
return NULL;
}
Expand Down Expand Up @@ -387,7 +398,9 @@ pairwise_next(pairwiseobject *po)

Py_XSETREF(po->old, new);
Py_DECREF(old); // instead of the decref here we could borrow the reference above
#ifdef Py_GIL_DISABLED
Py_DECREF(it);
#endif
return result;
}

Expand Down