Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
5c69d38
Make ThreadPoolExecutor extensible.
ericsnowcurrently Sep 25, 2024
01789be
Add InterpreterPoolExecutor.
ericsnowcurrently Sep 25, 2024
6def4be
Clean up the interpreter if initialize() fails.
ericsnowcurrently Sep 27, 2024
84993a5
Add a missing import.
ericsnowcurrently Sep 27, 2024
c540cf0
Fix some typos.
ericsnowcurrently Sep 27, 2024
45d584d
Add more tests.
ericsnowcurrently Sep 27, 2024
c90c016
Add docs.
ericsnowcurrently Sep 27, 2024
1cb4657
Add a NEwS entry.
ericsnowcurrently Sep 27, 2024
4dc0989
Fix the last test.
ericsnowcurrently Sep 27, 2024
57b2db6
Add more tests.
ericsnowcurrently Sep 27, 2024
75e11d2
Simplify ExecutionFailed.
ericsnowcurrently Sep 30, 2024
69c2b8e
Fix the signature of resolve_task().
ericsnowcurrently Sep 30, 2024
f03c314
Capture any uncaught exception.
ericsnowcurrently Sep 30, 2024
4806d9f
Add TODO comments.
ericsnowcurrently Sep 30, 2024
efc0395
Docs fixes.
ericsnowcurrently Sep 30, 2024
a29aee3
Automatically apply textwrap.dedent() to scripts.
ericsnowcurrently Sep 30, 2024
8bab457
Fix the WASI build.
ericsnowcurrently Sep 30, 2024
cd29914
wasi
ericsnowcurrently Oct 1, 2024
0287f3b
Ignore race in test.
ericsnowcurrently Oct 1, 2024
80cd7b1
Add BrokenInterpreterPool.
ericsnowcurrently Oct 8, 2024
f8d4273
Tweak the docs.
ericsnowcurrently Oct 8, 2024
3a8bfce
Clarify the InterpreterPoolExecutor docs.
ericsnowcurrently Oct 8, 2024
af6c27a
Catch all exceptions.
ericsnowcurrently Oct 8, 2024
8c0a405
Factor out exception serialization helpers.
ericsnowcurrently Oct 8, 2024
1ae7ca2
Set the ExecutionFailed error as __cause__.
ericsnowcurrently Oct 8, 2024
d24e85d
Drop the exception serialization helpers.
ericsnowcurrently Oct 8, 2024
05a03ad
Always finalize if there is an error in initialize().
ericsnowcurrently Oct 8, 2024
f150931
Explicitly note the problem with functions defined in __main__.
ericsnowcurrently Oct 8, 2024
97d0292
Handle the case where interpreters.queues doesn't exist.
ericsnowcurrently Oct 8, 2024
baf0504
Merge branch 'main' into interpreter-pool-executor
ericsnowcurrently Oct 15, 2024
5c3a327
Add a What's New entry about InterpreterPoolExecutor.
ericsnowcurrently Oct 15, 2024
a2032a8
Fix a typo.
ericsnowcurrently Oct 15, 2024
54119b8
Fix the documented signature.
ericsnowcurrently Oct 15, 2024
744dca7
Test and document asyncio support.
ericsnowcurrently Oct 15, 2024
f61d62d
Apply suggestions from code review
ericsnowcurrently Oct 16, 2024
ee65bb2
Expand the docs.
ericsnowcurrently Oct 16, 2024
a7f5c50
For now, drop support for scripts.
ericsnowcurrently Oct 16, 2024
b148e09
Fix a TODO comment.
ericsnowcurrently Oct 16, 2024
e365ae7
Fix the docs.
ericsnowcurrently Oct 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add more tests.
  • Loading branch information
ericsnowcurrently committed Sep 27, 2024
commit 45d584d43408772f0fe0cb00231161830c97caac
140 changes: 111 additions & 29 deletions Lib/test/test_concurrent_futures/test_interpreter_pool.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import contextlib
import os
import sys
import unittest
from concurrent.futures.interpreter import ExecutionFailed
from test import support
Expand All @@ -7,11 +10,119 @@
from .util import BaseTestCase, InterpreterPoolMixin, setup_module


def write_msg(fd, msg):
os.write(fd, msg + b'\0')


def read_msg(fd):
msg = b''
while ch := os.read(fd, 1):
if ch == b'\0':
return msg
msg += ch


def get_current_name():
return __name__


class InterpreterPoolExecutorTest(InterpreterPoolMixin, ExecutorTest, BaseTestCase):

def pipe(self):
r, w = os.pipe()
self.addCleanup(lambda: os.close(r))
self.addCleanup(lambda: os.close(w))
return r, w

def assertTaskRaises(self, exctype):
return self.assertRaisesRegex(ExecutionFailed, exctype.__name__)

def test_init_func(self):
msg = b'step: init'
r, w = self.pipe()
os.write(w, b'\0')

executor = self.executor_type(
initializer=write_msg, initargs=(w, msg))
before = os.read(r, 100)
executor.submit(mul, 10, 10)
after = read_msg(r)

self.assertEqual(before, b'\0')
self.assertEqual(after, msg)

def test_init_script(self):
msg1 = b'step: init'
msg2 = b'step: run'
r, w = self.pipe()
initscript = f"""if True:
import os
msg = {msg2!r}
os.write({w}, {msg1!r} + b'\\0')
"""
script = f"""if True:
os.write({w}, msg + b'\\0')
"""
os.write(w, b'\0')

executor = self.executor_type(initializer=initscript)
before_init = os.read(r, 100)
fut = executor.submit(script)
after_init = read_msg(r)
write_msg(w, b'')
before_run = read_msg(r)
fut.result()
after_run = read_msg(r)

self.assertEqual(before_init, b'\0')
self.assertEqual(after_init, msg1)
self.assertEqual(before_run, b'')
self.assertEqual(after_run, msg2)

def test_init_script_args(self):
with self.assertRaises(ValueError):
self.executor_type(initializer='pass', initargs=('spam',))

def test_init_shared(self):
msg = b'eggs'
r, w = self.pipe()
script = f"""if True:
import os
os.write({w}, spam + b'\\0')
"""

executor = self.executor_type(shared={'spam': msg})
fut = executor.submit(script)
fut.result()
after = read_msg(r)

self.assertEqual(after, msg)

def test_submit_script(self):
msg = b'spam'
r, w = self.pipe()
script = f"""if True:
import os
os.write({w}, __name__.encode('utf-8') + b'\\0')
"""
executor = self.executor_type()

fut = executor.submit(script)
res = fut.result()
after = read_msg(r)

self.assertEqual(after, b'__main__')
self.assertIs(res, None)

def test_submit_func_globals(self):
raise NotImplementedError
executor = self.executor_type()
fut = executor.submit(get_current_name)
name = fut.result()

self.assertEqual(name, '__main__')
self.assertNotEqual(name, __name__)

def test_saturation(self):
blocker = queues.create()
executor = self.executor_type(4, shared=dict(blocker=blocker))
Expand All @@ -32,35 +143,6 @@ def test_idle_thread_reuse(self):
self.assertEqual(len(executor._threads), 1)
executor.shutdown(wait=True)

# def test_executor_map_current_future_cancel(self):
# blocker = queues.create()
# log = queues.create()
#
# script = """if True:
# def log_n_wait({ident}):
# blocker(f"ident {ident} started")
# try:
# stop_event.wait()
# finally:
# log.append(f"ident {ident} stopped")
# """
#
# with self.executor_type(max_workers=1) as pool:
# # submit work to saturate the pool
# fut = pool.submit(script.format(ident="first"))
# gen = pool.map(log_n_wait, ["second", "third"], timeout=0)
# try:
# with self.assertRaises(TimeoutError):
# next(gen)
# finally:
# gen.close()
# blocker.put
# stop_event.set()
# fut.result()
# # ident='second' is cancelled as a result of raising a TimeoutError
# # ident='third' is cancelled because it remained in the collection of futures
# self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"])


def setUpModule():
setup_module()
Expand Down