Skip to content

bpo-30439: Add some helpful low-level functions for subinterpreters. #1802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ericsnowcurrently wants to merge 28 commits into from
Closed

bpo-30439: Add some helpful low-level functions for subinterpreters. #1802

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
8eb5adf
Add the _interpreters module to the stdlib.
ericsnowcurrently Dec 14, 2016
457567f
Add create() and destroy().
ericsnowcurrently Dec 29, 2016
e6af9f3
Finish nearly all the create/destroy tests.
ericsnowcurrently Jan 1, 2017
335967e
Add run_string().
ericsnowcurrently Dec 29, 2016
d2df973
Get tricky tests working.
ericsnowcurrently Jan 2, 2017
b70a496
Add a test for a still running interpreter when main exits.
ericsnowcurrently Jan 2, 2017
6f2d28f
Add run_string_unrestricted().
ericsnowcurrently Jan 4, 2017
80a931b
Exit out of the child process.
ericsnowcurrently Jan 4, 2017
6aa7d9c
Resolve several TODOs.
ericsnowcurrently Jan 4, 2017
083e13c
Set up the execution namespace before switching threads.
ericsnowcurrently Jan 4, 2017
ce081a7
Run in a copy of __main__.
ericsnowcurrently Jan 4, 2017
ec05cf5
Close stdin and stdout after the proc finishes.
ericsnowcurrently Jan 4, 2017
fe90466
Clean up a test.
ericsnowcurrently Jan 4, 2017
9082017
Chain exceptions during cleanup.
ericsnowcurrently Jan 4, 2017
21865d4
Finish the module docs.
ericsnowcurrently Jan 4, 2017
0342a4f
Fix docs.
ericsnowcurrently May 23, 2017
e9d9b04
Fix includes.
ericsnowcurrently Dec 5, 2017
722ae94
Add _interpreters.is_shareable().
ericsnowcurrently Nov 28, 2017
061ae13
Add _PyObject_CheckShareable().
ericsnowcurrently Dec 4, 2017
9a365ec
Add _PyCrossInterpreterData.
ericsnowcurrently Dec 4, 2017
8f299d4
Use the shared data in run() safely.
ericsnowcurrently Dec 7, 2017
92029a1
Do not use a copy of the __main__ ns.
ericsnowcurrently Dec 7, 2017
52c9c2f
Never return the execution namespace.
ericsnowcurrently Dec 8, 2017
a797883
Group sharing-related code.
ericsnowcurrently Dec 8, 2017
777838a
Fix a refcount.
ericsnowcurrently Dec 8, 2017
ab8f175
Add get_current() and enumerate().
ericsnowcurrently Dec 29, 2016
c50c51c
Add is_running().
ericsnowcurrently Dec 29, 2016
271d20d
Add get_main().
ericsnowcurrently Jan 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Finish nearly all the create/destroy tests.
  • Loading branch information
@ericsnowcurrently
ericsnowcurrently committed Dec 5, 2017
commit e6af9f31212a0ae24112ccbb357a4088959d3dfb
162 changes: 112 additions & 50 deletions Lib/test/test__interpreters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,67 +1,108 @@
import contextlib
import os
import threading
import unittest

from test import support

interpreters = support.import_module('_interpreters')


class InterpretersTests(unittest.TestCase):
@contextlib.contextmanager
def _blocked():
r, w = os.pipe()
wait_script = """if True:
import select
# Wait for a "done" signal.
select.select([{}], [], [])

#import time
#time.sleep(1_000_000)
""".format(r)
try:
yield wait_script
finally:
os.write(w, b'') # release!
os.close(r)
os.close(w)


def setUp(self):
self.ids = []
self.lock = threading.Lock()
class TestBase(unittest.TestCase):

def tearDown(self):
for id in self.ids:
for id in interpreters._enumerate():
if id == 0: # main
continue
try:
interpreters.destroy(id)
except RuntimeError:
pass # already destroyed

def _create(self):
id = interpreters.create()
self.ids.append(id)
return id

def test_create_in_main(self):
class CreateTests(TestBase):

def test_in_main(self):
id = interpreters.create()
self.ids.append(id)

self.assertIn(id, interpreters._enumerate())

def test_create_unique_id(self):
def test_unique_id(self):
seen = set()
for _ in range(100):
id = self._create()
id = interpreters.create()
interpreters.destroy(id)
seen.add(id)

self.assertEqual(len(seen), 100)

def test_create_in_thread(self):
def test_in_thread(self):
lock = threading.Lock()
id = None
def f():
nonlocal id
id = interpreters.create()
self.ids.append(id)
self.lock.acquire()
self.lock.release()
lock.acquire()
lock.release()

t = threading.Thread(target=f)
with self.lock:
with lock:
t.start()
t.join()
self.assertIn(id, interpreters._enumerate())

@unittest.skip('waiting for run_string()')
def test_create_in_subinterpreter(self):
raise NotImplementedError
def test_in_subinterpreter(self):
main, = interpreters._enumerate()
id = interpreters.create()
interpreters._run_string(id, """if True:
import _interpreters
id = _interpreters.create()
#_interpreters.create()
""")

ids = interpreters._enumerate()
self.assertIn(id, ids)
self.assertIn(main, ids)
self.assertEqual(len(ids), 3)

def test_in_threaded_subinterpreter(self):
main, = interpreters._enumerate()
id = interpreters.create()
def f():
interpreters._run_string(id, """if True:
import _interpreters
_interpreters.create()
""")

t = threading.Thread(target=f)
t.start()
t.join()

@unittest.skip('waiting for run_string()')
def test_create_in_threaded_subinterpreter(self):
raise NotImplementedError
ids = interpreters._enumerate()
self.assertIn(id, ids)
self.assertIn(main, ids)
self.assertEqual(len(ids), 3)

def test_create_after_destroy_all(self):
def test_after_destroy_all(self):
before = set(interpreters._enumerate())
# Create 3 subinterpreters.
ids = []
Expand All @@ -73,46 +114,46 @@ def test_create_after_destroy_all(self):
interpreters.destroy(id)
# Finally, create another.
id = interpreters.create()
self.ids.append(id)
self.assertEqual(set(interpreters._enumerate()), before | {id})

def test_create_after_destroy_some(self):
def test_after_destroy_some(self):
before = set(interpreters._enumerate())
# Create 3 subinterpreters.
id1 = interpreters.create()
id2 = interpreters.create()
self.ids.append(id2)
id3 = interpreters.create()
# Now destroy 2 of them.
interpreters.destroy(id1)
interpreters.destroy(id3)
# Finally, create another.
id = interpreters.create()
self.ids.append(id)
self.assertEqual(set(interpreters._enumerate()), before | {id, id2})

def test_destroy_one(self):
id1 = self._create()
id2 = self._create()
id3 = self._create()

class DestroyTests(TestBase):

def test_one(self):
id1 = interpreters.create()
id2 = interpreters.create()
id3 = interpreters.create()
self.assertIn(id2, interpreters._enumerate())
interpreters.destroy(id2)
self.assertNotIn(id2, interpreters._enumerate())
self.assertIn(id1, interpreters._enumerate())
self.assertIn(id3, interpreters._enumerate())

def test_destroy_all(self):
def test_all(self):
before = set(interpreters._enumerate())
ids = set()
for _ in range(3):
id = self._create()
id = interpreters.create()
ids.add(id)
self.assertEqual(set(interpreters._enumerate()), before | ids)
for id in ids:
interpreters.destroy(id)
self.assertEqual(set(interpreters._enumerate()), before)

def test_destroy_main(self):
def test_main(self):
main, = interpreters._enumerate()
with self.assertRaises(RuntimeError):
interpreters.destroy(main)
Expand All @@ -125,41 +166,62 @@ def f():
t.start()
t.join()

def test_destroy_already_destroyed(self):
def test_already_destroyed(self):
id = interpreters.create()
interpreters.destroy(id)
with self.assertRaises(RuntimeError):
interpreters.destroy(id)

def test_destroy_does_not_exist(self):
def test_does_not_exist(self):
with self.assertRaises(RuntimeError):
interpreters.destroy(1_000_000)

def test_destroy_bad_id(self):
def test_bad_id(self):
with self.assertRaises(RuntimeError):
interpreters.destroy(-1)

@unittest.skip('waiting for run_string()')
def test_destroy_from_current(self):
raise NotImplementedError
def test_from_current(self):
id = interpreters.create()
with self.assertRaises(RuntimeError):
interpreters._run_string(id, """if True:
import _interpreters
_interpreters.destroy({})
""".format(id))

@unittest.skip('waiting for run_string()')
def test_destroy_from_sibling(self):
raise NotImplementedError
def test_from_sibling(self):
main, = interpreters._enumerate()
id1 = interpreters.create()
id2 = interpreters.create()
interpreters._run_string(id1, """if True:
import _interpreters
_interpreters.destroy({})
""".format(id2))
self.assertEqual(set(interpreters._enumerate()), {main, id1})

def test_destroy_from_other_thread(self):
def test_from_other_thread(self):
id = interpreters.create()
self.ids.append(id)
def f():
interpreters.destroy(id)

t = threading.Thread(target=f)
t.start()
t.join()

@unittest.skip('waiting for run_string()')
def test_destroy_still_running(self):
raise NotImplementedError
@unittest.skip('not working yet')
def test_still_running(self):
main, = interpreters._enumerate()
id = interpreters.create()
def f():
interpreters._run_string(id, wait_script)

t = threading.Thread(target=f)
with _blocked() as wait_script:
t.start()
with self.assertRaises(RuntimeError):
interpreters.destroy(id)

t.join()
self.assertEqual(set(interpreters._enumerate()), {main, id})


if __name__ == "__main__":
Expand Down
Loading