From b57de671d1f07a133d628b3f4ce26ca8d449076f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Dec 2016 06:45:19 -0700 Subject: [PATCH 01/17] Add a unique ID to each interpreter. --- Doc/c-api/init.rst | 8 +++++ Include/pystate.h | 3 ++ Lib/test/test_capi.py | 84 ++++++++++++++++++++++++++++++++++++++++--- Programs/_testembed.c | 5 ++- Python/pystate.c | 15 ++++++++ 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 56cf77affe6042..42c04fceb944f0 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -821,6 +821,14 @@ been created. :c:func:`PyThreadState_Clear`. +.. c:function:: unsigned long PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then 0 is returned and an error is set. + + .. versionadded:: 3.7 + + .. c:function:: PyObject* PyThreadState_GetDict() Return a dictionary in which extensions can store thread-specific state diff --git a/Include/pystate.h b/Include/pystate.h index 62254fab96364e..3e546b34ca9dec 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -28,6 +28,8 @@ typedef struct _is { struct _is *next; struct _ts *tstate_head; + unsigned long id; // XXX random hash? UUID? + PyObject *modules; PyObject *modules_by_index; PyObject *sysdict; @@ -157,6 +159,7 @@ typedef struct _ts { PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); +PyAPI_FUNC(unsigned long) PyInterpreterState_GetID(PyInterpreterState *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); #endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index eb3e2c558d88e1..0990458d6c56a7 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -1,6 +1,7 @@ # Run the _testcapi module tests (tests for the Python/C API): by defn, # these are all functions _testcapi exports whose name begins with 'test_'. +from collections import namedtuple import os import pickle import random @@ -384,12 +385,85 @@ def run_embedded_interpreter(self, *args): return out, err def test_subinterps(self): - # This is just a "don't crash" test out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters") - if support.verbose: - print() - print(out) - print(err) + self.assertEqual(err, "") + + # The output from _testembed looks like this: + # --- Pass 0 --- + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 + # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 + # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # --- Pass 1 --- + # ... + + interp_pat = (r"^interp (\d+) <(0x[\da-f]+)>, " + r"thread state <(0x[\da-f]+)>: " + r"id\(modules\) = ([\d]+)$") + Interp = namedtuple("Interp", "id interp tstate modules") + + main = None + lastmain = None + numinner = None + numloops = 0 + for line in out.splitlines(): + if line == "--- Pass {} ---".format(numloops): + if numinner is not None: + self.assertEqual(numinner, 5) + if support.verbose: + print(line) + lastmain = main + main = None + mainid = numloops * 4 + 1 + numloops += 1 + numinner = 0 + continue + numinner += 1 + + self.assertLessEqual(numinner, 5) + match = re.match(interp_pat, line) + if match is None: + self.assertRegex(line, interp_pat) + + # The last line in the loop should be the same as the first. + if numinner == 5: + self.assertEqual(match.groups(), main) + continue + + # Parse the line from the loop. The first line is the main + # interpreter and the 3 afterward are subinterpreters. + interp = Interp(*match.groups()) + if support.verbose: + print(interp) + if numinner == 1: + main = interp + id = str(mainid) + else: + subid = mainid + numinner - 1 + id = str(subid) + + # Validate the loop line for each interpreter. + self.assertEqual(interp.id, id) + self.assertTrue(interp.interp) + self.assertTrue(interp.tstate) + self.assertTrue(interp.modules) + if interp is main: + if lastmain is not None: + # A new main interpreter may have the same interp + # and/or tstate pointer as an earlier finalized/ + # destroyed one. So we do not check interp or + # tstate here. + self.assertNotEqual(interp.modules, lastmain.modules) + else: + # A new subinterpreter may have the same + # PyInterpreterState pointer as a previous one if + # the earlier one has already been destroyed. So + # we compare with the main interpreter. The same + # applies to tstate. + self.assertNotEqual(interp.interp, main.interp) + self.assertNotEqual(interp.tstate, main.tstate) + self.assertNotEqual(interp.modules, main.modules) @staticmethod def _get_default_pipe_encoding(): diff --git a/Programs/_testembed.c b/Programs/_testembed.c index a68d4fa25f7cd0..d1fc61ccce38a9 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -24,7 +24,9 @@ static void print_subinterp(void) { /* Just output some debug stuff */ PyThreadState *ts = PyThreadState_Get(); - printf("interp %p, thread state %p: ", ts->interp, ts); + unsigned long id = PyInterpreterState_GetID(ts->interp); + printf("interp %lu <%p>, thread state <%p>: ", + id, ts->interp, ts); fflush(stdout); PyRun_SimpleString( "import sys;" @@ -56,6 +58,7 @@ static int test_repeated_init_and_subinterpreters(void) PyThreadState_Swap(NULL); for (j=0; j<3; j++) { + unsigned long mis = PyInterpreterState_GetID(mainstate->interp); substate = Py_NewInterpreter(); print_subinterp(); Py_EndInterpreter(substate); diff --git a/Python/pystate.c b/Python/pystate.c index 8e81707c7cc559..da612c5170a7a2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -66,6 +66,8 @@ static void _PyGILState_NoteThreadState(PyThreadState* tstate); #endif +static unsigned long _next_id = 1; // 0 is reserved for errors. + PyInterpreterState * PyInterpreterState_New(void) { @@ -103,6 +105,8 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; + interp->id = _next_id; + _next_id += 1; // XXX overflow... HEAD_UNLOCK(); } @@ -170,6 +174,17 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } +unsigned long +PyInterpreterState_GetID(PyInterpreterState *interp) +{ + if (interp == NULL) { + PyErr_SetString(PyExc_RuntimeError, "no interpreter provided"); + return 0; + } + return interp->id; +} + + /* Default implementation for _PyThreadState_GetFrame */ static struct _frame * threadstate_getframe(PyThreadState *self) From 160c90e763a08e54eea4c1b1fbac8d0bcde143af Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 Dec 2016 16:23:42 -0700 Subject: [PATCH 02/17] Reset _next_id to 1 in each Py_Initialize() call. --- Include/pystate.h | 9 +++++++++ Lib/test/test_capi.py | 2 +- Python/pylifecycle.c | 1 + Python/pystate.c | 15 ++++++++++++--- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 3e546b34ca9dec..b4101db7548d0e 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -14,6 +14,15 @@ removed (with effort). */ /* State shared between threads */ +#ifndef Py_LIMITED_API +/* This is initialized to 1 in Py_Initialize(). An interpreter + ID of 0 is reserved for errors. The main interpreter will always + have an ID of 1. Overflow results in a RuntimeError. If that + becomes a problem later then we can adjust, e.g. by using a 64-bit + int or even a Python int. */ +PyAPI_DATA(unsigned long) _PyInterpreterState_next_id; +#endif + struct _ts; /* Forward */ struct _is; /* Forward */ struct _frame; /* Forward declaration for PyFrameObject. */ diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 0990458d6c56a7..f5c95c1f36cbae 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -415,7 +415,7 @@ def test_subinterps(self): print(line) lastmain = main main = None - mainid = numloops * 4 + 1 + mainid = 1 numloops += 1 numinner = 0 continue diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c0f41b3ca73463..2a1ba97268c2bc 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -344,6 +344,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) _PyRandom_Init(); + _PyInterpreterState_next_id = 1; interp = PyInterpreterState_New(); if (interp == NULL) Py_FatalError("Py_Initialize: can't make first interpreter"); diff --git a/Python/pystate.c b/Python/pystate.c index da612c5170a7a2..d40dfa3f6a938c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -66,7 +66,9 @@ static void _PyGILState_NoteThreadState(PyThreadState* tstate); #endif -static unsigned long _next_id = 1; // 0 is reserved for errors. +/* We initialize this to 0 so that the pre-Py_Initialize() value + results in an error. */ +unsigned long _PyInterpreterState_next_id = 0; PyInterpreterState * PyInterpreterState_New(void) @@ -105,8 +107,15 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; - interp->id = _next_id; - _next_id += 1; // XXX overflow... + if (_PyInterpreterState_next_id == 0) { + /* overflow or Py_Initialize() not called! */ + PyErr_SetString(PyExc_RuntimeError, + "failed to get an interpreter ID"); + interp = NULL; + } else { + interp->id = _PyInterpreterState_next_id; + _PyInterpreterState_next_id += 1; + } HEAD_UNLOCK(); } From 3948bf22e682bf1b27fabdbd562c7b3e73f53bce Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 30 Dec 2016 16:30:07 -0700 Subject: [PATCH 03/17] Drop a TODO. --- Include/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/pystate.h b/Include/pystate.h index b4101db7548d0e..3d12ce278589b1 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -37,7 +37,7 @@ typedef struct _is { struct _is *next; struct _ts *tstate_head; - unsigned long id; // XXX random hash? UUID? + unsigned long id; PyObject *modules; PyObject *modules_by_index; From 5caac3dcbd74d99c3ca558e304ce816d4c77019a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 01:24:25 -0700 Subject: [PATCH 04/17] Use int_fast64_t for the interpreter ID. --- Include/pystate.h | 18 ++++++++---------- Lib/test/test_capi.py | 2 +- Python/pylifecycle.c | 2 +- Python/pystate.c | 6 +++--- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 3d12ce278589b1..c9903a5fd95888 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -14,15 +14,6 @@ removed (with effort). */ /* State shared between threads */ -#ifndef Py_LIMITED_API -/* This is initialized to 1 in Py_Initialize(). An interpreter - ID of 0 is reserved for errors. The main interpreter will always - have an ID of 1. Overflow results in a RuntimeError. If that - becomes a problem later then we can adjust, e.g. by using a 64-bit - int or even a Python int. */ -PyAPI_DATA(unsigned long) _PyInterpreterState_next_id; -#endif - struct _ts; /* Forward */ struct _is; /* Forward */ struct _frame; /* Forward declaration for PyFrameObject. */ @@ -30,6 +21,13 @@ struct _frame; /* Forward declaration for PyFrameObject. */ #ifdef Py_LIMITED_API typedef struct _is PyInterpreterState; #else +/* This is initialized to 0 in Py_Initialize(). A negative interpreter + ID indicates an error occurred. The main interpreter will always + have an ID of 0. Overflow results in a RuntimeError. If that + becomes a problem later then we can adjust, e.g. by using a Python + int. */ +PyAPI_DATA(int_fast64_t) _PyInterpreterState_next_id; + typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int); typedef struct _is { @@ -37,7 +35,7 @@ typedef struct _is { struct _is *next; struct _ts *tstate_head; - unsigned long id; + int_fast64_t id; PyObject *modules; PyObject *modules_by_index; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index f5c95c1f36cbae..f8eb60db74463e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -415,7 +415,7 @@ def test_subinterps(self): print(line) lastmain = main main = None - mainid = 1 + mainid = 0 numloops += 1 numinner = 0 continue diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2a1ba97268c2bc..34a3f8c069ace1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -344,7 +344,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) _PyRandom_Init(); - _PyInterpreterState_next_id = 1; + _PyInterpreterState_next_id = 0; interp = PyInterpreterState_New(); if (interp == NULL) Py_FatalError("Py_Initialize: can't make first interpreter"); diff --git a/Python/pystate.c b/Python/pystate.c index d40dfa3f6a938c..9c403fb363775f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -66,9 +66,9 @@ static void _PyGILState_NoteThreadState(PyThreadState* tstate); #endif -/* We initialize this to 0 so that the pre-Py_Initialize() value +/* We initialize this to -1 so that the pre-Py_Initialize() value results in an error. */ -unsigned long _PyInterpreterState_next_id = 0; +int_fast64_t _PyInterpreterState_next_id = -1; PyInterpreterState * PyInterpreterState_New(void) @@ -107,7 +107,7 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; - if (_PyInterpreterState_next_id == 0) { + if (_PyInterpreterState_next_id < 0) { /* overflow or Py_Initialize() not called! */ PyErr_SetString(PyExc_RuntimeError, "failed to get an interpreter ID"); From 6c6a6c597e10aecb5fa25e7622847899d2e6db52 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 02:27:19 -0700 Subject: [PATCH 05/17] _PyInterpreterState_next_id -> _PyInterpreterState_Init(). --- Include/pystate.h | 10 +++------- Python/pylifecycle.c | 2 +- Python/pystate.c | 25 +++++++++++++++++++------ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index c9903a5fd95888..19747189f946d1 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -21,13 +21,6 @@ struct _frame; /* Forward declaration for PyFrameObject. */ #ifdef Py_LIMITED_API typedef struct _is PyInterpreterState; #else -/* This is initialized to 0 in Py_Initialize(). A negative interpreter - ID indicates an error occurred. The main interpreter will always - have an ID of 0. Overflow results in a RuntimeError. If that - becomes a problem later then we can adjust, e.g. by using a Python - int. */ -PyAPI_DATA(int_fast64_t) _PyInterpreterState_next_id; - typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int); typedef struct _is { @@ -163,6 +156,9 @@ typedef struct _ts { #endif +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyInterpreterState_Init(void); +#endif /* !Py_LIMITED_API */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 34a3f8c069ace1..90f85518379f67 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -344,7 +344,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) _PyRandom_Init(); - _PyInterpreterState_next_id = 0; + _PyInterpreterState_Init(); interp = PyInterpreterState_New(); if (interp == NULL) Py_FatalError("Py_Initialize: can't make first interpreter"); diff --git a/Python/pystate.c b/Python/pystate.c index 9c403fb363775f..4918b3bb3cfebb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -65,10 +65,23 @@ PyThreadFrameGetter _PyThreadState_GetFrame = NULL; static void _PyGILState_NoteThreadState(PyThreadState* tstate); #endif - -/* We initialize this to -1 so that the pre-Py_Initialize() value +/* _next_interp_id is an auto-numbered sequence of small integers. + It gets initialized in _PyInterpreterState_Init(), which is called + in Py_Initialize(), and used in PyInterpreterState_New(). A negative + interpreter ID indicates an error occurred. The main interpreter + will always have an ID of 0. Overflow results in a RuntimeError. + If that becomes a problem later then we can adjust, e.g. by using + a Python int. + + We initialize this to -1 so that the pre-Py_Initialize() value results in an error. */ -int_fast64_t _PyInterpreterState_next_id = -1; +static int_fast64_t _next_interp_id = -1; + +void +_PyInterpreterState_Init(void) +{ + _next_interp_id = 0; +} PyInterpreterState * PyInterpreterState_New(void) @@ -107,14 +120,14 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; - if (_PyInterpreterState_next_id < 0) { + if (_next_interp_id < 0) { /* overflow or Py_Initialize() not called! */ PyErr_SetString(PyExc_RuntimeError, "failed to get an interpreter ID"); interp = NULL; } else { - interp->id = _PyInterpreterState_next_id; - _PyInterpreterState_next_id += 1; + interp->id = _next_interp_id; + _next_interp_id += 1; } HEAD_UNLOCK(); } From e335edd3e5ef3e38912bf8f2b644b946c7ec11f1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 02:37:18 -0700 Subject: [PATCH 06/17] Fix an outdated comment. --- Programs/_testembed.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d1fc61ccce38a9..65b04de9cd01ef 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -22,7 +22,8 @@ static void _testembed_Py_Initialize(void) static void print_subinterp(void) { - /* Just output some debug stuff */ + /* Output information about the interpreter in the format + expected in Lib/test/test_capi.py (test_subinterps). */ PyThreadState *ts = PyThreadState_Get(); unsigned long id = PyInterpreterState_GetID(ts->interp); printf("interp %lu <%p>, thread state <%p>: ", From d8150d40e346d151e4217c15f9c541089d1cf84a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 02:41:51 -0700 Subject: [PATCH 07/17] Fix PyInterpreterState_GetID(). --- Doc/c-api/init.rst | 4 ++-- Include/pystate.h | 2 +- Programs/_testembed.c | 2 +- Python/pystate.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 42c04fceb944f0..d68fb02b93da6c 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -821,10 +821,10 @@ been created. :c:func:`PyThreadState_Clear`. -.. c:function:: unsigned long PyInterpreterState_GetID(PyInterpreterState *interp) +.. c:function:: int_fast64_t PyInterpreterState_GetID(PyInterpreterState *interp) Return the interpreter's unique ID. If there was any error in doing - so then 0 is returned and an error is set. + so then -1 is returned and an error is set. .. versionadded:: 3.7 diff --git a/Include/pystate.h b/Include/pystate.h index 19747189f946d1..9351ae31346c54 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -162,7 +162,7 @@ PyAPI_FUNC(void) _PyInterpreterState_Init(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); -PyAPI_FUNC(unsigned long) PyInterpreterState_GetID(PyInterpreterState *); +PyAPI_FUNC(int_fast64_t) PyInterpreterState_GetID(PyInterpreterState *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); #endif /* !Py_LIMITED_API */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 65b04de9cd01ef..e4ad4552be0f9b 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -25,7 +25,7 @@ static void print_subinterp(void) /* Output information about the interpreter in the format expected in Lib/test/test_capi.py (test_subinterps). */ PyThreadState *ts = PyThreadState_Get(); - unsigned long id = PyInterpreterState_GetID(ts->interp); + int_fast64_t id = PyInterpreterState_GetID(ts->interp); printf("interp %lu <%p>, thread state <%p>: ", id, ts->interp, ts); fflush(stdout); diff --git a/Python/pystate.c b/Python/pystate.c index 4918b3bb3cfebb..acbdfdec9cef7b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -196,12 +196,12 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } -unsigned long +int_fast64_t PyInterpreterState_GetID(PyInterpreterState *interp) { if (interp == NULL) { PyErr_SetString(PyExc_RuntimeError, "no interpreter provided"); - return 0; + return -1; } return interp->id; } From 9a14e012f0ff3464b646982491c500a298640d90 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 02:44:09 -0700 Subject: [PATCH 08/17] Drop an ununsed variable. --- Programs/_testembed.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e4ad4552be0f9b..a663005ce0359b 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -59,7 +59,6 @@ static int test_repeated_init_and_subinterpreters(void) PyThreadState_Swap(NULL); for (j=0; j<3; j++) { - unsigned long mis = PyInterpreterState_GetID(mainstate->interp); substate = Py_NewInterpreter(); print_subinterp(); Py_EndInterpreter(substate); From 67cb77ea6d3c4d20c3eaa0d27f66c5decffdc855 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 10:49:55 -0700 Subject: [PATCH 09/17] int_fast64_t -> PY_INT64_T. --- Doc/c-api/init.rst | 2 +- Include/pystate.h | 4 ++-- Programs/_testembed.c | 2 +- Python/pystate.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index d68fb02b93da6c..ca7636b6f5a5f2 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -821,7 +821,7 @@ been created. :c:func:`PyThreadState_Clear`. -.. c:function:: int_fast64_t PyInterpreterState_GetID(PyInterpreterState *interp) +.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp) Return the interpreter's unique ID. If there was any error in doing so then -1 is returned and an error is set. diff --git a/Include/pystate.h b/Include/pystate.h index 9351ae31346c54..654321b567a720 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -28,7 +28,7 @@ typedef struct _is { struct _is *next; struct _ts *tstate_head; - int_fast64_t id; + PY_INT64_T id; PyObject *modules; PyObject *modules_by_index; @@ -162,7 +162,7 @@ PyAPI_FUNC(void) _PyInterpreterState_Init(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); -PyAPI_FUNC(int_fast64_t) PyInterpreterState_GetID(PyInterpreterState *); +PyAPI_FUNC(PY_INT64_T) PyInterpreterState_GetID(PyInterpreterState *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); #endif /* !Py_LIMITED_API */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index a663005ce0359b..71feaf8a8fd2d0 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -25,7 +25,7 @@ static void print_subinterp(void) /* Output information about the interpreter in the format expected in Lib/test/test_capi.py (test_subinterps). */ PyThreadState *ts = PyThreadState_Get(); - int_fast64_t id = PyInterpreterState_GetID(ts->interp); + PY_INT64_T id = PyInterpreterState_GetID(ts->interp); printf("interp %lu <%p>, thread state <%p>: ", id, ts->interp, ts); fflush(stdout); diff --git a/Python/pystate.c b/Python/pystate.c index acbdfdec9cef7b..9f7c7c15b73fa5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -75,7 +75,7 @@ static void _PyGILState_NoteThreadState(PyThreadState* tstate); We initialize this to -1 so that the pre-Py_Initialize() value results in an error. */ -static int_fast64_t _next_interp_id = -1; +static PY_INT64_T _next_interp_id = -1; void _PyInterpreterState_Init(void) @@ -196,7 +196,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } -int_fast64_t +PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp) { if (interp == NULL) { From 0f36bcc27d566248f3518c274534dd75ce7578cb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 May 2017 16:44:01 -0700 Subject: [PATCH 10/17] Use int64_t directly. --- Include/pystate.h | 4 ++-- Programs/_testembed.c | 2 +- Python/pystate.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 654321b567a720..82ac3a06d79665 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -28,7 +28,7 @@ typedef struct _is { struct _is *next; struct _ts *tstate_head; - PY_INT64_T id; + int64_t id; PyObject *modules; PyObject *modules_by_index; @@ -162,7 +162,7 @@ PyAPI_FUNC(void) _PyInterpreterState_Init(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); -PyAPI_FUNC(PY_INT64_T) PyInterpreterState_GetID(PyInterpreterState *); +PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); #endif /* !Py_LIMITED_API */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 71feaf8a8fd2d0..9e859921f12328 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -25,7 +25,7 @@ static void print_subinterp(void) /* Output information about the interpreter in the format expected in Lib/test/test_capi.py (test_subinterps). */ PyThreadState *ts = PyThreadState_Get(); - PY_INT64_T id = PyInterpreterState_GetID(ts->interp); + int64_t id = PyInterpreterState_GetID(ts->interp); printf("interp %lu <%p>, thread state <%p>: ", id, ts->interp, ts); fflush(stdout); diff --git a/Python/pystate.c b/Python/pystate.c index 9f7c7c15b73fa5..47422fb2e35d26 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -75,7 +75,7 @@ static void _PyGILState_NoteThreadState(PyThreadState* tstate); We initialize this to -1 so that the pre-Py_Initialize() value results in an error. */ -static PY_INT64_T _next_interp_id = -1; +static int64_t _next_interp_id = -1; void _PyInterpreterState_Init(void) @@ -196,7 +196,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } -PY_INT64_T +int64_t PyInterpreterState_GetID(PyInterpreterState *interp) { if (interp == NULL) { From 808aff4c01a0642431ab5c5daa4878e062c76bd4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 15:30:42 -0700 Subject: [PATCH 11/17] Not all platforms show a 0x prefix for memory addresses. --- Lib/test/test_capi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index f8eb60db74463e..5f9b1dddad6001 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -398,8 +398,8 @@ def test_subinterps(self): # --- Pass 1 --- # ... - interp_pat = (r"^interp (\d+) <(0x[\da-f]+)>, " - r"thread state <(0x[\da-f]+)>: " + interp_pat = (r"^interp (\d+) <((?:0x)?[\da-fA-F]+)>, " + r"thread state <((?:0x)?[\da-fA-F]+)>: " r"id\(modules\) = ([\d]+)$") Interp = namedtuple("Interp", "id interp tstate modules") From cb8122603d9a9f340a2fe0087aa028b4503bdfb2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 18 May 2017 17:23:11 -0700 Subject: [PATCH 12/17] Dereference the interp pointer before calling printf(). --- Programs/_testembed.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 9e859921f12328..24270ade791e74 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -25,9 +25,10 @@ static void print_subinterp(void) /* Output information about the interpreter in the format expected in Lib/test/test_capi.py (test_subinterps). */ PyThreadState *ts = PyThreadState_Get(); - int64_t id = PyInterpreterState_GetID(ts->interp); + PyInterpreterState *interp = ts->interp; + int64_t id = PyInterpreterState_GetID(interp); printf("interp %lu <%p>, thread state <%p>: ", - id, ts->interp, ts); + id, interp, ts); fflush(stdout); PyRun_SimpleString( "import sys;" From 3061a4084f79113c75e3e048e3db1e60e15f7311 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 19 May 2017 12:13:14 -0700 Subject: [PATCH 13/17] Use inttypes.h to format pointer addresses. --- Lib/test/test_capi.py | 4 ++-- Programs/_testembed.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 5f9b1dddad6001..13cc543ac15f8f 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -398,8 +398,8 @@ def test_subinterps(self): # --- Pass 1 --- # ... - interp_pat = (r"^interp (\d+) <((?:0x)?[\da-fA-F]+)>, " - r"thread state <((?:0x)?[\da-fA-F]+)>: " + interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " + r"thread state <(0x[\dA-F]+)>: " r"id\(modules\) = ([\d]+)$") Interp = namedtuple("Interp", "id interp tstate modules") diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 24270ade791e74..de88404465a75b 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1,4 +1,5 @@ #include +#include #include /********************************************************* @@ -27,8 +28,8 @@ static void print_subinterp(void) PyThreadState *ts = PyThreadState_Get(); PyInterpreterState *interp = ts->interp; int64_t id = PyInterpreterState_GetID(interp); - printf("interp %lu <%p>, thread state <%p>: ", - id, interp, ts); + printf("interp %lu <0x%" PRIXPTR ">, thread state <0x%" PRIXPTR ">: ", + id, (uintptr_t)interp, (uintptr_t)ts); fflush(stdout); PyRun_SimpleString( "import sys;" From 14d74f956a6393f789b982c14eb6ce5f88be8c79 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 20 May 2017 11:30:51 -0700 Subject: [PATCH 14/17] Skip some problematic (new) checks on Windows in a test. --- Lib/test/test_capi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 13cc543ac15f8f..766098134c4ec6 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -4,6 +4,7 @@ from collections import namedtuple import os import pickle +import platform import random import re import subprocess @@ -448,6 +449,12 @@ def test_subinterps(self): self.assertTrue(interp.interp) self.assertTrue(interp.tstate) self.assertTrue(interp.modules) + if platform.system() == 'Windows': + # XXX Fix on Windows: something is going on with the + # pointers in Programs/_testembed.c. interp.interp + # is 0x0 and # interp.modules is the same between + # interpreters. + continue if interp is main: if lastmain is not None: # A new main interpreter may have the same interp From 4797749e6f414bd175b21e67d24c9d28caf9838f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 May 2017 15:54:37 -0700 Subject: [PATCH 15/17] Add a news entry. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 5a7e377d6b2d7c..3c52308712f912 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -53,6 +53,9 @@ Core and Builtins - bpo-24821: Fixed the slowing down to 25 times in the searching of some unlucky Unicode characters. +- bpo-29102: Add a unique ID to PyInterpreterState. This makes it easier + to identify each subinterpreter. + - bpo-29894: The deprecation warning is emitted if __complex__ returns an instance of a strict subclass of complex. In a future versions of Python this can be an error. From c12e2289330cfd24308e1334a10a6b4f73b3bb6b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 May 2017 17:04:54 -0700 Subject: [PATCH 16/17] Only add the limited API in 3.7+. --- Include/pystate.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/pystate.h b/Include/pystate.h index 82ac3a06d79665..9170ba9f7a77b8 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -162,7 +162,10 @@ PyAPI_FUNC(void) _PyInterpreterState_Init(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 +/* New in 3.7 */ PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *); +#endif #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); #endif /* !Py_LIMITED_API */ From 85dbed770467802e73ace12da27b008e0714b2f4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 22 May 2017 17:11:30 -0700 Subject: [PATCH 17/17] Add PyInterpreterState_GetID() to the stable API on Windows. --- PC/python3.def | 1 + 1 file changed, 1 insertion(+) diff --git a/PC/python3.def b/PC/python3.def index bb3ca8814b4cef..ad65294045f677 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -538,6 +538,7 @@ EXPORTS PySlice_Type=python37.PySlice_Type DATA PySlice_Unpack=python37.PySlice_Unpack PySortWrapper_Type=python37.PySortWrapper_Type DATA + PyInterpreterState_GetID=python37.PyInterpreterState_GetID PyState_AddModule=python37.PyState_AddModule PyState_FindModule=python37.PyState_FindModule PyState_RemoveModule=python37.PyState_RemoveModule