diff --git a/package-lock.json b/package-lock.json index 810ebed..a6b13aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "node-calls-python", - "version": "1.11.1", + "version": "1.12.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "node-calls-python", - "version": "1.11.1", + "version": "1.12.0", "license": "MIT", "dependencies": { "chokidar": "^3.6.0" diff --git a/package.json b/package.json index 65feb48..23f8be7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-calls-python", - "version": "1.11.1", + "version": "1.12.0", "license": "MIT", "description": "This module lets you run python code inside node without spawning new processes", "authors": [ diff --git a/src/addon.cpp b/src/addon.cpp index 246377f..47915a2 100644 --- a/src/addon.cpp +++ b/src/addon.cpp @@ -50,7 +50,7 @@ namespace nodecallspython ~CallTask() { - GIL gil; + auto gil = m_py->gil(); m_args = CPyObject(); m_kwargs = CPyObject(); m_result = CPyObject(); @@ -67,7 +67,7 @@ namespace nodecallspython ~ExecTask() { - GIL gil; + auto gil = m_py->gil(); m_result = CPyObject(); } }; @@ -82,7 +82,7 @@ namespace nodecallspython ~Handler() { - GIL gil; + auto gil = m_py->gil(); m_py->release(m_handler); } @@ -119,7 +119,7 @@ namespace nodecallspython static void CallAsync(napi_env env, void* data) { auto task = static_cast(data); - GIL gil; + auto gil = task->m_py->gil(); try { if (task->m_isFunc) @@ -136,7 +136,7 @@ namespace nodecallspython static void ExecAsync(napi_env env, void* data) { auto task = static_cast(data); - GIL gil; + auto gil = task->m_py->gil(); try { task->m_result = task->m_py->exec(task->m_handler, task->m_code, task->m_eval); @@ -150,7 +150,7 @@ namespace nodecallspython static void ImportAsync(napi_env env, void* data) { auto task = static_cast(data); - GIL gil; + auto gil = task->m_py->gil(); try { task->m_handler = task->m_py->import(task->m_name, task->m_allowReimport); @@ -218,7 +218,7 @@ namespace nodecallspython napi_value args; if (task->m_isFunc) { - GIL gil; + auto gil = task->m_py->gil(); args = task->m_py->convert(env, *task->m_result); } else @@ -247,7 +247,7 @@ namespace nodecallspython CHECK(napi_get_global(env, &global)); napi_value args; - GIL gil; + auto gil = task->m_py->gil(); args = task->m_py->convert(env, *task->m_result); napi_value callback; @@ -342,8 +342,8 @@ namespace nodecallspython if (sync) { - GIL gil; auto& py = obj->getInterpreter(); + auto gil = py.gil(); auto pyArgs = py.convert(env, napiargs, true); napi_value result; @@ -381,7 +381,7 @@ namespace nodecallspython napi_create_string_utf8(env, "Python::call", NAPI_AUTO_LENGTH, &optname); { - GIL gil; + auto gil = task->m_py->gil(); std::tie(task->m_args, task->m_kwargs) = obj->getInterpreter().convert(env, napiargs, false); } @@ -437,10 +437,10 @@ namespace nodecallspython napi_value value; CHECKNULL(napi_get_property(env, args[0], key, &value)); + auto& py = obj->getInterpreter(); if (sync) { - GIL gil; - auto& py = obj->getInterpreter(); + auto gil = py.gil(); auto pyres = py.exec(convertString(env, value), convertString(env, args[1]), eval); napi_value result; if (pyres) @@ -457,8 +457,10 @@ namespace nodecallspython if (callbackT == napi_function) { ExecTask* task = new ExecTask; - task->m_py = &(obj->getInterpreter()); + + auto gil = py.gil(); + task->m_py = &py; task->m_handler = convertString(env, value); task->m_code = convertString(env, args[1]); task->m_eval = eval; @@ -517,9 +519,9 @@ namespace nodecallspython if (sync) { - GIL gil; - auto name = convertString(env, args[0]); auto& py = obj->getInterpreter(); + auto gil = py.gil(); + auto name = convertString(env, args[0]); auto handler = py.import(name, allowReimport); return createHandler(env, &py, handler); @@ -730,7 +732,7 @@ namespace nodecallspython auto& py = obj->getInterpreter(); try { - GIL gil; + auto gil = py.gil(); py.reimport(directory); } catch(const std::exception& e) @@ -752,7 +754,7 @@ namespace nodecallspython auto& py = obj->getInterpreter(); try { - GIL gil; + auto gil = py.gil(); py.addImportPath(path); } catch(const std::exception& e) diff --git a/src/pyinterpreter.cpp b/src/pyinterpreter.cpp index ba3e20b..ab99125 100644 --- a/src/pyinterpreter.cpp +++ b/src/pyinterpreter.cpp @@ -10,6 +10,35 @@ using namespace nodecallspython; bool nodecallspython::PyInterpreter::m_inited = false; std::mutex nodecallspython::PyInterpreter::m_mutex; +GIL::GIL(PyThreadState *ts, bool release) : m_ts(ts), m_release(release) +{ + if (m_ts) + PyEval_RestoreThread(m_ts); +} + +GIL::~GIL() +{ + if (m_ts) + { + if (m_release) + PyThreadState_Clear(m_ts); + PyEval_SaveThread(); + if (m_release) + PyThreadState_Delete(m_ts); + } +} + +GIL& GIL::operator=(GIL&& other) +{ + m_ts = other.m_ts; + other.m_ts = nullptr; + + m_release = other.m_release; + other.m_release = false; + + return *this; +} + namespace { void signal_handler_int(int) @@ -32,20 +61,22 @@ PyInterpreter::PyInterpreter() : m_state(nullptr), m_syncJsAndPy(true) #endif Py_DECREF(PyImport_ImportModule("threading")); - - m_state = PyEval_SaveThread(); - + if (!std::getenv("NODE_CALLS_PYTHON_IGNORE_SIGINT")) PyOS_setsig(SIGINT, ::signal_handler_int); + m_mainThread = std::this_thread::get_id(); + m_inited = true; + + m_state = PyEval_SaveThread(); } } PyInterpreter::~PyInterpreter() { { - GIL gil; + auto gil = this->gil(); m_objs = {}; } @@ -56,6 +87,17 @@ PyInterpreter::~PyInterpreter() } } +GIL PyInterpreter::gil() +{ + if (m_mainThread == std::this_thread::get_id()) + return GIL(m_state, false); + else + { + auto* tstate = PyThreadState_New(PyInterpreterState_Main()); + return GIL(tstate, true); + } +} + #define CHECK(func) { auto res = func; if (res != napi_ok) { throw std::runtime_error(std::string(#func) + " returned with an error: " + std::to_string(static_cast(func))); } } namespace @@ -243,14 +285,21 @@ namespace return result; } - std::pair convert(napi_env env, napi_value arg, bool isSync, bool allowFunc, bool syncJsAndPy); + std::pair convert(napi_env env, napi_value arg, PyInterpreter* interpreter, bool isSync, bool allowFunc, bool syncJsAndPy); + + struct AsyncCall + { + PyInterpreter* interpreter; + PyObject* args; + }; void callJs(napi_env env, napi_value func, void* context, void* data) { - GIL gil; + std::unique_ptr call(reinterpret_cast(data)); + auto gil = call->interpreter->gil(); try { - CPyObject args(reinterpret_cast(data)); + CPyObject args(call->args); auto params = convertParams(env, *args); callJsImpl(env, func, params); @@ -260,20 +309,27 @@ namespace } } + struct AsyncCallback + { + napi_threadsafe_function* func; + PyInterpreter* interpreter; + }; + PyObject* __callback_function_napi_async(PyObject *self, PyObject* args) { - auto func = reinterpret_cast(PyCapsule_GetPointer(self, nullptr)); + auto func = reinterpret_cast(PyCapsule_GetPointer(self, nullptr)); Py_INCREF(args); - napi_call_threadsafe_function(*func, args, napi_tsfn_nonblocking); + napi_call_threadsafe_function(*func->func, new AsyncCall{func->interpreter, args}, napi_tsfn_nonblocking); Py_RETURN_NONE; } struct Promise { std::promise promise; + PyInterpreter* interpreter; PyObject* args; - Promise(PyObject* args) : args(args) + Promise(PyInterpreter* interpreter, PyObject* args) : interpreter(interpreter), args(args) { } }; @@ -285,7 +341,7 @@ namespace { std::vector params; { - GIL gil; + auto gil = promise->interpreter->gil(); CPyObject args(promise->args); params = convertParams(env, *args); } @@ -293,8 +349,8 @@ namespace auto result = callJsImpl(env, func, params); { - GIL gil; - auto pyResult = convert(env, result, true, false, false).first; + auto gil = promise->interpreter->gil(); + auto pyResult = convert(env, result, promise->interpreter, true, false, false).first; promise->promise.set_value(pyResult); } } @@ -306,13 +362,13 @@ namespace PyObject* __callback_function_napi_async_promise(PyObject *self, PyObject* args) { - auto func = reinterpret_cast(PyCapsule_GetPointer(self, nullptr)); + auto func = reinterpret_cast(PyCapsule_GetPointer(self, nullptr)); Py_INCREF(args); - auto promise = std::make_unique(args); + auto promise = std::make_unique(func->interpreter, args); auto future = promise->promise.get_future(); Py_BEGIN_ALLOW_THREADS; - napi_call_threadsafe_function(*func, promise.get(), napi_tsfn_nonblocking); + napi_call_threadsafe_function(*func->func, promise.get(), napi_tsfn_nonblocking); future.wait(); Py_END_ALLOW_THREADS; @@ -323,20 +379,22 @@ namespace { napi_env env; napi_value func; + PyInterpreter* interpreter; }; - PyObject* __callback_function_napi_sync(PyObject *self, PyObject* args) + PyObject* __callback_function_napi_sync(PyObject* self, PyObject* args) { auto func = reinterpret_cast(PyCapsule_GetPointer(self, nullptr)); auto params = convertParams(func->env, args); auto result = callJsImpl(func->env, func->func, params); - return convert(func->env, result, true, false, false).first; + return convert(func->env, result, func->interpreter, true, false, false).first; } void capsuleDestructor(PyObject* obj) { - auto func = reinterpret_cast(PyCapsule_GetPointer(obj, nullptr)); - napi_release_threadsafe_function(*func, napi_tsfn_abort); + auto func = reinterpret_cast(PyCapsule_GetPointer(obj, nullptr)); + napi_release_threadsafe_function(*func->func, napi_tsfn_abort); + delete func->func; delete func; } @@ -374,7 +432,7 @@ namespace return PyLong_FromLong(i); } - std::pair convert(napi_env env, napi_value arg, bool isSync, bool allowFunc, bool syncJsAndPy) + std::pair convert(napi_env env, napi_value arg, PyInterpreter* interpreter, bool isSync, bool allowFunc, bool syncJsAndPy) { napi_valuetype type; CHECK(napi_typeof(env, arg, &type)); @@ -393,7 +451,7 @@ namespace { napi_value value; CHECK(napi_get_element(env, arg, i, &value)); - PyList_SetItem(list, i, ::convert(env, value, isSync, allowFunc, syncJsAndPy).first); + PyList_SetItem(list, i, ::convert(env, value, interpreter, isSync, allowFunc, syncJsAndPy).first); } return { list, false }; @@ -536,9 +594,9 @@ namespace kwargs = true; else { - CPyObject pykey = ::convert(env, key, isSync, allowFunc, syncJsAndPy).first; + CPyObject pykey = ::convert(env, key, interpreter, isSync, allowFunc, syncJsAndPy).first; - CPyObject pyvalue = ::convert(env, value, isSync, allowFunc, syncJsAndPy).first; + CPyObject pyvalue = ::convert(env, value, interpreter, isSync, allowFunc, syncJsAndPy).first; PyDict_SetItem(dict, *pykey, *pyvalue); } @@ -550,7 +608,7 @@ namespace { if (isSync) { - CPyObject capsule = PyCapsule_New(new SycnCallback{env, arg}, nullptr, capsuleDestructorSync); + CPyObject capsule = PyCapsule_New(new SycnCallback{env, arg, interpreter}, nullptr, capsuleDestructorSync); auto function = PyCFunction_New(&mlSync, *capsule); return { function, false }; } @@ -561,7 +619,7 @@ namespace napi_value workName; CHECK(napi_create_string_utf8(env, "ThreadSafeCallback", NAPI_AUTO_LENGTH, &workName)); - CPyObject capsule = PyCapsule_New(tsfn, nullptr, capsuleDestructor); + CPyObject capsule = PyCapsule_New(new AsyncCallback{tsfn, interpreter}, nullptr, capsuleDestructor); PyObject* function = nullptr; if (syncJsAndPy) { @@ -590,7 +648,7 @@ std::pair PyInterpreter::convert(napi_env env, const std:: paramsVect.reserve(args.size()); for (auto i=0u;i #include #include +#include #include namespace nodecallspython { + class PyInterpreter; + class GIL { - PyGILState_STATE m_gstate; + friend class PyInterpreter; + + PyThreadState* m_ts; + bool m_release; + + GIL(PyThreadState *ts, bool release); + public: - GIL() - { - m_gstate = PyGILState_Ensure(); - } - ~GIL() - { - PyGILState_Release(m_gstate); - } + ~GIL(); GIL(const GIL&) = delete; GIL& operator=(const GIL&) = delete; GIL(GIL&&) = delete; - GIL& operator=(GIL&&) = delete; + + GIL& operator=(GIL&& other); }; class PyInterpreter @@ -36,6 +39,8 @@ namespace nodecallspython std::unordered_map m_objs; std::unordered_map m_imports; bool m_syncJsAndPy; + std::thread::id m_mainThread; + static std::mutex m_mutex; static bool m_inited; public: @@ -43,6 +48,8 @@ namespace nodecallspython ~PyInterpreter(); + GIL gil(); + std::pair convert(napi_env env, const std::vector& args, bool isSync); napi_value convert(napi_env env, PyObject* obj);