/*********************************************************************************************************************** * OpenStudio(R), Copyright (c) Alliance for Energy Innovation, LLC. * See also https://openstudio.net/license ***********************************************************************************************************************/ #include "PythonEngine.hpp" #include #include "../../src/utilities/core/Filesystem.hpp" #include #include #include #include #ifdef __GNUC__ # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif #include #include #include #ifdef __GNUC__ # pragma GCC diagnostic pop #endif // >= 3.13.0 // #if PY_VERSION_HEX >= 0x30d0000 #if PY_MAJOR_VERSION > 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION > 12) extern "C" { // These functions was removed from Python 3.13 API but are still exported // for the stable ABI. Until we upgrade to PyConfig, just keep using it extern void Py_SetPath(const wchar_t* path); } #endif namespace openstudio { void addToPythonPath(const openstudio::path& includePath) { if (includePath.empty()) { return; } // fmt::print("Prepending '{}' to sys.path\n", includePath); PyObject* unicodeIncludePath = nullptr; if constexpr (std::is_same_v) { const std::wstring ws = includePath.generic_wstring(); unicodeIncludePath = PyUnicode_FromWideChar(ws.c_str(), static_cast(ws.size())); // New reference } else { const std::string s = includePath.generic_string(); unicodeIncludePath = PyUnicode_FromString(s.c_str()); // New reference } if (unicodeIncludePath == nullptr) { throw std::runtime_error(fmt::format("Unable to convert path '{}' for addition to sys.path in Python", includePath.generic_string())); } PyObject* sysPath = PySys_GetObject("path"); // Borrowed reference int ret = PyList_Insert(sysPath, 0, unicodeIncludePath); Py_DECREF(unicodeIncludePath); if (ret != 0) { throw std::runtime_error(fmt::format("Unable to add path '{}' to the sys.path in Python", includePath.generic_string())); } } void PythonEngine::pyimport(const std::string& importName, const std::string& includePath) { addToPythonPath(includePath); PyImport_ImportModule(importName.c_str()); } void PythonEngine::setupPythonPath(const std::vector& includeDirs) { // Iterate in reverse order since addToPythonPath always inserts at pos 0 // --python_path path1 --python_path path2 => includeDirs = ["path1", "path2"] // std::ranges::reverse_view needs modern compilers for (auto it = includeDirs.rbegin(); it != includeDirs.rend(); it++) { addToPythonPath(*it); } } PythonEngine::PythonEngine(int argc, char* argv[]) : ScriptEngine(argc, argv), program(Py_DecodeLocale(pythonProgramName, nullptr)) { // TODO: modernize and use PyConfig (new in 3.8): https://docs.python.org/3/c-api/init_config.html // this frozen flag tells Python that the package and library have been frozen for embedding, so it shouldn't warn about missing prefixes Py_FrozenFlag = 1; // Path to the E+ shipped standard library auto pathToPythonPackages = getEnergyPlusDirectory() / "python_lib"; // The PYTHONPATH / PYTHONHOME should be set before initializing Python // If this Py_SetPath is called before Py_Initialize, then Py_GetPath won't attempt to compute a default search path // The default search path is affected by the Py_SetPythonHome // * if the user passed --python_home, we use that as the Python Home, and do not use Py_SetPath. But later we add the E+ standard_lib anyways // so it takes precedence (to limit incompatibility issues...) // * If the user didn't pass it, we use Py_SetPath set to the E+ standard_lib std::vector args(argv, std::next(argv, static_cast(argc))); bool pythonHomePassed = false; auto it = std::find(args.cbegin(), args.cend(), "--python_home"); if (it != args.cend()) { openstudio::path pythonHomeDir(*std::next(it)); wchar_t* h = Py_DecodeLocale(pythonHomeDir.make_preferred().string().c_str(), nullptr); Py_SetPythonHome(h); pythonHomePassed = true; } else { wchar_t* a = Py_DecodeLocale(pathToPythonPackages.make_preferred().string().c_str(), nullptr); Py_SetPath(a); } Py_SetProgramName(program); // optional but recommended Py_Initialize(); if (pythonHomePassed) { addToPythonPath(pathToPythonPackages); } #if defined(__APPLE__) || defined(__linux___) || defined(__unix__) addToPythonPath(pathToPythonPackages / "lib-dynload"); #endif PyObject* m = PyImport_AddModule("__main__"); if (m == nullptr) { throw std::runtime_error("Unable to add module __main__ for python script execution"); } m_globalDict = PyModule_GetDict(m); //PyRun_SimpleString("from time import time,ctime\n" // "print('Today is', ctime(time()))\n"); importOpenStudio(); } PythonEngine::~PythonEngine() { if (Py_FinalizeEx() < 0) { exit(120); } PyMem_RawFree(program); } void PythonEngine::importOpenStudio() { #if defined(__APPLE__) // RTLD_LOCAL is import an Apple so that that Python and Ruby do not conflict const std::string set_dlflags_cmd = R"( import sys import os pre_os_dl_open_flags = sys.getdlopenflags() sys.setdlopenflags(os.RTLD_LOCAL) )"; exec(set_dlflags_cmd.c_str()); #endif // generic_string() converts to a POSIX path, with forward slashes, so that pyimport doesn't choke on backslashes understood as escape sequence if (moduleIsRunningFromBuildDirectory()) { const auto bindingsDir = getOpenStudioModuleDirectory(); pyimport("openstudiodev", bindingsDir.generic_string()); } else { const auto bindingsDir = getOpenStudioModuleDirectory() / "../Python"; pyimport("openstudio", bindingsDir.generic_string()); } // Somehow that doesn't suffice to register it... exec("import openstudio"); #if defined(__APPLE__) // Reset the dlopen flags to the value prior to importOpenStudio const std::string reset_dlflags_cmd = R"( sys.setdlopenflags(pre_os_dl_open_flags) )"; exec(reset_dlflags_cmd.c_str()); #endif } struct PythonObject { PythonObject() = default; PythonObject(PyObject* obj) noexcept : obj_(obj) { if (obj_) { Py_INCREF(obj_); } } PythonObject(const PythonObject& other) noexcept : obj_(other.obj_) { if (obj_) { Py_INCREF(obj_); } } PythonObject(PythonObject&& other) noexcept : obj_(other.obj_) { // no reason to inc/dec, we just stole the ref counted object // from other other.obj_ = nullptr; } PythonObject& operator=(const PythonObject& rhs) noexcept { if (&rhs != this) { obj_ = rhs.obj_; if (obj_) { Py_INCREF(obj_); } } return *this; } PythonObject& operator=(PythonObject&& rhs) noexcept { if (&rhs != this) { obj_ = rhs.obj_; rhs.obj_ = nullptr; } return *this; } ~PythonObject() { if (obj_) { Py_DECREF(obj_); } } PyObject* obj_ = nullptr; }; void PythonEngine::exec(std::string_view sv) { std::string command{sv}; PyObject* v = PyRun_String(command.c_str(), Py_file_input, m_globalDict, m_globalDict); // PyObject* v = PyRun_SimpleString(command.c_str()); if (v == nullptr) { PyErr_Print(); throw std::runtime_error("Error executing Python code"); } Py_DECREF(v); } ScriptObject PythonEngine::eval(std::string_view sv) { std::string command{sv}; PyObject* v = PyRun_String(command.c_str(), Py_eval_input, m_globalDict, m_globalDict); if (v == nullptr) { PyErr_Print(); throw std::runtime_error("Error executing Python code"); } // share in ownership PythonObject return_value(v); // decref count returned from Python Py_DECREF(v); return ScriptObject{return_value}; } void* PythonEngine::getAs_impl(ScriptObject& obj, const std::type_info& ti) { auto val = std::any_cast(obj.object); void* return_value = nullptr; const auto& type_name = getRegisteredTypeName(ti); auto* type = SWIG_Python_TypeQuery(type_name.c_str()); if (!type) { throw std::runtime_error("Unable to find type in SWIG"); } const auto result = SWIG_Python_ConvertPtr(val.obj_, &return_value, type, 0); if (!SWIG_IsOK(result)) { throw std::runtime_error(fmt::format("Error getting object from SWIG/Python of supposed type {}, {}\n", ti.name(), type_name.c_str())); } return return_value; } bool PythonEngine::getAs_impl_bool(ScriptObject& obj) { auto val = std::any_cast(obj.object); if (!PyBool_Check(val.obj_)) { throw std::runtime_error("PyObject is not a bool"); } return static_cast(PyObject_IsTrue(val.obj_)); } int PythonEngine::getAs_impl_int(ScriptObject& obj) { auto val = std::any_cast(obj.object); if (!PyLong_Check(val.obj_)) { throw std::runtime_error("PyObject is not a PyLong"); } return static_cast(PyLong_AsLong(val.obj_)); } double PythonEngine::getAs_impl_double(ScriptObject& obj) { auto val = std::any_cast(obj.object); if (!PyFloat_Check(val.obj_)) { throw std::runtime_error("PyObject is not a PyFloat"); } return PyFloat_AsDouble(val.obj_); } std::string PythonEngine::getAs_impl_string(ScriptObject& obj) { auto val = std::any_cast(obj.object); if (!PyUnicode_Check(val.obj_)) { throw std::runtime_error("PyObject is not a String"); } Py_ssize_t size = 0; char const* pc = PyUnicode_AsUTF8AndSize(val.obj_, &size); // No decref needed if (!pc) { throw std::runtime_error("Unable to convert to std::string in SWIG Python"); } return std::string{pc, static_cast(size)}; } std::string PythonEngine::inferMeasureClassName(const openstudio::path& measureScriptPath) { auto inferClassNameCmd = fmt::format(R"python( import importlib.util import inspect spec = importlib.util.spec_from_file_location(f"throwaway", "{}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) class_members = inspect.getmembers(module, lambda x: inspect.isclass(x) and issubclass(x, openstudio.measure.OSMeasure)) assert len(class_members) == 1 measure_name, measure_typeinfo = class_members[0] )python", measureScriptPath.generic_string()); std::string className; try { exec(inferClassNameCmd); ScriptObject measureClassNameObject = eval("measure_name"); className = getAs(measureClassNameObject); } catch (const std::runtime_error& e) { auto msg = fmt::format("Failed to infer measure name from {}: {}", measureScriptPath.generic_string(), e.what()); fmt::print(stderr, "{}\n", msg); } return className; } // Ideally this would return a openstudio::measure::OSMeasure* or a shared_ptr but this poses memory management // issue for the underlying ScriptObject (and VALUE or PyObject), so just return the ScriptObject ScriptObject PythonEngine::loadMeasure(const openstudio::path& measureScriptPath, std::string_view className) { ScriptObject result; auto importCmd = fmt::format(R"python( import importlib.util spec = importlib.util.spec_from_file_location('{}', r'{}') module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) )python", className, measureScriptPath.generic_string()); // fmt::print("\nimportCmd:\n{}\n", importCmd); try { exec(importCmd); } catch (const std::runtime_error& e) { auto msg = fmt::format("Failed to load measure '{}' from '{}': {}", className, measureScriptPath.generic_string(), e.what()); fmt::print(stderr, "{}\n", msg); } try { result = eval(fmt::format("module.{}()", className)); } catch (const std::runtime_error& e) { auto msg = fmt::format("Failed to instantiate measure '{}' from '{}': {}", className, measureScriptPath.generic_string(), e.what()); fmt::print(stderr, "{}\n", msg); } return result; } int PythonEngine::numberOfArguments(ScriptObject& methodObject, std::string_view methodName) { int numberOfArguments = -1; auto val = std::any_cast(methodObject.object); if (PyObject_HasAttrString(val.obj_, methodName.data()) == 0) { // FAILED return numberOfArguments; } PyObject* method = PyObject_GetAttrString(val.obj_, methodName.data()); // New reference if (PyMethod_Check(method)) { PyObject* func = PyMethod_Function(method); // Borrowed if (auto* code = PyFunction_GetCode(func)) { // Borrowed // cppcheck-suppress cstyleCast const auto* co = (PyCodeObject*)code; numberOfArguments = co->co_argcount - 1; // This includes `self` } } else if (PyFunction_Check(method)) { // Shouldn't enter this block here if (auto* code = PyFunction_GetCode(method)) { // cppcheck-suppress cstyleCast const auto* co = (PyCodeObject*)code; numberOfArguments = co->co_argcount; } } Py_DECREF(method); return numberOfArguments; } bool PythonEngine::hasMethod(ScriptObject& methodObject, std::string_view methodName, bool overriden_only) { auto val = std::any_cast(methodObject.object); if (PyObject_HasAttrString(val.obj_, methodName.data()) == 0) { return false; } PyObject* method = PyObject_GetAttrString(val.obj_, methodName.data()); // New reference if (PyMethod_Check(method) == 0) { // Should never happen with modelOutputRequests since the Base class (C++) has it return false; } Py_DECREF(method); if (!overriden_only) { return true; } // equivalent to getattr(instance_obj.__class__, method_name) == getattr(instance_obj.__class__.__bases__[0], method_name) PyTypeObject* class_type = Py_TYPE(val.obj_); // PyObject_Type returns a strong (New) reference, not needed for us // cppcheck-suppress cstyleCast PyObject* class_method = PyObject_GetAttrString((PyObject*)class_type, methodName.data()); // New reference assert(class_type->tp_base != nullptr); // cppcheck-suppress cstyleCast auto* base = (PyTypeObject*)class_type->tp_base; // cppcheck-suppress cstyleCast PyObject* base_method = PyObject_GetAttrString((PyObject*)base, methodName.data()); // New reference bool result = class_method != base_method; Py_DECREF(class_method); Py_DECREF(base_method); return result; } } // namespace openstudio extern "C" { openstudio::ScriptEngine* makeScriptEngine(int argc, char* argv[]) { return new openstudio::PythonEngine(argc, argv); } }