From e8067c1a10fa48614980de75dbca3a57d1bf4606 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sun, 17 Dec 2017 15:05:01 -0500 Subject: [PATCH 01/10] Added docs about Embedding with an frozen module limitation. --- Doc/extending/embedding.rst | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index e64db3733440383..447a0098ed621ce 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -275,6 +275,50 @@ will depend on the details of the C++ system used; in general you will need to write the main program in C++, and use the C++ compiler to compile and link your program. There is no need to recompile Python itself using C++. +.. _freezingandembeddingmanually: + +Freezing Modules in Manually Embedded Python +============================================ + +While it is possible to use frozen modules in embedded python if the main module +is also frozen. It seems people are bitten when they don't have their main module +frozen and use ``PyRun_SimpleString`` that contains their main module(s) but +also depend on an frozen module (like the ``__hello__`` module). However if we +wanted to add our own module to the frozen list we would normally do this:: + + #include + #include + #include "mymodule.h" + + static const struct _frozen _PyImport_FrozenModules[] = { + /* importlib */ + {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, + {"_frozen_importlib_external", _Py_M__importlib_external, + (int)sizeof(_Py_M__importlib_external)}, + /* mymodule */ + {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, + {0, 0, 0} /* sentinel */ + }; + + const struct _frozen * PyImport_FrozenModules = _PyImport_FrozenModules; + +As you can see the above code will compile (with an warning on Windows +which will export ``PyImport_FrozenModules`` on the embedded python program). +This is not what we want. And if we were to run it with ``myprogram`` We will +get this traceback on running the string based main module(s). + +.. code-block:: py + + Traceback (most recent call last): + File "", line 1, in + ModuleNotFoundError: No module named 'mymodule' + +There is got to be a better way to have what we want but to also make python +aware of our ``mymodule`` being an frozen module. This module could be +anything from an import hook or anything else where you want to freeze +it similar to how ``importlib`` is frozen to support how your code base +currently is. This seems like an limitiation with using frozen modules in +embedded python since I have not found the actual fix for this senerio yet. .. _compiling: From a6e671bb41cf48912b421450a02a8a243a05a57f Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sun, 17 Dec 2017 16:35:30 -0500 Subject: [PATCH 02/10] bpo-32353: Document the fix to the common mistake with frozen modules. --- Doc/extending/embedding.rst | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 447a0098ed621ce..948c153373d0abc 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -317,8 +317,31 @@ There is got to be a better way to have what we want but to also make python aware of our ``mymodule`` being an frozen module. This module could be anything from an import hook or anything else where you want to freeze it similar to how ``importlib`` is frozen to support how your code base -currently is. This seems like an limitiation with using frozen modules in -embedded python since I have not found the actual fix for this senerio yet. +currently is. + +The fix to this is to change the C code above to:: + + #include + #include + #include "mymodule.h" + + static const struct _frozen _PyImport_FrozenModules[] = { + /* importlib */ + {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, + {"_frozen_importlib_external", _Py_M__importlib_external, + (int)sizeof(_Py_M__importlib_external)}, + /* mymodule */ + {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, + {0, 0, 0} /* sentinel */ + }; + +And then in the main() C or C++ function in your embedded interpreter add this line:: + + PyImport_FrozenModules = _PyImport_FrozenModules; + +Now your Embedded python *should* be able to load your frozen modules perfectly fine. + +.. note:: This logic was borrowed from Programs/_freeze_importlib.c .. _compiling: From 2d574364ad1747c46dbf95c00b89f9b429cda5b7 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sun, 17 Dec 2017 18:12:10 -0500 Subject: [PATCH 03/10] bpo-32353: fix whitespace. --- Doc/extending/embedding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 948c153373d0abc..40d2797c85a21da 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -334,7 +334,7 @@ The fix to this is to change the C code above to:: {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, {0, 0, 0} /* sentinel */ }; - + And then in the main() C or C++ function in your embedded interpreter add this line:: PyImport_FrozenModules = _PyImport_FrozenModules; From ef1bf23ffbf08b7ebf07cff9e15db42aa34d2d31 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sun, 17 Dec 2017 18:44:26 -0500 Subject: [PATCH 04/10] bpo-32353: Expand the note with more info. --- Doc/extending/embedding.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 40d2797c85a21da..08a0705c166ca43 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -341,7 +341,15 @@ And then in the main() C or C++ function in your embedded interpreter add this l Now your Embedded python *should* be able to load your frozen modules perfectly fine. -.. note:: This logic was borrowed from Programs/_freeze_importlib.c +.. note:: + This logic was borrowed from Programs/_freeze_importlib.c. Also using this method is + the only way to use frozen modules in Windows as well. On further note + Programs/_freeze_importlib.c in the cpython source tree (with minimal changes) could + be changed to act like the normal python freeze tool as the normal freeze tool + currently does not work on Windows so it makes it only possible to freeze to an + header file, make the frozen module list manually, write the embedded python + entry manually and then compile to have an fully featured embedded python interpreter + with your own frozen modules. .. _compiling: From 5d0c47f06eed000d9162e383eb66b6b9799d58f6 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Tue, 19 Dec 2017 19:57:55 -0500 Subject: [PATCH 05/10] bpo-32353: Add news entry. --- .../next/Documentation/2017-12-19-19-57-41.bpo-32353.cHpGHn.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2017-12-19-19-57-41.bpo-32353.cHpGHn.rst diff --git a/Misc/NEWS.d/next/Documentation/2017-12-19-19-57-41.bpo-32353.cHpGHn.rst b/Misc/NEWS.d/next/Documentation/2017-12-19-19-57-41.bpo-32353.cHpGHn.rst new file mode 100644 index 000000000000000..3131725c025b03b --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2017-12-19-19-57-41.bpo-32353.cHpGHn.rst @@ -0,0 +1,2 @@ +Added more documentation on manually embedding python with custom frozen +modules. From 70cbd9a6003a5efd3c7faa12b9244bb645099d68 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Tue, 19 Dec 2017 20:33:37 -0500 Subject: [PATCH 06/10] bpo-32353: Add yet another note. --- Doc/extending/embedding.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 08a0705c166ca43..06782b3becaef9f 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -302,6 +302,16 @@ wanted to add our own module to the frozen list we would normally do this:: const struct _frozen * PyImport_FrozenModules = _PyImport_FrozenModules; +.. note:: + The reason why this example includes ``importlib.h`` and + ``importlib_external.h`` is because of the fact that your embedded + interpreter might need them to run properly (load the python standard + library from an zip file or the sources like normally). Also in order + for your compiler to work you must point it to the Python subdirectory + in an active clone of the cpython repository or source tarball. + However it is not required to manually build the python core if you are + using an clone or source tarbar of an version instealled on your system. + As you can see the above code will compile (with an warning on Windows which will export ``PyImport_FrozenModules`` on the embedded python program). This is not what we want. And if we were to run it with ``myprogram`` We will From 2d5d20c2b983c362035b78a46cf931ad56fdbb29 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Tue, 19 Dec 2017 20:35:11 -0500 Subject: [PATCH 07/10] bpo-32353: fix typo. --- Doc/extending/embedding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 06782b3becaef9f..980957636574416 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -310,7 +310,7 @@ wanted to add our own module to the frozen list we would normally do this:: for your compiler to work you must point it to the Python subdirectory in an active clone of the cpython repository or source tarball. However it is not required to manually build the python core if you are - using an clone or source tarbar of an version instealled on your system. + using an clone or source tarbar of an version installed on your system. As you can see the above code will compile (with an warning on Windows which will export ``PyImport_FrozenModules`` on the embedded python program). From 5a6198c1e4a2e1b20190d0ffe5e8335bada8ca29 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Fri, 12 Aug 2022 20:56:32 -0400 Subject: [PATCH 08/10] Update Doc/extending/embedding.rst --- Doc/extending/embedding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 980957636574416..33f7d86513d26c0 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -310,7 +310,7 @@ wanted to add our own module to the frozen list we would normally do this:: for your compiler to work you must point it to the Python subdirectory in an active clone of the cpython repository or source tarball. However it is not required to manually build the python core if you are - using an clone or source tarbar of an version installed on your system. + using an clone or source tarball of an version installed on your system. As you can see the above code will compile (with an warning on Windows which will export ``PyImport_FrozenModules`` on the embedded python program). From 959394de24b900419a8d5a15f5c81bf99afcaf82 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sat, 22 Oct 2022 09:38:58 -0400 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Stanley <46876382+slateny@users.noreply.github.com> --- Doc/extending/embedding.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 33f7d86513d26c0..291c5b3254b97d2 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -280,7 +280,7 @@ program. There is no need to recompile Python itself using C++. Freezing Modules in Manually Embedded Python ============================================ -While it is possible to use frozen modules in embedded python if the main module +It is possible to use frozen modules in embedded Python if the main module is also frozen. It seems people are bitten when they don't have their main module frozen and use ``PyRun_SimpleString`` that contains their main module(s) but also depend on an frozen module (like the ``__hello__`` module). However if we @@ -304,8 +304,8 @@ wanted to add our own module to the frozen list we would normally do this:: .. note:: The reason why this example includes ``importlib.h`` and - ``importlib_external.h`` is because of the fact that your embedded - interpreter might need them to run properly (load the python standard + ``importlib_external.h`` is because of the fact that the embedded + interpreter might need them to run properly (load the Python standard library from an zip file or the sources like normally). Also in order for your compiler to work you must point it to the Python subdirectory in an active clone of the cpython repository or source tarball. @@ -314,8 +314,8 @@ wanted to add our own module to the frozen list we would normally do this:: As you can see the above code will compile (with an warning on Windows which will export ``PyImport_FrozenModules`` on the embedded python program). -This is not what we want. And if we were to run it with ``myprogram`` We will -get this traceback on running the string based main module(s). +This is not what we want. And if we were to run it with ``myprogram`` we will +get the following traceback on running the string-based main module(s). .. code-block:: py @@ -349,7 +349,7 @@ And then in the main() C or C++ function in your embedded interpreter add this l PyImport_FrozenModules = _PyImport_FrozenModules; -Now your Embedded python *should* be able to load your frozen modules perfectly fine. +Now your embedded Python should be able to load your frozen modules perfectly fine. .. note:: This logic was borrowed from Programs/_freeze_importlib.c. Also using this method is From ff907a0fc1bb7f5f1dd4ad8b58ef0a059ffee178 Mon Sep 17 00:00:00 2001 From: AraHaan Date: Sun, 23 Oct 2022 10:28:17 -0400 Subject: [PATCH 10/10] Apply suggestions from code review --- Doc/extending/embedding.rst | 64 ++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 291c5b3254b97d2..207c62fcc06e080 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -286,21 +286,21 @@ frozen and use ``PyRun_SimpleString`` that contains their main module(s) but also depend on an frozen module (like the ``__hello__`` module). However if we wanted to add our own module to the frozen list we would normally do this:: - #include - #include - #include "mymodule.h" - - static const struct _frozen _PyImport_FrozenModules[] = { - /* importlib */ - {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, - {"_frozen_importlib_external", _Py_M__importlib_external, - (int)sizeof(_Py_M__importlib_external)}, - /* mymodule */ - {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, - {0, 0, 0} /* sentinel */ - }; - - const struct _frozen * PyImport_FrozenModules = _PyImport_FrozenModules; + #include + #include + #include "mymodule.h" + + static const struct _frozen _PyImport_FrozenModules[] = { + /* importlib */ + {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, + {"_frozen_importlib_external", _Py_M__importlib_external, + (int)sizeof(_Py_M__importlib_external)}, + /* mymodule */ + {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, + {0, 0, 0} /* sentinel */ + }; + + const struct _frozen * PyImport_FrozenModules = _PyImport_FrozenModules; .. note:: The reason why this example includes ``importlib.h`` and @@ -319,9 +319,9 @@ get the following traceback on running the string-based main module(s). .. code-block:: py - Traceback (most recent call last): - File "", line 1, in - ModuleNotFoundError: No module named 'mymodule' + Traceback (most recent call last): + File "", line 1, in + ModuleNotFoundError: No module named 'mymodule' There is got to be a better way to have what we want but to also make python aware of our ``mymodule`` being an frozen module. This module could be @@ -331,23 +331,23 @@ currently is. The fix to this is to change the C code above to:: - #include - #include - #include "mymodule.h" - - static const struct _frozen _PyImport_FrozenModules[] = { - /* importlib */ - {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, - {"_frozen_importlib_external", _Py_M__importlib_external, - (int)sizeof(_Py_M__importlib_external)}, - /* mymodule */ - {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, - {0, 0, 0} /* sentinel */ - }; + #include + #include + #include "mymodule.h" + + static const struct _frozen _PyImport_FrozenModules[] = { + /* importlib */ + {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)}, + {"_frozen_importlib_external", _Py_M__importlib_external, + (int)sizeof(_Py_M__importlib_external)}, + /* mymodule */ + {"mymodule", M_mymodule, (int)sizeof(M_mymodule)}, + {0, 0, 0} /* sentinel */ + }; And then in the main() C or C++ function in your embedded interpreter add this line:: - PyImport_FrozenModules = _PyImport_FrozenModules; + PyImport_FrozenModules = _PyImport_FrozenModules; Now your embedded Python should be able to load your frozen modules perfectly fine.