Skip to content

gh-117953: Track Extra Details in Global Extensions Cache #118532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ericsnowcurrently merged 24 commits into from
May 4, 2024
Merged

gh-117953: Track Extra Details in Global Extensions Cache #118532

Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1fc1149
Store the cached def in a wrapping struct rather than directly.
ericsnowcurrently Apr 30, 2024
254e1f1
Pass the cached value around where appropriate.
ericsnowcurrently May 2, 2024
0694318
Pass the cached value to check_singlephase().
ericsnowcurrently May 1, 2024
9ca330b
Add extensions_cache_value helpers.
ericsnowcurrently Apr 30, 2024
7e0c4ee
Add get_cached_m_dict().
ericsnowcurrently Apr 30, 2024
c1c1379
Track ext module origin.
ericsnowcurrently May 1, 2024
8485fb5
Look up core module dicts in get_cached_m_dict().
ericsnowcurrently May 1, 2024
abbddde
Pass m_dict to _extensions_cache_set().
ericsnowcurrently May 1, 2024
12bcd96
Pass m_init to _extensions_cache_set().
ericsnowcurrently May 1, 2024
2d56ada
Add some comments.
ericsnowcurrently May 2, 2024
9c70d53
Use del_cached_m_dict() in set_cached_m_dict().
ericsnowcurrently May 2, 2024
0098057
Add m_init and m_dict fields to extensions_cache_value.
ericsnowcurrently May 1, 2024
57aaba2
Track which interpreter was used to create m_dict.
ericsnowcurrently May 1, 2024
96932eb
Track the interpreter ID.
ericsnowcurrently May 1, 2024
a4cb134
Look up core module dicts in get_cached_m_dict().
ericsnowcurrently May 1, 2024
c4d3be1
Make sure we only set m_dict for non-isolated interpreters.
ericsnowcurrently May 1, 2024
d87584d
Simplify.
ericsnowcurrently May 3, 2024
336d1da
Fix a comment.
ericsnowcurrently May 3, 2024
bd1e134
Separate the def logic from the rest of the extensions cache code.
ericsnowcurrently May 3, 2024
e42e797
Add the per-interpreter cache index to each global cache entry.
ericsnowcurrently May 3, 2024
faa4254
Fix an assert.
ericsnowcurrently May 4, 2024
5276981
Properly sync the cached m_index and the def m_index.
ericsnowcurrently May 3, 2024
f9a2a05
Merge branch 'main' into extensions-cache-better-tracking
ericsnowcurrently May 4, 2024
7d3dbc1
Fix make smelly.
ericsnowcurrently May 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add the per-interpreter cache index to each global cache entry.
  • Loading branch information
@ericsnowcurrently
ericsnowcurrently committed May 3, 2024
commit e42e797a7daf572a0a8edd8d77230d6efca5290e
21 changes: 18 additions & 3 deletions Python/import.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,11 @@ struct extensions_cache_value {
It is set by update_global_state_for_extension(). */
PyModInitFunction m_init;

/* The module's index into its interpreter's modules_by_index cache.
This is set for all extension modules but only used for legacy ones.
(See PyInterpreterState.modules_by_index for more info.) */
Py_ssize_t m_index;

/* A copy of the module's __dict__ after the first time it was loaded.
This is only set/used for legacy modules that do not support
multiple initializations.
Expand Down Expand Up @@ -1261,7 +1266,8 @@ _extensions_cache_get(PyObject *path, PyObject *name)
static int
_extensions_cache_set(PyObject *path, PyObject *name,
PyModuleDef *def, PyModInitFunction m_init,
PyObject *m_dict, _Py_ext_module_origin origin)
Py_ssize_t m_index, PyObject *m_dict,
_Py_ext_module_origin origin)
{
int res = -1;
assert(def != NULL);
Expand Down Expand Up @@ -1309,6 +1315,7 @@ _extensions_cache_set(PyObject *path, PyObject *name,
*newvalue = (struct extensions_cache_value){
.def=def,
.m_init=m_init,
.m_index=m_index,
/* m_dict is set by set_cached_m_dict(). */
.origin=origin,
};
Expand Down Expand Up @@ -1531,8 +1538,9 @@ _get_extension_kind(PyModuleDef *def, bool check_size)


struct singlephase_global_update {
PyObject *m_dict;
PyModInitFunction m_init;
Py_ssize_t m_index;
PyObject *m_dict;
_Py_ext_module_origin origin;
};

Expand Down Expand Up @@ -1591,7 +1599,8 @@ update_global_state_for_extension(PyThreadState *tstate,
assert(cached == NULL || cached->def == def);
#endif
if (_extensions_cache_set(
path, name, def, m_init, m_dict, singlephase->origin) < 0)
path, name, def, m_init, singlephase->m_index, m_dict,
singlephase->origin) < 0)
{
// XXX Ignore this error? Doing so would effectively
// mark the module as not loadable.
Expand Down Expand Up @@ -1817,6 +1826,10 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0,

/* Update global import state. */
struct singlephase_global_update singlephase = {
// XXX Modules that share a def should each get their own index,
// whereas currently they share (which means the per-interpreter
// cache is less reliable than it should be).
.m_index=def->m_base.m_index,
.origin=info->origin,
};
// gh-88216: Extensions and def->m_base.m_copy can be updated
Expand Down Expand Up @@ -1921,13 +1934,15 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name,
assert_singlephase_def(def);
assert(def->m_size == -1);
assert(def->m_base.m_copy == NULL);
assert(def->m_base.m_index >= 0);

/* We aren't using import_find_extension() for core modules,
* so we have to do the extra check to make sure the module
* isn't already in the global cache before calling
* update_global_state_for_extension(). */
if (_extensions_cache_get(nameobj, nameobj) == NULL) {
struct singlephase_global_update singlephase = {
.m_index=def->m_base.m_index,
/* We don't want def->m_base.m_copy populated. */
.m_dict=NULL,
.origin=_Py_ext_module_origin_CORE,
Expand Down