Skip to content

Commit a2d8c4b

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45019: Do some cleanup related to frozen modules. (pythongh-28319)
There are a few things I missed in pythongh-27980. This is a follow-up that will make subsequent PRs cleaner. It includes fixes to tests and tools that reference the frozen modules. https://bugs.python.org/issue45019
1 parent 1fc41ae commit a2d8c4b

12 files changed

Lines changed: 339 additions & 137 deletions

File tree

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ Objects/clinic/*.h linguist-generated=true
4747
PC/clinic/*.h linguist-generated=true
4848
Python/clinic/*.h linguist-generated=true
4949
Python/frozen_modules/*.h linguist-generated=true
50+
Python/frozen_modules/MANIFEST linguist-generated=true
5051
Include/internal/pycore_ast.h linguist-generated=true
5152
Python/Python-ast.c linguist-generated=true
5253
Include/opcode.h linguist-generated=true

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,13 @@ Tools/msi/obj
121121
Tools/ssl/amd64
122122
Tools/ssl/win32
123123

124+
# TODO: Once we auto-regen frozem modules for Windows builds
125+
# we can drop the .h files from the repo and ignore them here.
126+
# At that point we will rely the frozen manifest file to identify
127+
# changed generated files. We'll drop the entry for it then.
128+
# See: Tools/scripts/freeze_modules.py.
129+
#Python/frozen_modules/*.h
130+
124131
# Two-trick pony for OSX and other case insensitive file systems:
125132
# Ignore ./python binary on Unix but still look into ./Python/ directory.
126133
/python

‎Lib/ctypes/test/test_values.py‎

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
A testcase which accesses *values* in a dll.
33
"""
44

5+
import imp
6+
import importlib.util
57
import unittest
68
import sys
79
from ctypes import *
10+
from test.support import import_helper, captured_stdout
811

912
import _ctypes_test
1013

@@ -55,41 +58,32 @@ class struct_frozen(Structure):
5558

5659
ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
5760
# ft is a pointer to the struct_frozen entries:
58-
items = []
59-
# _frozen_importlib changes size whenever importlib._bootstrap
60-
# changes, so it gets a special case. We should make sure it's
61-
# found, but don't worry about its size too much. The same
62-
# applies to _frozen_importlib_external.
63-
bootstrap_seen = []
64-
bootstrap_expected = [
65-
b'_frozen_importlib',
66-
b'_frozen_importlib_external',
67-
b'zipimport',
68-
]
61+
modules = []
6962
for entry in ft:
7063
# This is dangerous. We *can* iterate over a pointer, but
7164
# the loop will not terminate (maybe with an access
7265
# violation;-) because the pointer instance has no size.
7366
if entry.name is None:
7467
break
75-
76-
if entry.name in bootstrap_expected:
77-
bootstrap_seen.append(entry.name)
78-
self.assertTrue(entry.size,
79-
"{!r} was reported as having no size".format(entry.name))
80-
continue
81-
items.append((entry.name.decode("ascii"), entry.size))
82-
83-
expected = [("__hello__", 164),
84-
("__phello__", -164),
85-
("__phello__.spam", 164),
86-
]
87-
self.assertEqual(items, expected, "PyImport_FrozenModules example "
68+
modname = entry.name.decode("ascii")
69+
modules.append(modname)
70+
with self.subTest(modname):
71+
# Do a sanity check on entry.size and entry.code.
72+
self.assertGreater(abs(entry.size), 10)
73+
self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
74+
# Check the module's package-ness.
75+
spec = importlib.util.find_spec(modname)
76+
if entry.size < 0:
77+
# It's a package.
78+
self.assertIsNotNone(spec.submodule_search_locations)
79+
else:
80+
self.assertIsNone(spec.submodule_search_locations)
81+
82+
expected = imp._frozen_module_names()
83+
self.maxDiff = None
84+
self.assertEqual(modules, expected, "PyImport_FrozenModules example "
8885
"in Doc/library/ctypes.rst may be out of date")
8986

90-
self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
91-
"frozen bootstrap modules did not match PyImport_FrozenModules")
92-
9387
from ctypes import _pointer_type_cache
9488
del _pointer_type_cache[struct_frozen]
9589

‎Lib/imp.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from _imp import (lock_held, acquire_lock, release_lock,
1010
get_frozen_object, is_frozen_package,
1111
init_frozen, is_builtin, is_frozen,
12-
_fix_co_filename)
12+
_fix_co_filename, _frozen_module_names)
1313
try:
1414
from _imp import create_dynamic
1515
except ImportError:

‎Makefile.pre.in‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -750,22 +750,22 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
750750

751751
# BEGIN: freezing modules
752752

753-
Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py
753+
Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
754754
$(srcdir)/Programs/_freeze_module importlib._bootstrap \
755755
$(srcdir)/Lib/importlib/_bootstrap.py \
756756
$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
757757

758-
Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py
758+
Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
759759
$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
760760
$(srcdir)/Lib/importlib/_bootstrap_external.py \
761761
$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
762762

763-
Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py
763+
Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
764764
$(srcdir)/Programs/_freeze_module zipimport \
765765
$(srcdir)/Lib/zipimport.py \
766766
$(srcdir)/Python/frozen_modules/zipimport.h
767767

768-
Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py
768+
Python/frozen_modules/hello.h: Programs/_freeze_module Tools/freeze/flag.py
769769
$(srcdir)/Programs/_freeze_module hello \
770770
$(srcdir)/Tools/freeze/flag.py \
771771
$(srcdir)/Python/frozen_modules/hello.h

‎Python/clinic/import.c.h‎

Lines changed: 19 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/frozen.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
/* Note that a negative size indicates a package. */
4848

4949
static const struct _frozen _PyImport_FrozenModules[] = {
50-
/* importlib */
50+
/* import system */
5151
{"_frozen_importlib", _Py_M__importlib__bootstrap,
5252
(int)sizeof(_Py_M__importlib__bootstrap)},
5353
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,

‎Python/frozen_modules/MANIFEST‎

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/frozen_modules/README.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
This directory contains the generated .h files for all the frozen
2+
modules. Python/frozen.c depends on these files.
3+
4+
Note that, other than the required frozen modules, none of these files
5+
are committed into the repo.
6+
7+
See Tools/scripts/freeze_modules.py for more info.

‎Python/import.c‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "code.h"
1717
#include "importdl.h"
1818
#include "pydtrace.h"
19+
#include <stdbool.h>
1920

2021
#ifdef HAVE_FCNTL_H
2122
#include <fcntl.h>
@@ -1049,6 +1050,32 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
10491050

10501051
/* Frozen modules */
10511052

1053+
static PyObject *
1054+
list_frozen_module_names(bool force)
1055+
{
1056+
PyObject *names = PyList_New(0);
1057+
if (names == NULL) {
1058+
return NULL;
1059+
}
1060+
for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
1061+
if (p->name == NULL) {
1062+
break;
1063+
}
1064+
PyObject *name = PyUnicode_FromString(p->name);
1065+
if (name == NULL) {
1066+
Py_DECREF(names);
1067+
return NULL;
1068+
}
1069+
int res = PyList_Append(names, name);
1070+
Py_DECREF(name);
1071+
if (res != 0) {
1072+
Py_DECREF(names);
1073+
return NULL;
1074+
}
1075+
}
1076+
return names;
1077+
}
1078+
10521079
static const struct _frozen *
10531080
find_frozen(PyObject *name)
10541081
{
@@ -1954,6 +1981,19 @@ _imp_is_frozen_impl(PyObject *module, PyObject *name)
19541981
return PyBool_FromLong((long) (p == NULL ? 0 : p->size));
19551982
}
19561983

1984+
/*[clinic input]
1985+
_imp._frozen_module_names
1986+
1987+
Returns the list of available frozen modules.
1988+
[clinic start generated code]*/
1989+
1990+
static PyObject *
1991+
_imp__frozen_module_names_impl(PyObject *module)
1992+
/*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/
1993+
{
1994+
return list_frozen_module_names(true);
1995+
}
1996+
19571997
/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */
19581998
static int
19591999
exec_builtin_or_dynamic(PyObject *mod) {
@@ -2114,6 +2154,7 @@ static PyMethodDef imp_methods[] = {
21142154
_IMP_INIT_FROZEN_METHODDEF
21152155
_IMP_IS_BUILTIN_METHODDEF
21162156
_IMP_IS_FROZEN_METHODDEF
2157+
_IMP__FROZEN_MODULE_NAMES_METHODDEF
21172158
_IMP_CREATE_DYNAMIC_METHODDEF
21182159
_IMP_EXEC_DYNAMIC_METHODDEF
21192160
_IMP_EXEC_BUILTIN_METHODDEF

0 commit comments

Comments
 (0)