Skip to content

Commit e67f48c

Browse files
pitroupitrou
committed
Issue python#14928: Fix importlib bootstrap issues by using a custom executable (Modules/_freeze_importlib) to build Python/importlib.h.
1 parent 0006aac commit e67f48c

File tree

8 files changed

+806
-697
lines changed

8 files changed

+806
-697
lines changed

‎.hgignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ PCbuild/x64-temp-*
7979
PCbuild/amd64
8080
BuildLog.htm
8181
__pycache__
82+
Modules/_freeze_importlib
8283
Modules/_testembed
8384
.coverage
8485
coverage/

‎Include/pythonrun.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
3030

3131
PyAPI_FUNC(void) Py_Initialize(void);
3232
PyAPI_FUNC(void) Py_InitializeEx(int);
33+
#ifndef Py_LIMITED_API
34+
PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
35+
#endif
3336
PyAPI_FUNC(void) Py_Finalize(void);
3437
PyAPI_FUNC(int) Py_IsInitialized(void);
3538
PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);

‎Makefile.pre.in‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,6 @@ PYTHON_OBJS= \
324324
Python/codecs.o \
325325
Python/dynamic_annotations.o \
326326
Python/errors.o \
327-
Python/frozen.o \
328327
Python/frozenmain.o \
329328
Python/future.o \
330329
Python/getargs.o \
@@ -410,7 +409,7 @@ SYSCONFIGDATA=$(srcdir)/Lib/_sysconfigdata.py
410409

411410
##########################################################################
412411
# objects that get linked into the Python library
413-
LIBRARY_OBJS= \
412+
LIBRARY_OBJS_OMIT_FROZEN= \
414413
Modules/getbuildinfo.o \
415414
$(PARSER_OBJS) \
416415
$(OBJECT_OBJS) \
@@ -419,6 +418,10 @@ LIBRARY_OBJS= \
419418
$(SIGNAL_OBJS) \
420419
$(MODOBJS)
421420

421+
LIBRARY_OBJS= \
422+
$(LIBRARY_OBJS_OMIT_FROZEN) \
423+
Python/frozen.o
424+
422425
#########################################################################
423426
# Rules
424427

@@ -478,7 +481,7 @@ $(LIBRARY): $(LIBRARY_OBJS)
478481
$(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o
479482
$(AR) $(ARFLAGS) $@ $(PARSER_OBJS)
480483
$(AR) $(ARFLAGS) $@ $(OBJECT_OBJS)
481-
$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS)
484+
$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o
482485
$(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS)
483486
$(AR) $(ARFLAGS) $@ $(MODOBJS)
484487
$(RANLIB) $@
@@ -578,18 +581,14 @@ Modules/_testembed: Modules/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
578581
############################################################################
579582
# Importlib
580583

581-
Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/freeze_importlib.py
582-
@if test -f ./$(BUILDPYTHON); then \
583-
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Python/freeze_importlib.py \
584-
$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h; \
585-
else \
586-
echo "----------------------------------------------------------"; \
587-
echo "Python/importlib.h needs to be rebuilt, but no interpreter"; \
588-
echo "is available to do so. Leaving the previous version in"; \
589-
echo "place. You may want to run ''make'' a second time after"; \
590-
echo "this build is complete."; \
591-
echo "----------------------------------------------------------"; \
592-
fi
584+
Modules/_freeze_importlib: Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
585+
$(LINKCC) $(PY_LDFLAGS) -o $@ Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
586+
587+
Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
588+
$(MAKE) Modules/_freeze_importlib
589+
./Modules/_freeze_importlib \
590+
$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
591+
593592

594593
############################################################################
595594
# Special rules for object files
@@ -1389,7 +1388,7 @@ clean: pycremoval
13891388
find build -name 'fficonfig.py' -exec rm -f {} ';' || true
13901389
-rm -f Lib/lib2to3/*Grammar*.pickle
13911390
-rm -f $(SYSCONFIGDATA)
1392-
-rm -f Modules/_testembed
1391+
-rm -f Modules/_testembed Modules/_freeze_importlib
13931392

13941393
profile-removal:
13951394
find . -name '*.gc??' -exec rm -f {} ';'

‎Misc/NEWS‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,12 @@ Tests
154154
- Issue #14963: Add test cases for exception handling behaviour
155155
in contextlib.ExitStack (Initial patch by Alon Horev)
156156

157+
Build
158+
-----
159+
160+
- Issue #14928: Fix importlib bootstrap issues by using a custom executable
161+
(Modules/_freeze_importlib) to build Python/importlib.h.
162+
157163

158164
What's New in Python 3.3.0 Alpha 4?
159165
===================================

‎Modules/_freeze_importlib.c‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/* This is built as a stand-alone executable by the Makefile, and helps turn
2+
Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h
3+
*/
4+
5+
#include <Python.h>
6+
#include <marshal.h>
7+
8+
#include <stdio.h>
9+
#include <sys/types.h>
10+
#include <sys/stat.h>
11+
#include <unistd.h>
12+
13+
14+
/* To avoid a circular dependency on frozen.o, we create our own structure
15+
of frozen modules instead, left deliberately blank so as to avoid
16+
unintentional import of a stale version of _frozen_importlib. */
17+
18+
static struct _frozen _PyImport_FrozenModules[] = {
19+
{0, 0, 0} /* sentinel */
20+
};
21+
22+
struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
23+
24+
25+
const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */";
26+
27+
int
28+
main(int argc, char *argv[])
29+
{
30+
char *inpath, *outpath;
31+
FILE *infile, *outfile = NULL;
32+
struct stat st;
33+
size_t text_size, data_size, n;
34+
char *text, *data;
35+
PyObject *code, *marshalled;
36+
37+
if (argc != 3) {
38+
fprintf(stderr, "need to specify input and output paths\n");
39+
return 2;
40+
}
41+
inpath = argv[1];
42+
outpath = argv[2];
43+
infile = fopen(inpath, "rb");
44+
if (infile == NULL) {
45+
fprintf(stderr, "cannot open '%s' for reading\n", inpath);
46+
return 1;
47+
}
48+
if (fstat(fileno(infile), &st)) {
49+
fclose(infile);
50+
fprintf(stderr, "cannot fstat '%s'\n", inpath);
51+
return 1;
52+
}
53+
text_size = st.st_size;
54+
text = (char *) malloc(text_size + 1);
55+
if (text == NULL) {
56+
fclose(infile);
57+
fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size);
58+
return 1;
59+
}
60+
n = fread(text, 1, text_size, infile);
61+
fclose(infile);
62+
infile = NULL;
63+
if (n < text_size) {
64+
fprintf(stderr, "read too short: got %ld instead of %ld bytes\n",
65+
(long) n, (long) text_size);
66+
return 1;
67+
}
68+
text[text_size] = '\0';
69+
70+
Py_NoUserSiteDirectory++;
71+
Py_NoSiteFlag++;
72+
Py_IgnoreEnvironmentFlag++;
73+
74+
Py_SetProgramName(L"./_freeze_importlib");
75+
/* Don't install importlib, since it could execute outdated bytecode. */
76+
_Py_InitializeEx_Private(1, 0);
77+
78+
code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
79+
Py_file_input, NULL, 0);
80+
if (code == NULL)
81+
goto error;
82+
marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION);
83+
Py_DECREF(code);
84+
if (marshalled == NULL)
85+
goto error;
86+
87+
assert(PyBytes_CheckExact(marshalled));
88+
data = PyBytes_AS_STRING(marshalled);
89+
data_size = PyBytes_GET_SIZE(marshalled);
90+
91+
outfile = fopen(outpath, "wb");
92+
if (outfile == NULL) {
93+
fprintf(stderr, "cannot open '%s' for writing\n", outpath);
94+
return 1;
95+
}
96+
fprintf(outfile, "%s\n", header);
97+
fprintf(outfile, "unsigned char _Py_M__importlib[] = {\n");
98+
for (n = 0; n < data_size; n += 16) {
99+
size_t i, end = Py_MIN(n + 16, data_size);
100+
fprintf(outfile, " ");
101+
for (i = n; i < end; i++) {
102+
fprintf(outfile, "%d,", (int) data[i]);
103+
}
104+
fprintf(outfile, "\n");
105+
}
106+
fprintf(outfile, "};\n");
107+
108+
Py_DECREF(marshalled);
109+
110+
Py_Finalize();
111+
if (infile)
112+
fclose(infile);
113+
if (outfile) {
114+
if (ferror(outfile)) {
115+
fprintf(stderr, "error when writing to '%s'\n", outpath);
116+
fclose(outfile);
117+
return 1;
118+
}
119+
fclose(outfile);
120+
}
121+
return 0;
122+
123+
error:
124+
PyErr_Print();
125+
Py_Finalize();
126+
if (infile)
127+
fclose(infile);
128+
if (outfile)
129+
fclose(outfile);
130+
return 1;
131+
}

‎Python/freeze_importlib.py‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

0 commit comments

Comments
 (0)