Skip to content

Commit 39754d4

Browse files
timfeltimfel
committed
[GR-23351][GR-28624][GR-42663] remove intrinsified zipimport module
Since we cannot use java.zip.ZipFile (as that would escape the truffle sandbox), we would have to replicate the ZipFile code to deal with all zip features to a) pass the tests and b) improve performance for random access to files in the zipimport case. Switching to the stdlib zipimport.py (as CPython has done) will ensure a) and for larger zip files even b) immediately, with the option of improving peak performance later.
1 parent 66b6d10 commit 39754d4

File tree

15 files changed

+88
-1374
lines changed

15 files changed

+88
-1374
lines changed

‎graalpython/com.oracle.graal.python.frozen/freeze_modules.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, 2022, Oracle and/or its affiliates.
1+
# Copyright (c) 2021, 2023, Oracle and/or its affiliates.
22
# Copyright (C) 1996-2020 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -32,8 +32,7 @@
3232
# system.
3333
"importlib._bootstrap : _frozen_importlib",
3434
"importlib._bootstrap_external : _frozen_importlib_external",
35-
# CPython freezes zipimport, but we have it entirely intrinsified
36-
# 'zipimport',
35+
'zipimport',
3736
],
3837
),
3938
(
@@ -120,7 +119,6 @@ def add_graalpython_core():
120119
"java",
121120
"pip_hook",
122121
"unicodedata",
123-
"zipimport",
124122
]:
125123
modname = f"graalpy.{os.path.basename(name)}"
126124
modpath = os.path.join(lib_graalpython, f"{name}.py")
@@ -486,7 +484,7 @@ def lower_camel_case(str):
486484
# write frozen files
487485

488486
FROZEN_MODULES_HEADER = """/*
489-
* Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
487+
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
490488
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
491489
*
492490
* The Universal Permissive License (UPL), Version 1.0

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_zipimport.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,37 +9,65 @@
99
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testAFakeZlib
1010
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBadMTime
1111
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBadMagic
12+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBadMagic2
1213
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBeginningCruftAndComment
14+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBoth
1315
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testBytesPath
1416
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testComment
1517
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testDefaultOptimizationLevel
1618
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testDoctestFile
1719
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testDoctestSuite
1820
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testEmptyPy
21+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testGetCompiledSource
1922
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testGetData
2023
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testGetSource
2124
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testImport_WithStuff
25+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testImporterAttr
26+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testInvalidateCaches
2227
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testLargestPossibleComment
28+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testMixedNamespacePackage
29+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testNamespacePackage
30+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testPackage
2331
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testPy
32+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testPyc
33+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testSubNamespacePackage
34+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testSubPackage
2435
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testTraceback
36+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testUncheckedHashBasedPyc
2537
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testUnencodable
38+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testZipImporterMethods
39+
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.testZipImporterMethodsInSubDirectory
2640
*graalpython.lib-python.3.test.test_zipimport.CompressedZipImportTestCase.test_checked_hash_based_change_pyc
2741
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.test2038MTime
2842
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testAFakeZlib
2943
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBadMTime
3044
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBadMagic
45+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBadMagic2
3146
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBeginningCruftAndComment
47+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBoth
3248
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testBytesPath
3349
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testComment
3450
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testDefaultOptimizationLevel
3551
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testDoctestFile
3652
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testDoctestSuite
3753
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testEmptyPy
54+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testGetCompiledSource
3855
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testGetData
3956
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testGetSource
4057
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testImport_WithStuff
58+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testImporterAttr
59+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testInvalidateCaches
4160
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testLargestPossibleComment
61+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testMixedNamespacePackage
62+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testNamespacePackage
63+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testPackage
4264
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testPy
65+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testPyc
66+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testSubNamespacePackage
67+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testSubPackage
4368
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testTraceback
69+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testUncheckedHashBasedPyc
4470
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testUnencodable
71+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testZipImporterMethods
72+
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.testZipImporterMethodsInSubDirectory
4573
*graalpython.lib-python.3.test.test_zipimport.UncompressedZipImportTestCase.test_checked_hash_based_change_pyc

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import static com.oracle.graal.python.nodes.BuiltinNames.T_STDERR;
3434
import static com.oracle.graal.python.nodes.BuiltinNames.T_SYS;
3535
import static com.oracle.graal.python.nodes.BuiltinNames.T__WEAKREF;
36+
import static com.oracle.graal.python.nodes.BuiltinNames.T___BUILTINS__;
3637
import static com.oracle.graal.python.nodes.BuiltinNames.T___IMPORT__;
3738
import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___PACKAGE__;
3839
import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REPR__;
@@ -112,7 +113,6 @@
112113
import com.oracle.graal.python.builtins.modules.WarningsModuleBuiltins;
113114
import com.oracle.graal.python.builtins.modules.WeakRefModuleBuiltins;
114115
import com.oracle.graal.python.builtins.modules.WinregModuleBuiltins;
115-
import com.oracle.graal.python.builtins.modules.ZipImportModuleBuiltins;
116116
import com.oracle.graal.python.builtins.modules.ast.AstBuiltins;
117117
import com.oracle.graal.python.builtins.modules.ast.AstModuleBuiltins;
118118
import com.oracle.graal.python.builtins.modules.bz2.BZ2CompressorBuiltins;
@@ -221,12 +221,14 @@
221221
import com.oracle.graal.python.builtins.modules.zlib.ZlibCompressBuiltins;
222222
import com.oracle.graal.python.builtins.modules.zlib.ZlibDecompressBuiltins;
223223
import com.oracle.graal.python.builtins.objects.NotImplementedBuiltins;
224+
import com.oracle.graal.python.builtins.objects.PNone;
224225
import com.oracle.graal.python.builtins.objects.array.ArrayBuiltins;
225226
import com.oracle.graal.python.builtins.objects.bool.BoolBuiltins;
226227
import com.oracle.graal.python.builtins.objects.bytes.ByteArrayBuiltins;
227228
import com.oracle.graal.python.builtins.objects.bytes.BytesBuiltins;
228229
import com.oracle.graal.python.builtins.objects.cell.CellBuiltins;
229230
import com.oracle.graal.python.builtins.objects.code.CodeBuiltins;
231+
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
230232
import com.oracle.graal.python.builtins.objects.complex.ComplexBuiltins;
231233
import com.oracle.graal.python.builtins.objects.contextvars.ContextBuiltins;
232234
import com.oracle.graal.python.builtins.objects.contextvars.ContextVarBuiltins;
@@ -296,6 +298,7 @@
296298
import com.oracle.graal.python.builtins.objects.itertools.ZipLongestBuiltins;
297299
import com.oracle.graal.python.builtins.objects.keywrapper.KeyWrapperBuiltins;
298300
import com.oracle.graal.python.builtins.objects.list.ListBuiltins;
301+
import com.oracle.graal.python.builtins.objects.list.PList;
299302
import com.oracle.graal.python.builtins.objects.map.MapBuiltins;
300303
import com.oracle.graal.python.builtins.objects.mappingproxy.MappingproxyBuiltins;
301304
import com.oracle.graal.python.builtins.objects.memoryview.BufferBuiltins;
@@ -348,7 +351,6 @@
348351
import com.oracle.graal.python.builtins.objects.type.SpecialMethodSlot;
349352
import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
350353
import com.oracle.graal.python.builtins.objects.types.GenericAliasBuiltins;
351-
import com.oracle.graal.python.builtins.objects.zipimporter.ZipImporterBuiltins;
352354
import com.oracle.graal.python.lib.PyDictSetItem;
353355
import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
354356
import com.oracle.graal.python.lib.PyObjectLookupAttr;
@@ -362,6 +364,7 @@
362364
import com.oracle.graal.python.runtime.exception.PException;
363365
import com.oracle.graal.python.runtime.interop.PythonMapScope;
364366
import com.oracle.graal.python.runtime.object.PythonObjectSlowPathFactory;
367+
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
365368
import com.oracle.graal.python.util.Supplier;
366369
import com.oracle.truffle.api.CallTarget;
367370
import com.oracle.truffle.api.CompilerDirectives;
@@ -401,7 +404,6 @@ private static TruffleString[] initializeCoreFiles() {
401404
toTruffleStringUncached("_sre"),
402405
toTruffleStringUncached("function"),
403406
toTruffleStringUncached("_sysconfig"),
404-
PythonOptions.WITHOUT_COMPRESSION_LIBRARIES ? null : toTruffleStringUncached("zipimport"),
405407
toTruffleStringUncached("java"),
406408
toTruffleStringUncached("pip_hook"),
407409
toTruffleStringUncached("_struct")));
@@ -652,8 +654,6 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
652654
PythonOptions.WITHOUT_PLATFORM_ACCESS ? null : new PosixSubprocessModuleBuiltins(),
653655
new ReadlineModuleBuiltins(),
654656
new OperatorModuleBuiltins(),
655-
PythonOptions.WITHOUT_COMPRESSION_LIBRARIES ? null : new ZipImporterBuiltins(),
656-
PythonOptions.WITHOUT_COMPRESSION_LIBRARIES ? null : new ZipImportModuleBuiltins(),
657657

658658
// hashlib
659659
PythonOptions.WITHOUT_DIGEST ? null : new Md5ModuleBuiltins(),
@@ -903,6 +903,8 @@ private void initializeImportlib() {
903903
loadFile(toTruffleStringUncached("importlib/_bootstrap"), getContext().getStdlibHome(), bootstrap);
904904
} else {
905905
bootstrapExternal = ImpModuleBuiltins.importFrozenModuleObject(this, T__FROZEN_IMPORTLIB_EXTERNAL, true);
906+
setItem.execute(null, sysModules, T_IMPORTLIB_BOOTSTRAP, bootstrap);
907+
setItem.execute(null, sysModules, T_IMPORTLIB_BOOTSTRAP_EXTERNAL, bootstrapExternal);
906908
LOGGER.log(Level.FINE, () -> "import '" + T__FROZEN_IMPORTLIB + "' # <frozen>");
907909
LOGGER.log(Level.FINE, () -> "import '" + T__FROZEN_IMPORTLIB_EXTERNAL + "' # <frozen>");
908910
}
@@ -915,7 +917,51 @@ private void initializeImportlib() {
915917

916918
callNode.execute(null, bootstrap, toTruffleStringUncached("_install"), getSysModule(), lookupBuiltinModule(T__IMP));
917919
writeNode.execute(getBuiltins(), T___IMPORT__, readNode.execute(bootstrap, T___IMPORT__));
920+
// see CPython's init_importlib_external
918921
callNode.execute(null, bootstrap, toTruffleStringUncached("_install_external_importers"));
922+
if (!PythonOptions.WITHOUT_COMPRESSION_LIBRARIES) {
923+
// see CPython's _PyImportZip_Init
924+
Object pathHooks = readNode.execute(sysModule, toTruffleStringUncached("path_hooks"));
925+
if (!(pathHooks instanceof PList pathHooksList)) {
926+
LOGGER.log(Level.FINE, () -> "unable to get sys.path_hooks");
927+
LOGGER.log(Level.FINE, () -> "initializing zipimport failed");
928+
} else {
929+
LOGGER.log(Level.FINE, () -> "# installing zipimport hook");
930+
TruffleString t_zipimport = toTruffleStringUncached("zipimport");
931+
PythonModule zipimport = null;
932+
if (!ImageInfo.inImageBuildtimeCode()) {
933+
zipimport = ImpModuleBuiltins.importFrozenModuleObject(this, t_zipimport, false);
934+
}
935+
if (zipimport == null) {
936+
// true when the frozen module is not available
937+
zipimport = createModule(t_zipimport);
938+
try {
939+
loadFile(t_zipimport, getContext().getStdlibHome(), zipimport);
940+
} catch (PException e) {
941+
zipimport = null;
942+
removeBuiltinModule(t_zipimport);
943+
}
944+
} else {
945+
setItem.execute(null, sysModules, t_zipimport, zipimport);
946+
LOGGER.log(Level.FINE, () -> "import 'zipimport' # <frozen>");
947+
}
948+
if (zipimport == null) {
949+
LOGGER.log(Level.FINE, () -> "# can't import zipimport");
950+
} else {
951+
writeNode.execute(zipimport, T___BUILTINS__, getBuiltins());
952+
Object zipimporter = readNode.execute(zipimport, toTruffleStringUncached("zipimporter"));
953+
if (zipimporter == PNone.NO_VALUE) {
954+
LOGGER.log(Level.FINE, () -> "# can't import zipimport.zipimporter");
955+
} else {
956+
SequenceStorageNodes.InsertItemNode insertItem = SequenceStorageNodes.InsertItemNode.getUncached();
957+
SequenceStorage store = pathHooksList.getSequenceStorage();
958+
pathHooksList.setSequenceStorage(insertItem.execute(store, 0, zipimporter));
959+
LOGGER.log(Level.FINE, () -> "# installed zipimport hook");
960+
}
961+
}
962+
}
963+
}
964+
919965
importFunc = (PFunction) readNode.execute(bootstrap, T___IMPORT__);
920966
importlib = bootstrap;
921967

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,6 @@ public enum PythonBuiltinClassType implements TruffleObject {
159159
PGenericAlias("GenericAlias", J_TYPES, Flags.PUBLIC_BASE_WODICT),
160160
PUnionType("UnionType", J_TYPES, Flags.PUBLIC_DERIVED_WODICT),
161161
PZip("zip", J_BUILTINS),
162-
PZipImporter("zipimporter", "zipimport"),
163162
PBuffer("buffer", J_BUILTINS, Flags.PUBLIC_DERIVED_WODICT),
164163
PThread("start_new_thread", J__THREAD),
165164
PThreadLocal("_local", J__THREAD),
@@ -349,7 +348,6 @@ public enum PythonBuiltinClassType implements TruffleObject {
349348
PermissionError("PermissionError", J_BUILTINS, Flags.EXCEPTION),
350349
ProcessLookupError("ProcessLookupError", J_BUILTINS, Flags.EXCEPTION),
351350
TimeoutError("TimeoutError", J_BUILTINS, Flags.EXCEPTION),
352-
ZipImportError("ZipImportError", "zipimport", Flags.EXCEPTION),
353351
ZLibError("error", "zlib", Flags.EXCEPTION),
354352
CSVError("Error", "_csv", Flags.EXCEPTION),
355353
LZMAError("LZMAError", "_lzma", Flags.EXCEPTION),
@@ -655,7 +653,6 @@ public final Shape getInstanceShape(PythonLanguage lang) {
655653
PermissionError.base = OSError;
656654
ProcessLookupError.base = OSError;
657655
TimeoutError.base = OSError;
658-
ZipImportError.base = ImportError;
659656
ZLibError.base = Exception;
660657
CSVError.base = Exception;
661658
LZMAError.base = Exception;

0 commit comments

Comments
 (0)