Skip to content

Commit c0564ee

Browse files
msimacekmsimacek
committed
Support lazy loading of source texts on DSL interpreter
1 parent faa8ffe commit c0564ee

11 files changed

Lines changed: 127 additions & 145 deletions

File tree

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,15 @@ public void testInspectJavaArray() throws Throwable {
550550

551551
@Test
552552
public void testSourceFileURI() throws Throwable {
553+
testSourceFileURIImpl(false);
554+
}
555+
556+
@Test
557+
public void testSourceFileURIBytecode() throws Throwable {
558+
testSourceFileURIImpl(true);
559+
}
560+
561+
private void testSourceFileURIImpl(boolean runFromBytecode) throws Throwable {
553562
if (System.getProperty("os.name").toLowerCase().contains("mac")) {
554563
// on the mac machines we run with symlinked directories and such, and it's annoying to
555564
// cater for that
@@ -565,6 +574,11 @@ public void testSourceFileURI() throws Throwable {
565574
"sys.path.insert(0, '" + tempDir.toString() + "')\n" +
566575
"import imported\n" +
567576
"imported.sum(2, 3)\n").getBytes());
577+
578+
if (runFromBytecode) {
579+
compileToBytecode(importedFile, importingFile);
580+
}
581+
568582
Source source = Source.newBuilder("python", importingFile.toFile()).build();
569583
try (DebuggerSession session = tester.startSession()) {
570584
Breakpoint breakpoint = Breakpoint.newBuilder(importingFile.toUri()).lineIs(4).build();
@@ -602,6 +616,16 @@ public void testSourceFileURI() throws Throwable {
602616
}
603617
}
604618

619+
private void compileToBytecode(Path... files) {
620+
StringBuilder sourceCode = new StringBuilder("import py_compile\n");
621+
for (Path file : files) {
622+
sourceCode.append("py_compile.compile(r\"").append(file).append("\")\n");
623+
}
624+
Source compileSource = Source.newBuilder("python", sourceCode.toString(), "compile_source_uri.py").buildLiteral();
625+
tester.startEval(compileSource);
626+
tester.expectDone();
627+
}
628+
605629
@Test
606630
public void testInlineEvaluationBreakpointBuiltin() throws Throwable {
607631
final Source source = Source.newBuilder("python", """

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java‎

Lines changed: 32 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727

2828
import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_WIN32;
2929
import static com.oracle.graal.python.nodes.BuiltinNames.T__SIGNAL;
30-
import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION;
3130
import static com.oracle.graal.python.nodes.StringLiterals.T_PY_EXTENSION;
3231
import static com.oracle.graal.python.nodes.truffle.TruffleStringMigrationHelpers.isJavaString;
3332
import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR;
@@ -110,7 +109,6 @@
110109
import com.oracle.graal.python.runtime.exception.PException;
111110
import com.oracle.graal.python.runtime.object.PFactory;
112111
import com.oracle.graal.python.util.Function;
113-
import com.oracle.graal.python.util.LazySource;
114112
import com.oracle.graal.python.util.PythonUtils;
115113
import com.oracle.graal.python.util.Supplier;
116114
import com.oracle.truffle.api.Assumption;
@@ -549,53 +547,16 @@ protected CallTarget parse(ParsingRequest request) {
549547
}
550548

551549
public static RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) {
552-
boolean internal = shouldMarkSourceInternal(context);
553-
SourceBuilder builder = null;
554-
// The original file path should be passed as the name
555-
String name = source.getName();
556-
if (name != null && !name.isEmpty()) {
557-
builder = sourceForOriginalFile(context, code, internal, name);
558-
if (builder == null) {
559-
if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) {
560-
String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length());
561-
String fs = context.getEnv().getFileNameSeparator();
562-
String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION;
563-
builder = sourceForOriginalFile(context, code, internal, path);
564-
if (builder == null) {
565-
path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py";
566-
builder = sourceForOriginalFile(context, code, internal, path);
567-
}
568-
}
569-
}
570-
}
571-
if (builder == null) {
572-
builder = Source.newBuilder(source).internal(internal).content(Source.CONTENT_NONE);
573-
}
574550
RootNode rootNode;
575-
LazySource lazySource = new LazySource(builder);
576-
577551
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
578-
// TODO lazily load source in bytecode DSL interpreter too
579-
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, lazySource.getSource());
552+
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, source);
580553
} else {
581-
rootNode = PBytecodeRootNode.create(context.getLanguage(), (BytecodeCodeUnit) code, lazySource, internal);
554+
rootNode = PBytecodeRootNode.create(context.getLanguage(), (BytecodeCodeUnit) code, source, source.isInternal());
582555
}
583556

584557
return PythonUtils.getOrCreateCallTarget(rootNode);
585558
}
586559

587-
private static SourceBuilder sourceForOriginalFile(PythonContext context, CodeUnit code, boolean internal, String path) {
588-
try {
589-
TruffleFile file = context.getEnv().getPublicTruffleFile(path);
590-
if (!file.isReadable()) {
591-
return null;
592-
}
593-
return Source.newBuilder(PythonLanguage.ID, file).name(code.name.toJavaStringUncached()).internal(internal);
594-
} catch (SecurityException | UnsupportedOperationException | InvalidPathException e) {
595-
return null;
596-
}
597-
}
598-
599560
public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames,
600561
EnumSet<FutureFeature> futureFeatures) {
601562
return parse(context, source, type, topLevel, optimize, interactiveTerminal, false, argumentNames, futureFeatures);
@@ -677,7 +638,7 @@ private RootNode compileForBytecodeInterpreter(ModTy mod, Source source, int opt
677638
Compiler compiler = new Compiler(parserCallbacks);
678639
CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures);
679640
BytecodeCodeUnit co = cu.assemble();
680-
return PBytecodeRootNode.create(this, co, new LazySource(source), source.isInternal(), parserCallbacks);
641+
return PBytecodeRootNode.create(this, co, source, source.isInternal(), parserCallbacks);
681642
}
682643

683644
private RootNode compileForBytecodeDSLInterpreter(ModTy mod, Source source, int optimize,
@@ -957,7 +918,7 @@ private static Source newSource(PythonContext context, SourceBuilder srcBuilder)
957918
return srcBuilder.build();
958919
}
959920

960-
private static boolean shouldMarkSourceInternal(PythonContext ctxt) {
921+
public static boolean shouldMarkSourceInternal(PythonContext ctxt) {
961922
return !ctxt.isCoreInitialized() && !ctxt.getLanguage().getEngineOption(PythonOptions.ExposeInternalSources);
962923
}
963924

@@ -1259,6 +1220,34 @@ public Source getOrCreateSource(Function<Object, Source> rootNodeFunction, Objec
12591220
return sourceCache.computeIfAbsent(key, rootNodeFunction);
12601221
}
12611222

1223+
public Source getOrCreateSourceWithContent(Source sourceWithoutContent) {
1224+
if (sourceWithoutContent.hasCharacters() || sourceWithoutContent.getPath() == null) {
1225+
return sourceWithoutContent;
1226+
}
1227+
String path = sourceWithoutContent.getPath();
1228+
return getOrCreateSource(ignored -> loadSourceWithContent(sourceWithoutContent), path);
1229+
}
1230+
1231+
private static Source loadSourceWithContent(Source sourceWithoutContent) {
1232+
String path = sourceWithoutContent.getPath();
1233+
if (path == null) {
1234+
return sourceWithoutContent;
1235+
}
1236+
PythonContext context = PythonContext.get(null);
1237+
if (context == null) {
1238+
return sourceWithoutContent;
1239+
}
1240+
try {
1241+
TruffleFile file = context.getEnv().getPublicTruffleFile(path);
1242+
if (!file.isReadable()) {
1243+
return sourceWithoutContent;
1244+
}
1245+
return Source.newBuilder(PythonLanguage.ID, file).name(sourceWithoutContent.getName()).internal(sourceWithoutContent.isInternal()).build();
1246+
} catch (IOException | SecurityException | UnsupportedOperationException | InvalidPathException e) {
1247+
return sourceWithoutContent;
1248+
}
1249+
}
1250+
12621251
public static PythonOS getPythonOS() {
12631252
if (PythonOS.internalCurrent == PythonOS.PLATFORM_ANY) {
12641253
if (ImageInfo.inImageBuildtimeCode()) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___LOADER__;
5050
import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___ORIGNAME__;
5151
import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___PATH__;
52+
import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION;
5253
import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_PYD;
5354
import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_SO;
5455
import static com.oracle.graal.python.nodes.StringLiterals.T_NAME;
@@ -117,6 +118,7 @@
117118
import com.oracle.truffle.api.CompilerDirectives;
118119
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
119120
import com.oracle.truffle.api.RootCallTarget;
121+
import com.oracle.truffle.api.TruffleFile;
120122
import com.oracle.truffle.api.TruffleSafepoint;
121123
import com.oracle.truffle.api.dsl.Bind;
122124
import com.oracle.truffle.api.dsl.Cached;
@@ -642,8 +644,28 @@ public static PythonModule importFrozenModuleObject(Node inliningTarget, PConstr
642644
private static RootCallTarget createCallTarget(PythonContext context, FrozenInfo info) {
643645
return (RootCallTarget) context.getLanguage().cacheCode(new PythonLanguage.CodeCacheKey(info.origName, System.identityHashCode(info.code)), () -> {
644646
String name = PythonLanguage.FROZEN_FILENAME_PREFIX + info.name + PythonLanguage.FROZEN_FILENAME_SUFFIX;
645-
Source source = Source.newBuilder("python", "", name).content(Source.CONTENT_NONE).build();
646-
return PythonLanguage.callTargetFromBytecode(context, source, info.code);
647+
Source.LiteralBuilder builder = null;
648+
try {
649+
String fs = context.getEnv().getFileNameSeparator();
650+
String basename = context.getStdlibHome() + fs + info.name.toJavaStringUncached().replace(".", fs);
651+
String originalPath = basename + J_PY_EXTENSION;
652+
TruffleFile originalFile = context.getEnv().getInternalTruffleFile(originalPath);
653+
if (!originalFile.isReadable()) {
654+
originalPath = basename + fs + "__init__.py";
655+
originalFile = context.getEnv().getInternalTruffleFile(originalPath);
656+
}
657+
if (originalFile.isReadable()) {
658+
builder = Source.newBuilder(PythonLanguage.ID, originalFile).content(Source.CONTENT_NONE).name(name);
659+
}
660+
} catch (UnsupportedOperationException | IllegalArgumentException | SecurityException e) {
661+
// Fallthrough
662+
}
663+
if (builder == null) {
664+
builder = Source.newBuilder(PythonLanguage.ID, "", name).content(Source.CONTENT_NONE);
665+
}
666+
builder.internal(PythonLanguage.shouldMarkSourceInternal(context));
667+
builder.mimeType(PythonLanguage.MIME_TYPE);
668+
return PythonLanguage.callTargetFromBytecode(context, builder.build(), info.code);
647669
});
648670
}
649671

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1553,8 +1553,22 @@ private PCode readCode() {
15531553
byte[] lnoTab = readBytes();
15541554
com.oracle.graal.python.util.Supplier<CallTarget> supplier = () -> {
15551555
String jFilename = fileName.toJavaStringUncached();
1556-
Source subSource = Source.newBuilder(PythonLanguage.ID, "", jFilename).content(Source.CONTENT_NONE).build();
1557-
return PythonLanguage.callTargetFromBytecode(context, subSource, code);
1556+
String jName = code.qualname.toJavaStringUncached();
1557+
Source.LiteralBuilder builder = null;
1558+
try {
1559+
TruffleFile file = context.getEnv().getPublicTruffleFile(jFilename);
1560+
if (file.isReadable()) {
1561+
builder = Source.newBuilder(PythonLanguage.ID, file).content(Source.CONTENT_NONE).name(jName);
1562+
}
1563+
} catch (UnsupportedOperationException | IllegalArgumentException | SecurityException e) {
1564+
// Fallthrough
1565+
}
1566+
if (builder == null) {
1567+
builder = Source.newBuilder(PythonLanguage.ID, "", jName).content(Source.CONTENT_NONE);
1568+
}
1569+
builder.internal(PythonLanguage.shouldMarkSourceInternal(context));
1570+
builder.mimeType(PythonLanguage.MIME_TYPE);
1571+
return PythonLanguage.callTargetFromBytecode(context, builder.build(), code);
15581572
};
15591573
CallTarget callTarget;
15601574
if (getLanguage().isSingleContext() || cacheKey == 0) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
import com.oracle.graal.python.runtime.PythonContext;
6161
import com.oracle.graal.python.runtime.PythonOptions;
6262
import com.oracle.graal.python.runtime.object.PFactory;
63-
import com.oracle.graal.python.util.LazySource;
6463
import com.oracle.graal.python.util.PythonUtils;
6564
import com.oracle.truffle.api.CompilerAsserts;
6665
import com.oracle.truffle.api.CompilerDirectives;
@@ -168,7 +167,7 @@ private static RootCallTarget deserializeForBytecodeInterpreter(PythonContext co
168167
code.variableShouldUnbox,
169168
code.generalizeInputsKeys, code.generalizeInputsIndices, code.generalizeInputsValues, code.generalizeVarsIndices, code.generalizeVarsValues);
170169
}
171-
rootNode = PBytecodeRootNode.create(context.getLanguage(), code, new LazySource(PythonUtils.createFakeSource()), false);
170+
rootNode = PBytecodeRootNode.create(context.getLanguage(), code, PythonUtils.createFakeSource(), false);
172171
if (code.isGeneratorOrCoroutine()) {
173172
rootNode = new PBytecodeGeneratorFunctionRootNode(context.getLanguage(), rootNode.getFrameDescriptor(), (PBytecodeRootNode) rootNode, code.name);
174173
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ private PCode createCode(BytecodeCodeUnit codeUnit) {
526526
PBytecodeRootNode outerRootNode = (PBytecodeRootNode) getRootNodeForExtraction();
527527
PythonLanguage language = outerRootNode.getLanguage();
528528
RootCallTarget callTarget = language.createCachedCallTarget(
529-
l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getLazySource(), outerRootNode.isInternal()), codeUnit);
529+
l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getSource(), outerRootNode.isInternal()), codeUnit);
530530
RootNode rootNode = callTarget.getRootNode();
531531
if (rootNode instanceof PBytecodeGeneratorFunctionRootNode generatorRoot) {
532532
rootNode = generatorRoot.getBytecodeRootNode();

0 commit comments

Comments
 (0)