# Copyright (c) 2018, 2026, Oracle and/or its affiliates.

# Copyright (c) 2013, Regents of the University of California

#

# All rights reserved.

#

# Redistribution and use in source and binary forms, with or without modification, are

# permitted provided that the following conditions are met:

#

# 1. Redistributions of source code must retain the above copyright notice, this list of

# conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice, this list of

# conditions and the following disclaimer in the documentation and/or other materials provided

# with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF

# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE

# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE

# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING

# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED

# OF THE POSSIBILITY OF SUCH DAMAGE.

from __future__ import print_function

import contextlib

import datetime

import glob

import gzip

import itertools

import os

import pathlib

import re

import shlex

import shutil

import subprocess

import sys

import time

from functools import wraps

from pathlib import Path

from textwrap import dedent

from typing import cast, Union, Literal, overload

import downstream_tests

import mx_graalpython_benchmark

import mx_urlrewrites

import tempfile

from argparse import ArgumentParser

from dataclasses import dataclass

import mx

import mx_util

import mx_gate

import mx_native

import mx_unittest

import mx_sdk

import mx_sdk_vm_ng

import mx_subst

import mx_truffle

import mx_graalpython_bisect

import mx_graalpython_import

import mx_graalpython_python_benchmarks

# re-export custom mx project classes so they can be used from suite.py

from mx import MavenProject #pylint: disable=unused-import

from mx_cmake import CMakeNinjaProject #pylint: disable=unused-import

from mx_gate import Task

from mx_graalpython_bench_param import PATH_MESO

# re-export custom mx project classes, so they can be used from suite.py

from mx_sdk_vm_ng import StandaloneLicenses, ThinLauncherProject, LanguageLibraryProject, DynamicPOMDistribution, DeliverableStandaloneArchive # pylint: disable=unused-import

if not sys.modules.get("__main__"):

# workaround for pdb++

sys.modules["__main__"] = type(sys)("<empty>")

def get_boolean_env(name, default=False):

env = os.environ.get(name)

if env is None:

return default

return env.lower() in ('true', '1')

SUITE = cast(mx.SourceSuite, mx.suite('graalpython'))

SUITE_COMPILER = mx.suite("compiler", fatalIfMissing=False)

GRAALPY_ABI_VERSION = 'graalpy250'

GRAAL_VERSION = SUITE.suiteDict['version']

IS_RELEASE = SUITE.suiteDict['release']

GRAAL_VERSION_MAJ_MIN = ".".join(GRAAL_VERSION.split(".")[:2])

PYTHON_VERSION = SUITE.suiteDict[f'{SUITE.name}:pythonVersion']

PYTHON_VERSION_MAJ_MIN = ".".join(PYTHON_VERSION.split('.')[:2])

LATEST_JAVA_HOME = {"JAVA_HOME": os.environ.get("LATEST_JAVA_HOME", mx.get_jdk().home)}

RUNNING_ON_LATEST_JAVA = os.environ.get("LATEST_JAVA_HOME", os.environ.get("JAVA_HOME")) == mx.get_jdk().home

# this environment variable is used by some of our maven projects and jbang integration to build against the unreleased master version during development

os.environ["GRAALPY_VERSION"] = GRAAL_VERSION

MAIN_BRANCH = 'master'

GRAALPYTHON_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"

SANDBOXED_OPTIONS = [

'--experimental-options',

'--python.PosixModuleBackend=java',

'--python.Sha3ModuleBackend=java',

'--python.CompressionModulesBackend=java',

'--python.PyExpatModuleBackend=java',

'--python.UnicodeCharacterDatabaseNativeFallback=false',

]

# Allows disabling rebuild for some mx commands such as graalpytest

DISABLE_REBUILD = get_boolean_env('GRAALPYTHON_MX_DISABLE_REBUILD')

_COLLECTING_COVERAGE = False

CI = get_boolean_env("CI")

GITHUB_CI = get_boolean_env("GITHUB_CI")

WIN32 = sys.platform == "win32"

BUILD_NATIVE_IMAGE_WITH_ASSERTIONS = get_boolean_env('BUILD_WITH_ASSERTIONS', CI)

BYTECODE_DSL_INTERPRETER = get_boolean_env('BYTECODE_DSL_INTERPRETER', True)

mx_gate.add_jacoco_excludes([

"com.oracle.graal.python.pegparser.sst",

"com.oracle.graal.python.pegparser.test",

"com.oracle.truffle.api.staticobject.test",

"com.oracle.truffle.regex.tregex.test",

"com.oracle.truffle.tck",

"com.oracle.truffle.tools.chromeinspector.test",

"com.oracle.truffle.tools.coverage.test",

"com.oracle.truffle.tools.dap.test",

"com.oracle.truffle.tools.profiler.test",

"org.graalvm.tools.insight.test",

"org.graalvm.tools.lsp.test",

])

def is_collecting_coverage():

return bool(mx_gate.get_jacoco_agent_args() or _COLLECTING_COVERAGE)

def wants_debug_build(flags=os.environ.get("CFLAGS", "")):

return any(x in flags for x in ["-g", "-ggdb", "-ggdb3"])

if wants_debug_build():

setattr(mx_native.DefaultNativeProject, "_original_cflags", mx_native.DefaultNativeProject.cflags)

setattr(mx_native.DefaultNativeProject, "cflags", property(

lambda self: self._original_cflags + (["/Z7"] if WIN32 else ["-fPIC", "-ggdb3"])

))

if WIN32:

# let's check if VS compilers are on the PATH

if not os.environ.get("LIB"):

mx.log("LIB not in environment, not a VS shell")

elif not os.environ.get("INCLUDE"):

mx.log("INCLUDE not in environment, not a VS shell")

else:

for p in os.environ.get("PATH", "").split(os.pathsep):

if os.path.isfile(os.path.join(os.path.abspath(p), "cl.exe")):

mx.log("LIB and INCLUDE set, cl.exe on PATH, assuming this is a VS shell")

os.environ["DISTUTILS_USE_SDK"] = "1"

if not os.environ.get("MSSdk"):

os.environ["MSSdk"] = os.environ.get("WindowsSdkDir", "unset")

break

else:

mx.log("cl.exe not on PATH, not a VS shell")

def _get_stdlib_home():

return os.path.join(SUITE.dir, "graalpython", "lib-python", "3")

def _get_capi_home():

native_libs_output = mx.distribution("GRAALPYTHON_NATIVE_LIBS").get_output()

assert native_libs_output

return os.path.join(native_libs_output, mx.get_os(), mx.get_arch())

def _extract_graalpython_internal_options(args):

non_internal = []

additional_dists = []

for arg in args:

# Class path extensions

if arg.startswith('-add-dist='):

additional_dists += [arg[10:]]

else:

non_internal += [arg]

return non_internal, additional_dists

def mx_register_dynamic_suite_constituents(register_project, register_distribution):

if register_project and register_distribution:

isolate_build_options = [

'-H:+DetectUserDirectoriesInImageHeap',

]

meta_pom = None

for dist in SUITE.dists:

if dist.name == 'PYTHON_POM':

meta_pom = dist

assert meta_pom, "Cannot find python meta-POM distribution in the graalpython suite"

mx_truffle.register_polyglot_isolate_distributions(SUITE, register_project, register_distribution, 'python',

'graalpython', meta_pom.name, meta_pom.maven_group_id(), meta_pom.theLicense,

isolate_build_options)

def extend_os_env(**kwargs):

env = os.environ.copy()

env.update(**kwargs)

return env

def delete_bad_env_keys(env):

for k in ["SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"]:

if k in env:

del env[k]

def check_vm(vm_warning=True, must_be_jvmci=False):

if not SUITE_COMPILER:

if must_be_jvmci:

mx.abort('** Error ** : graal compiler was not found!')

if vm_warning:

mx.log('** warning ** : graal compiler was not found!! Executing using standard VM..')

def get_jdk():

return mx.get_jdk()

# Called from suite.py

def graalpy_standalone_deps():

include_truffle_runtime = not mx.env_var_to_bool("EXCLUDE_TRUFFLE_RUNTIME")

deps = mx_truffle.resolve_truffle_dist_names(use_optimized_runtime=include_truffle_runtime)

return deps

def _is_overridden_native_image_arg(prefix):

extras = mx.get_opts().extra_image_builder_argument

return any(arg.startswith(prefix) for arg in extras)

def github_ci_build_args():

# Determine memory and parallelism for GitHub CI builds

# Use 90% of available memory up to 14GB, but at least 8GB

# Set cores to number of CPUs if at least 4 cores and enough memory, otherwise 1

total_mem = 0

try:

if mx.is_windows():

for m in subprocess.check_output(['wmic', 'memorychip', 'get', 'capacity'], encoding='utf-8').splitlines():

try:

total_mem += int(m) / 1024**3

except ValueError:

pass

else:

total_mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / 1024**3

except:

total_mem = 16.0

min_bound = 8

max_mem = 14*1024

min_mem = int(1024 * (total_mem if total_mem < min_bound else total_mem * .9))

os_cpu = os.cpu_count() or int(os.environ.get("NUMBER_OF_PROCESSORS", 1)) or 1

build_mem = min(min_mem, max_mem)

parallelism = os_cpu if os_cpu >= 4 and build_mem >= min_bound*1024 else 1

return ["-Ob",

f"-J-Xms{build_mem}m",

f"--parallelism={parallelism}"

]

def libpythonvm_build_args():

build_args = bytecode_dsl_build_args()

if limit := os.environ.get('GRAALPY_UncachedInterpreterLimit'):

mx.log(f"Uncached interpreter limit explicitly set to {limit}")

build_args += [f'-Dpython.UncachedInterpreterLimit={limit}']

if os.environ.get("GITHUB_CI"):

build_args += github_ci_build_args()

if graalos := ("musl" in mx_subst.path_substitutions.substitute("<multitarget_libc_selection>")):

build_args += ['-H:+GraalOS']

else:

build_args += ["-Dpolyglot.image-build-time.PreinitializeContexts=python"]

if (

mx.is_linux()

and not graalos

and mx_sdk_vm_ng.is_nativeimage_ee()

and not os.environ.get('NATIVE_IMAGE_AUXILIARY_ENGINE_CACHE')

and not _is_overridden_native_image_arg("--gc")

):

build_args += ['--gc=G1', '-H:-ProtectionKeys']

profile = None

if (

"GRAALPY_PGO_PROFILE" not in os.environ

and mx.suite('graalpython-enterprise', fatalIfMissing=False)

and mx_sdk_vm_ng.get_bootstrap_graalvm_version() >= mx.VersionSpec("25.0")

and not _is_overridden_native_image_arg("--pgo")

):

vc = SUITE.vc

commit = str(vc.tip(SUITE.dir)).strip()

branch = str(vc.active_branch(SUITE.dir, abortOnError=False) or 'master').strip()

if script := os.environ.get("ARTIFACT_DOWNLOAD_SCRIPT"):

# This is always available in the GraalPy CI

profile = f"cached_profile.iprof.gz"

run(

[

sys.executable,

script,

f"graalpy/pgo-{commit}",

profile,

],

nonZeroIsFatal=False,

)

else:

# Locally, we try to get a reasonable profile

get_profile = mx.command_function('python-get-latest-profile', fatalIfMissing=False)

if get_profile:

for b in [branch, "master"]:

if not profile:

try:

profile = get_profile(["--branch", b])

except BaseException:

pass

if CI and (not profile or not os.path.isfile(profile)):

mx.log("No profile in CI job")

# When running on a release branch or attempting to merge into

# a release branch, make sure we can use a PGO profile, and

# when running in the CI on a bench runner, ensure a PGO profile

if (

any(b.startswith("release/") for b in [branch, os.environ.get("TO_BRANCH", "")])

or ("bench" in os.environ.get('BUILD_NAME', ''))

):

mx.warn("PGO profile must exist for benchmarking and release, creating one now...")

profile = graalpy_native_pgo_build_and_test()

if os.path.isfile(profile or ""):

print(invert(f"Automatically chose PGO profile {profile}. To disable this, set GRAALPY_PGO_PROFILE to an empty string'", blinking=True), file=sys.stderr)

build_args += [

f"--pgo={profile}",

"-H:+UnlockExperimentalVMOptions",

"-H:+PGOPrintProfileQuality",

"-H:-UnlockExperimentalVMOptions",

]

else:

print(invert("Not using an automatically selected PGO profile"), file=sys.stderr)

return build_args

def graalpy_native_pgo_build_and_test(args=None):

"""

Builds a PGO-instrumented GraalPy native standalone, runs the unittests to generate a profile,

then builds a PGO-optimized GraalPy native standalone with the collected profile.

The profile file will be named 'default.iprof' in native image build directory.

"""

if mx_sdk_vm_ng.get_bootstrap_graalvm_version() < mx.VersionSpec("25.0"):

mx.abort("python-native-pgo not supported on GraalVM < 25")

with set_env(GRAALPY_PGO_PROFILE=""):

mx.log(mx.colorize("[PGO] Building PGO-instrumented native image", color="yellow"))

build_home = graalpy_standalone_home('native', enterprise=True, build=True)

instrumented_home = build_home + "_PGO_INSTRUMENTED"

shutil.rmtree(instrumented_home, ignore_errors=True)

shutil.copytree(build_home, instrumented_home, symlinks=True, ignore_dangling_symlinks=True)

instrumented_launcher = os.path.join(instrumented_home, 'bin', _graalpy_launcher())

mx.log(mx.colorize(f"[PGO] Instrumented build complete: {instrumented_home}", color="yellow"))

mx.log(mx.colorize(f"[PGO] Running graalpytest with instrumented binary: {instrumented_launcher}", color="yellow"))

with tempfile.TemporaryDirectory() as d:

with set_env(

GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS="true",

GRAAL_PYTHON_VM_ARGS="\v".join([

f"--vm.XX:ProfilesDumpFile={os.path.join(d, '$UUID$.iprof')}",

f"--vm.XX:ProfilesLCOVFile={os.path.join(d, '$UUID$.info')}",

]),

GRAALPY_HOME=instrumented_home,

):

graalpytest(["--python", instrumented_launcher, "test_venv.py"])

mx.command_function('benchmark')(["meso-small:*", "--", "--python-vm", "graalpython", "--python-vm-config", 'custom'])

iprof_path = Path(SUITE.dir) / f'default.iprof'

lcov_path = Path(SUITE.dir) / f'default.lcov'

run([

os.path.join(

graalvm_jdk(enterprise=True),

"bin",

f"native-image-configure{'.exe' if mx.is_windows() else ''}",

),

"merge-pgo-profiles",

f"--input-dir={d}",

f"--output-file={iprof_path}"

])

run([

"/usr/bin/env",

"lcov",

"-o", str(lcov_path),

*itertools.chain.from_iterable([

["-a", f.absolute().as_posix()] for f in Path(d).glob("*.info")

])

], nonZeroIsFatal=False)

run([

"/usr/bin/env",

"genhtml",

"--source-directory", str(Path(SUITE.dir) / "com.oracle.graal.python" / "src"),

"--source-directory", str(Path(SUITE.dir) / "com.oracle.graal.python.pegparser" / "src"),

"--source-directory", str(Path(SUITE.get_output_root()) / "com.oracle.graal.python" / "src_gen"),

"--include", "com/oracle/graal/python",

"--keep-going",

"-o", "lcov_html",

str(lcov_path),

], nonZeroIsFatal=False)

if not os.path.isfile(iprof_path):

mx.abort(f"[PGO] Could not find profile file at expected location: {iprof_path}")

with set_env(GRAALPY_PGO_PROFILE=str(iprof_path)):

mx.log(mx.colorize("[PGO] Building optimized native image with collected profile", color="yellow"))

native_bin = graalpy_standalone('native', enterprise=True, build=True)

mx.log(mx.colorize(f"[PGO] Optimized PGO build complete: {native_bin}", color="yellow"))

iprof_gz_path = str(iprof_path) + '.gz'

with open(iprof_path, 'rb') as f_in, gzip.open(iprof_gz_path, 'wb') as f_out:

shutil.copyfileobj(f_in, f_out)

mx.log(mx.colorize(f"[PGO] Gzipped profile at: {iprof_gz_path}", color="yellow"))

if script := os.environ.get("ARTIFACT_UPLOADER_SCRIPT"):

commit = str(SUITE.vc.tip(SUITE.dir)).strip()

run(

[

sys.executable,

script,

iprof_gz_path,

f"pgo-{commit}",

"graalpy",

"--lifecycle",

"cache",

"--artifact-repo-key",

os.environ.get("ARTIFACT_REPO_KEY_LOCATION"),

'--skip-existing',

],

)

if args is None:

return iprof_gz_path

def full_python(args, env=None):

"""Run python from standalone build (unless kwargs are given). Does not build GraalPython sources automatically."""

if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):

args.insert(0, '--python.WithJavaStacktrace=1')

if "--hosted" in args[:2]:

return do_run_python(args)

if '--vm.da' not in args:

args.insert(0, '--vm.ea')

if not any(arg.startswith('--experimental-options') for arg in args):

args.insert(0, '--experimental-options')

handle_debug_arg(args)

for arg in itertools.chain(

itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_sfx))),

reversed(shlex.split(mx._opts.java_args if mx._opts.java_args else "")),

itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_pfx))),

):

if arg.startswith("-"):

args.insert(0, f"--vm.{arg[1:]}")

else:

mx.warn(f"Dropping {arg}, cannot pass it to launcher")

standalone_home = graalpy_standalone_home('jvm', dev=True, build=False)

graalpy_path = os.path.join(standalone_home, 'bin', _graalpy_launcher())

if not os.path.exists(graalpy_path):

mx.abort("GraalPy standalone doesn't seem to be built.\n" +

"To build it: mx python-jvm\n" +

"Alternatively use: mx python --hosted")

run([graalpy_path] + args, env=env)

def handle_debug_arg(args):

if mx._opts.java_dbg_port:

args.insert(0,

f"--vm.agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=127.0.0.1:{mx._opts.java_dbg_port}")

def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None, cp_prefix=None, cp_suffix=None, main_class=GRAALPYTHON_MAIN_CLASS, minimal=False, **kwargs):

if "--hosted" in args[:2]:

args.remove("--hosted")

if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):

args.insert(0, '--python.WithJavaStacktrace=1')

if not any(arg.startswith("--python.CAPI") for arg in args):

capi_home = _get_capi_home()

args.insert(0, "--python.CAPI=%s" % capi_home)

args.insert(0, "--experimental-options")

if not env:

env = os.environ.copy()

env.setdefault("GRAAL_PYTHONHOME", _dev_pythonhome())

delete_bad_env_keys(env)

check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)

if check_vm_env:

if check_vm_env == '1':

check_vm(must_be_jvmci=True)

elif check_vm_env == '0':

check_vm()

if minimal:

x = [x for x in SUITE.dists if x.name == "GRAALPYTHON"][0]

dists = [dep for dep in x.deps if dep.isJavaProject() or dep.isJARDistribution() and dep.exists()]

# Hack: what we should just do is + ['GRAALPYTHON_VERSIONS_MAIN'] and let MX figure out

# the class-path and other VM arguments necessary for it. However, due to a bug in MX,

# LayoutDirDistribution causes an exception if passed to mx.get_runtime_jvm_args,

# because it does not properly initialize its super class ClasspathDependency, see MX PR: 1665.

ver_dep = mx.distribution('GRAALPYTHON_VERSIONS_MAIN').get_output()

cp_prefix = ver_dep if cp_prefix is None else (str(ver_dep) + os.pathsep + cp_prefix)

else:

dists = ['GRAALPYTHON']

dists += ['GRAALPYTHON-LAUNCHER']

vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)

if minimal:

vm_args.insert(0, f"-Dorg.graalvm.language.python.home={_dev_pythonhome()}")

graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)

dists += additional_dists

if extra_dists:

dists += extra_dists

if not CI:

# Try eagerly to include tools for convenience when running Python

if not mx.suite("tools", fatalIfMissing=False):

SUITE.import_suite("tools", version=None, urlinfos=None, in_subdir=True)

if mx.suite("tools", fatalIfMissing=False):

for tool in ["CHROMEINSPECTOR", "TRUFFLE_COVERAGE"]:

if os.path.exists(mx.distribution(tool).path):

dists.append(tool)

graalpython_args.insert(0, '--experimental-options=true')

vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk, cp_prefix=cp_prefix, cp_suffix=cp_suffix, force_cp=True)

if not jdk:

jdk = get_jdk()

# default: assertion checking is enabled

if extra_vm_args is None or '-da' not in extra_vm_args:

vm_args += ['-ea', '-esa']

if extra_vm_args:

vm_args += extra_vm_args

vm_args.append(main_class)

return mx.run_java(vm_args + graalpython_args, jdk=jdk, env=env, **kwargs)

def node_footprint_analyzer(args, **kwargs):

main_class = 'com.oracle.graal.python.test.advanced.NodeFootprintAnalyzer'

vm_args = mx.get_runtime_jvm_args(['GRAALPYTHON_UNIT_TESTS', 'GRAALPYTHON'])

return mx.run_java(vm_args + [main_class] + args, **kwargs)

def _dev_pythonhome_context():

home = os.environ.get("GRAAL_PYTHONHOME", _dev_pythonhome())

return set_env(GRAAL_PYTHONHOME=home)

def _dev_pythonhome():

return os.path.join(SUITE.dir, "graalpython")

def get_path_with_patchelf():

path = os.environ.get("PATH", "")

if mx.is_linux() and not shutil.which("patchelf"):

venv = Path(SUITE.get_output_root()).absolute() / "patchelf-venv"

path += os.pathsep + str(venv / "bin")

if not shutil.which("patchelf", path=path):

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [patchelf not found on PATH]")

t0 = time.time()

subprocess.check_call([sys.executable, "-m", "venv", str(venv)])

subprocess.check_call([str(venv / "bin" / "pip"), "install", "patchelf"])

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [duration: {time.time() - t0}]")

if mx.is_windows() and not shutil.which("delvewheel"):

venv = Path(SUITE.get_output_root()).absolute() / "delvewheel-venv"

path += os.pathsep + str(venv / "Scripts")

if not shutil.which("delvewheel", path=path):

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building delvewheel-venv with {sys.executable}... [delvewheel not found on PATH]")

t0 = time.time()

subprocess.check_call([sys.executable, "-m", "venv", str(venv)])

subprocess.check_call([str(venv / "Scripts" / "pip.exe"), "install", "delvewheel"])

mx.log(f"{time.strftime('[%H:%M:%S] ')} Building delvewheel-venv with {sys.executable}... [duration: {time.time() - t0}]")

return path

def punittest(ars, report: Union[Task, bool, None] = False):

"""

Runs GraalPython junit tests and memory leak tests, which can be skipped using --no-leak-tests.

Pass --regex to further filter the junit and TSK tests. GraalPy tests are always run in two configurations:

with language home on filesystem and with language home served from the Truffle resources.

"""

path = get_path_with_patchelf()

args = [] if ars is None else ars

@dataclass

class TestConfig:

identifier: str

args: list

useResources: bool

reportConfig: Union[Task, bool, None] = report

def __str__(self):

return f"args={self.args!r}, useResources={self.useResources}, report={self.reportConfig}"

def __post_init__(self):

assert ' ' not in self.identifier

configs = []

skip_leak_tests = False

if "--no-leak-tests" in args:

skip_leak_tests = True

args.remove("--no-leak-tests")

if is_collecting_coverage():

skip_leak_tests = True

vm_args = ['-Dpolyglot.engine.WarnInterpreterOnly=false'] + bytecode_dsl_build_args()

# Note: we must use filters instead of --regex so that mx correctly processes the unit test configs,

# but it is OK to apply --regex on top of the filters

graalpy_tests = ['com.oracle.graal.python.test', 'com.oracle.graal.python.pegparser.test', 'org.graalvm.python.embedding.test']

configs += [

TestConfig("junit", vm_args + graalpy_tests + args, True),

TestConfig("junit", vm_args + graalpy_tests + args, False)]

if not mx.is_windows():

configs += [

# Tests that must run in their own process due to C extensions usage, for now ignored on Windows

TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, True),

TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, False),

TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, True),

TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, False),

]

if '--regex' not in args:

async_regex = ['--regex', r'com\.oracle\.graal\.python\.test\.integration\.advanced\.AsyncActionThreadingTest']

configs.append(TestConfig("async", vm_args + ['-Dpython.AutomaticAsyncActions=false', 'com.oracle.graal.python.test', 'org.graalvm.python.embedding.test'] + async_regex + args, True, False))

else:

skip_leak_tests = True

for c in configs:

mx.log(f"Python JUnit tests configuration: {c}")

PythonMxUnittestConfig.useResources = c.useResources

with set_env(PATH=path):

mx_unittest.unittest(c.args, test_report_tags=({"task": f"punittest-{c.identifier}-{'w' if c.useResources else 'wo'}-resources"} if c.reportConfig else None))

if skip_leak_tests:

return

# test leaks with Python code only

run_leak_launcher(["--code", "pass", ])

run_leak_launcher(["--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])

# test leaks when some C module code is involved

run_leak_launcher(["--code", 'import _testcapi, mmap, bz2; print(memoryview(b"").nbytes)'])

# test leaks with shared engine Python code only

run_leak_launcher(["--shared-engine", "--code", "pass"])

run_leak_launcher(["--shared-engine", "--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])

# test leaks with shared engine when some C module code is involved

run_leak_launcher(["--shared-engine", "--code", 'import _testcapi, mmap, bz2; print(memoryview(b"").nbytes)'])

run_leak_launcher(["--shared-engine", "--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",

"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",

"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])

run_leak_launcher(["--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",

"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",

"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])

PYTHON_ARCHIVES = ["GRAALPYTHON_GRAALVM_SUPPORT"]

PYTHON_NATIVE_PROJECTS = ["python-libbz2",

"python-liblzma",

"python-libzsupport",

"python-libposix",

"com.oracle.graal.python.cext"]

def nativebuild(_):

"Build the non-Java Python projects and archives"

mx.build(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])

def nativeclean(_):

"Clean the non-Java Python projects"

mx.clean(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])

class GraalPythonTags(object):

junit = 'python-junit'

junit_maven = 'python-junit-maven'

junit_maven_isolates = 'python-junit-polyglot-isolates'

jvmbuild = 'python-jvm-build'

unittest = 'python-unittest'

unittest_cpython = 'python-unittest-cpython'

unittest_sandboxed = 'python-unittest-sandboxed'

unittest_multi = 'python-unittest-multi-context'

unittest_jython = 'python-unittest-jython'

unittest_arrow = 'python-unittest-arrow-storage'

unittest_hpy = 'python-unittest-hpy'

unittest_hpy_sandboxed = 'python-unittest-hpy-sandboxed'

unittest_standalone = 'python-unittest-standalone'

tagged = 'python-tagged-unittest'

svmbuild = 'python-svm-build'

svmunit = 'python-svm-unittest'

svmunit_sandboxed = 'python-svm-unittest-sandboxed'

graalvm = 'python-graalvm'

embedding = 'python-standalone-embedding'

graalvm_sandboxed = 'python-graalvm-sandboxed'

svm = 'python-svm'

native_image_embedder = 'python-native-image-embedder'

license = 'python-license'

language_checker = 'python-language-checker'

exclusions_checker = 'python-class-exclusion-checker'

def python_gate(args):

if not os.environ.get("JDT"):

os.environ["JDT"] = "builtin"

if not os.environ.get("ECLIPSE_EXE"):

find_eclipse()

if "--tags" not in args:

args += ["--tags"]

tags = ["style"]

for x in dir(GraalPythonTags):

v = getattr(GraalPythonTags, x)

if isinstance(v, str) and v.startswith("python-"):

if "sandboxed" not in v:

tags.append(v)

args.append(",".join(tags))

mx.log("Running mx python-gate " + " ".join(args))

return mx.command_function("gate")(args)

python_gate.__doc__ = 'Custom gates are %s' % ", ".join([

getattr(GraalPythonTags, t) for t in dir(GraalPythonTags) if not t.startswith("__")

])

def find_eclipse():

pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))

for f in [os.path.join(SUITE.dir, f)

for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f)

for f in os.listdir(pardir)]:

if os.path.basename(f) == "eclipse" and os.path.isdir(f):

mx.log("Automatically choosing %s for Eclipse" % f)

eclipse_exe = os.path.join(f, f"eclipse{'.exe' if mx.is_windows() else ''}")

if os.path.exists(eclipse_exe):

os.environ["ECLIPSE_EXE"] = eclipse_exe

return

@contextlib.contextmanager

def set_env(**environ):

"""Temporarily set the process environment variables"""

old_environ = dict(os.environ)

for k, v in environ.items():

if v is None:

if k in os.environ:

del os.environ[k]

else:

os.environ[k] = v

try:

yield

finally:

os.environ.clear()

os.environ.update(old_environ)

def _graalpy_launcher():

name = 'graalpy'

return f"{name}.exe" if WIN32 else name

# dev only has effect if standalone_type is 'jvm' and means minimal, Default TruffleRuntime (no JIT)

def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build=True):

assert standalone_type in ['native', 'jvm']

jdk_version = mx.get_jdk().version

# Check if GRAALPY_HOME points to some compatible pre-built GraalPy standalone

python_home = os.environ.get("GRAALPY_HOME", None)

if python_home:

python_home = os.path.abspath(glob.glob(python_home)[0])

mx.log("Using GraalPy standalone from GRAALPY_HOME: " + python_home)

# Try to verify that we're getting what we expect:

has_java = os.path.exists(os.path.join(python_home, 'jvm', 'bin', mx.exe_suffix('java')))

if has_java != (standalone_type == 'jvm'):

mx.abort(f"GRAALPY_HOME is not compatible with the requested distribution type.\n"

f"jvm/bin/java exists?: {has_java}, requested type={standalone_type}.")

line = ''

with open(os.path.join(python_home, 'release'), 'r') as f:

while 'JAVA_VERSION=' not in line:

line = f.readline()

if 'JAVA_VERSION=' not in line:

mx.abort(f"GRAALPY_HOME does not contain 'release' file. Cannot check Java version.")

actual_jdk_version = mx.VersionSpec(line.strip('JAVA_VERSION=').strip(' "\n\r'))

if actual_jdk_version != jdk_version:

mx.abort(f"GRAALPY_HOME is not compatible with the requested JDK version.\n"

f"actual version: '{actual_jdk_version}', version string: {line}, requested version: {jdk_version}.")

launcher = os.path.join(python_home, 'bin', _graalpy_launcher())

out = mx.OutputCapture()

mx.run([launcher, "-c", "print(__graalpython__.is_bytecode_dsl_interpreter)"], nonZeroIsFatal=False, out=out, err=out)

is_bytecode_dsl_interpreter = out.data.strip() == "True"

if is_bytecode_dsl_interpreter != BYTECODE_DSL_INTERPRETER:

requested = "Bytecode DSL" if BYTECODE_DSL_INTERPRETER else "Manual"

actual = "Bytecode DSL" if is_bytecode_dsl_interpreter else "Manual"

mx.abort(f"GRAALPY_HOME is not compatible with requested interpreter kind ({requested=}, {actual=})\n"

f"Used launcher: {launcher}\n"

f"Found Python home: {python_home}\n"

f"GRAALPY_HOME env variable: {os.environ.get('GRAALPY_HOME', None)}\n"

f"Raw output:\n\n{out}\n\n")

return python_home

# Build

if standalone_type == 'jvm':

if dev or jdk_version < mx.VersionSpec("22.0.0"):

env_file = 'jvm'

else:

env_file = 'jvm-ee-libgraal' if enterprise else 'jvm-ce-libgraal'

standalone_dist = 'GRAALPY_JVM_STANDALONE'

if "GraalVM" in subprocess.check_output([get_jdk().java, '-version'], stderr=subprocess.STDOUT, universal_newlines=True):

env_file = ""

else:

env_file = 'native-ee' if enterprise else 'native-ce'

standalone_dist = 'GRAALPY_NATIVE_STANDALONE'

mx_args = ['-p', SUITE.dir, *(['--env', env_file] if env_file else [])]

if GITHUB_CI:

mx_args.append("--extra-image-builder-argument=-Ob")

else:

mx_args.append("--extra-image-builder-argument=-g")

pgo_profile = os.environ.get("GRAALPY_PGO_PROFILE")

if pgo_profile is not None:

if not enterprise or standalone_type != "native":

mx.abort("PGO is only supported on enterprise NI")

if pgo_profile:

mx_args.append(f"--extra-image-builder-argument=--pgo={pgo_profile}")

mx_args.append(f"--extra-image-builder-argument=-H:+UnlockExperimentalVMOptions")

mx_args.append(f"--extra-image-builder-argument=-H:+PGOPrintProfileQuality")

else:

mx_args.append(f"--extra-image-builder-argument=--pgo-instrument")

mx_args.append(f"--extra-image-builder-argument=-H:+UnlockExperimentalVMOptions")

mx_args.append(f"--extra-image-builder-argument=-H:+ProfilingLCOV")

elif BUILD_NATIVE_IMAGE_WITH_ASSERTIONS:

mx_args.append("--extra-image-builder-argument=-ea")

mx_args.append("--extra-image-builder-argument=-J-ea")

if mx_gate.get_jacoco_agent_args() or (build and not DISABLE_REBUILD):

mx_build_args = mx_args + bytecode_dsl_build_args(prefix="--extra-image-builder-argument=")

# This build is purposefully done without the LATEST_JAVA_HOME in the

# environment, so we can build JVM standalones on an older Graal JDK

run_mx(mx_build_args + ["build", "--target", standalone_dist])

python_home = os.path.join(SUITE.dir, 'mxbuild', f"{mx.get_os()}-{mx.get_arch()}", standalone_dist)

if standalone_type == 'native':

debuginfo = os.path.join(SUITE.dir, 'mxbuild', f"{mx.get_os()}-{mx.get_arch()}", "libpythonvm", "libpythonvm.so.debug")

if os.path.exists(debuginfo):

shutil.copy(debuginfo, os.path.join(python_home, 'lib'))

return python_home

def graalpy_standalone(standalone_type, enterprise=False, dev=False, build=True):

assert standalone_type in ['native', 'jvm']

if standalone_type == 'native' and mx_gate.get_jacoco_agent_args():

return graalpy_standalone('jvm', enterprise=enterprise, dev=dev, build=build)

home = graalpy_standalone_home(standalone_type, enterprise=enterprise, dev=dev, build=build)

launcher = os.path.join(home, 'bin', _graalpy_launcher())

return make_coverage_launcher_if_needed(launcher)

def graalpy_standalone_jvm():

return graalpy_standalone('jvm')

def graalpy_standalone_native():

return graalpy_standalone('native')

def graalpy_standalone_jvm_enterprise():

return os.path.join(graalpy_standalone_home('jvm', enterprise=True), 'bin', _graalpy_launcher())

def graalpy_standalone_native_enterprise():

return os.path.join(graalpy_standalone_home('native', enterprise=True), 'bin', _graalpy_launcher())

def graalvm_jdk(enterprise=False):

jdk_version = mx.get_jdk().version

# Check if GRAAL_JDK_HOME points to some compatible pre-built gvm

graal_jdk_home = os.environ.get("GRAAL_JDK_HOME", None)

if graal_jdk_home:

graal_jdk_home = os.path.abspath(glob.glob(graal_jdk_home)[0])

if sys.platform == "darwin":

jdk_home_subdir = os.path.join(graal_jdk_home, 'Contents', 'Home')

if os.path.exists(jdk_home_subdir):

graal_jdk_home = jdk_home_subdir

mx.log("Using Graal from GRAAL_JDK_HOME: " + graal_jdk_home)

# Try to verify that we're getting what we expect:

has_java = os.path.exists(os.path.join(graal_jdk_home, 'bin', mx.exe_suffix('java')))

if not has_java:

mx.abort(f"GRAAL_JDK_HOME does not contain java executable.")

release = os.path.join(graal_jdk_home, 'release')

if not os.path.exists(release):

mx.abort(f"No 'release' file in GRAAL_JDK_HOME.")

java_version = None

with open(release, 'r') as f:

while not java_version:

line = f.readline()

if 'JAVA_VERSION=' in line:

java_version = line

if not java_version:

mx.abort(f"Could not check Java version in GRAAL_JDK_HOME 'release' file.")

actual_jdk_version = mx.VersionSpec(java_version.strip('JAVA_VERSION=').strip(' "\n\r'))

if actual_jdk_version.parts[0] != jdk_version.parts[0]:

mx.abort(f"GRAAL_JDK_HOME is not compatible with the requested JDK version.\n"

f"actual version: '{actual_jdk_version}', version string: {java_version}, requested version: {jdk_version}.")

return graal_jdk_home

jdk_major_version = mx.get_jdk().version.parts[0]

if enterprise:

mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', '..', 'graal-enterprise', 'vm-enterprise'), '--env', 'ee']

edition = ""

else:

mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', 'vm'), '--env', 'ce']

edition = "COMMUNITY_"

if not DISABLE_REBUILD:

run_mx(mx_args + ["build", "--dep", f"GRAALVM_{edition}JAVA{jdk_major_version}"], env={**os.environ, **LATEST_JAVA_HOME})

out = mx.OutputCapture()

run_mx(["--quiet"] + mx_args + ["graalvm-home"], out=out)

return out.data.splitlines()[-1].strip()

def get_maven_cache():

buildnr = os.environ.get('BUILD_NUMBER')

# don't worry about maven.repo.local if not running on gate

return os.path.join(SUITE.get_mx_output_dir(), 'm2_cache_' + buildnr) if buildnr else None

def update_maven_opts(env):

m2_cache = get_maven_cache()

if m2_cache:

mvn_repo_local = f'-Dmaven.repo.local={m2_cache}'

maven_opts = env.get('MAVEN_OPTS')

maven_opts = maven_opts + " " + mvn_repo_local if maven_opts else mvn_repo_local

if mx.is_windows():

maven_opts = maven_opts.replace("|", "^|")

env['MAVEN_OPTS'] = maven_opts

mx.log(f"Added '{mvn_repo_local}' to MAVEN_OPTS={maven_opts}")

return env

def deploy_local_maven_repo(env=None):

env = update_maven_opts({**os.environ.copy(), **(env or {})})

run_mx_args = [

'-p',

os.path.join(mx.suite('truffle').dir, '..', 'vm'),

'--dy',

'graalpython',

]

if not DISABLE_REBUILD: