diff --git a/.gitignore b/.gitignore index c03ae1a997e..d173739854d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py +Lib/_sysconfig_vars*.json Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 8365236f61c..ae83243fa3e 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -116,8 +116,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories - if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -323,17 +325,37 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -341,26 +363,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) + +def _get_sysconfigdata(): + import importlib + + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) + + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() + # GH-126920: Make sure we don't overwrite any of the keys already set + vars.update(_get_sysconfigdata() | vars) - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -371,9 +401,20 @@ def _init_non_posix(vars): vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED + # Add EXT_SUFFIX, SOABI, Py_DEBUG, and Py_GIL_DISABLED vars.update(_sysconfig.config_vars()) + # NOTE: ABIFLAGS is only an emulated value. It is not present during build + # on Windows. sys.abiflags is absent on Windows and vars['abiflags'] + # is already widely used to calculate paths, so it should remain an + # empty string. + vars['ABIFLAGS'] = ''.join( + ( + 't' if vars['Py_GIL_DISABLED'] else '', + '_d' if vars['Py_DEBUG'] else '', + ), + ) + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) if hasattr(sys, 'dllhandle'): dllhandle = _winapi.GetModuleFileName(sys.dllhandle) @@ -427,7 +468,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: @@ -468,29 +509,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['host_prefix'] + exec_prefix = _CONFIG_VARS['host_exec_prefix'] + base_prefix = _CONFIG_VARS['host_prefix'] + base_exec_prefix = _CONFIG_VARS['host_exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _PREFIX = os.path.normpath(sys.prefix) - _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) - _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -499,8 +555,6 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in @@ -550,7 +604,16 @@ def get_config_vars(*args): global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. - if not _CONFIG_VARS_INITIALIZED: + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: # Test again with the lock held to avoid races. Note that # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, @@ -558,15 +621,6 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() - else: - # If the site module initialization happened after _CONFIG_VARS was - # initialized, a virtual environment might have been activated, resulting in - # variables like sys.prefix changing their value, so we need to re-init the - # config vars (see GH-126789). - if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): - with _CONFIG_VARS_LOCK: - _CONFIG_VARS_INITIALIZED = False - _init_config_vars() if args: vals = [] @@ -627,34 +681,34 @@ def get_platform(): # Set for cross builds explicitly if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - # Try to distinguish various flavours of Unix - osname, host, release, version, machine = os.uname() - - # Convert the OS name to lowercase, remove '/' characters, and translate - # spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - if sys.platform == "android": - osname = "android" - release = get_config_var("ANDROID_API_LEVEL") - - # Wheel tags use the ABI names from Android's own tools. - machine = { - "x86_64": "x86_64", - "i686": "x86", - "aarch64": "arm64_v8a", - "armv7l": "armeabi_v7a", - }[machine] - else: - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + osname, _, machine = os.environ["_PYTHON_HOST_PLATFORM"].partition('-') + release = None + else: + # Try to distinguish various flavours of Unix + osname, host, release, version, machine = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname == "android" or sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + elif osname == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -686,7 +740,7 @@ def get_platform(): get_config_vars(), osname, release, machine) - return f"{osname}-{release}-{machine}" + return '-'.join(map(str, filter(None, (osname, release, machine)))) def get_python_version(): @@ -705,6 +759,15 @@ def expand_makefile_vars(s, vars): variable expansions; if 'vars' is the output of 'parse_makefile()', you're fine. Returns a variable-expanded version of 's'. """ + + import warnings + warnings.warn( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + DeprecationWarning, + stacklevel=2, + ) + import re _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d0..bc2197cfe79 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,10 +1,13 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -157,6 +160,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if get_config_var('Py_DEBUG') == '1': + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +201,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +214,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +230,19 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(install_vars, f, indent=2) + + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 4a14cb35213..0813c4804c1 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -20,12 +20,13 @@ def setUp(self): self.prog_name = 'bogus_program_xxxx' self.temp_path_dir = os.path.abspath(os.getcwd()) self.env = self.enterContext(os_helper.EnvironmentVarGuard()) - for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', - 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', - 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS'): - if cv in self.env: - self.env.unset(cv) + + self.env.unset( + 'CFLAGS', 'LDFLAGS', 'CPPFLAGS', + 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', + 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS' + ) def add_expected_saved_initial_values(self, config_vars, expected_vars): # Ensure that the initial values for all modified config vars @@ -65,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 82c11bdf7e2..7da1326a721 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -9,8 +9,10 @@ import textwrap from copy import copy +from test import support from test.support import ( captured_stdout, + is_android, is_apple_mobile, is_wasi, PythonSymlink, @@ -19,14 +21,15 @@ from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.venv import VirtualEnvironment +from test.support.venv import VirtualEnvironmentMixin import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes) -from sysconfig.__main__ import _main, _parse_makefile + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support import _sysconfig @@ -35,10 +38,11 @@ HAS_USER_BASE = sysconfig._HAS_USER_BASE -class TestSysConfig(unittest.TestCase): +class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): @@ -50,6 +54,8 @@ def setUp(self): os.uname = self._get_uname # saving the environment self.name = os.name + self.prefix = sys.prefix + self.exec_prefix = sys.exec_prefix self.platform = sys.platform self.version = sys.version self._framework = sys._framework @@ -74,6 +80,8 @@ def tearDown(self): else: del os.uname os.name = self.name + sys.prefix = self.prefix + sys.exec_prefix = self.exec_prefix sys.platform = self.platform sys.version = self.version sys._framework = self._framework @@ -104,12 +112,6 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) - def venv(self, **venv_create_args): - return VirtualEnvironment.from_tmpdir( - prefix=f'{self.id()}-venv-', - **venv_create_args, - ) - def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -160,14 +162,14 @@ def test_get_preferred_schemes(self): self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - # XXX: RUSTPYTHON - f'rustpython{sysconfig._get_python_version_abi()}', + f'python{sysconfig._get_python_version_abi()}', 'site-packages') # Resolve the paths in an imaginary venv/ directory @@ -185,7 +187,7 @@ def test_posix_venv_scheme(self): # The include directory on POSIX isn't exactly the same as before, # but it is "within" sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + self.assertStartsWith(sysconfig_includedir, incpath + os.sep) def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module @@ -386,7 +388,7 @@ def test_get_platform(self): # XXX more platforms to tests here - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : /Users/youknowone/Projects/RustPython2/include/python3.14t/pyconfig.h @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") @unittest.skipIf(is_apple_mobile, f"{sys.platform} doesn't distribute header files in the runtime environment") @@ -459,25 +461,25 @@ def test_soabi(self): soabi = sysconfig.get_config_var('SOABI') self.assertIn(soabi, _imp.extension_suffixes()[0]) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Expected str, not NoneType def test_library(self): library = sysconfig.get_config_var('LIBRARY') ldlibrary = sysconfig.get_config_var('LDLIBRARY') major, minor = sys.version_info[:2] - if sys.platform == 'win32': - self.assertTrue(library.startswith(f'python{major}{minor}')) - self.assertTrue(library.endswith('.dll')) + abiflags = sysconfig.get_config_var('ABIFLAGS') + if sys.platform.startswith('win'): + self.assertEqual(library, f'python{major}{minor}{abiflags}.dll') self.assertEqual(library, ldlibrary) elif is_apple_mobile: framework = sysconfig.get_config_var('PYTHONFRAMEWORK') self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") else: - self.assertTrue(library.startswith(f'libpython{major}.{minor}')) - self.assertTrue(library.endswith('.a')) + self.assertStartsWith(library, f'libpython{major}.{minor}') + self.assertEndsWith(library, '.a') if sys.platform == 'darwin' and sys._framework: self.skipTest('gh-110824: skip LDLIBRARY test for framework build') else: - self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + self.assertStartsWith(ldlibrary, f'libpython{major}.{minor}') @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() @@ -523,7 +525,7 @@ def test_platform_in_subprocess(self): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) - @unittest.expectedFailureIf(sys.platform != 'win32', 'TODO: RUSTPYTHON') + @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") @unittest.skipIf(is_apple_mobile, f"{sys.platform} doesn't include config folder at runtime") @@ -540,13 +542,10 @@ def test_srcdir(self): Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) # /PC/pyconfig.h.in always exists even if unused - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) if os.name == 'nt': - # /pyconfig.h exists on Windows in a build tree - pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) @@ -581,10 +580,8 @@ def test_linux_ext_suffix(self): expected_suffixes = 'i386-linux-gnu.so', 'x86_64-linux-gnux32.so', 'i386-linux-musl.so' else: # 8 byte pointer size expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' - self.assertTrue(suffix.endswith(expected_suffixes), - f'unexpected suffix {suffix!r}') + self.assertEndsWith(suffix, expected_suffixes) - @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') def test_android_ext_suffix(self): machine = platform.machine() @@ -595,86 +592,169 @@ def test_android_ext_suffix(self): "aarch64": "aarch64-linux-android", "armv7l": "arm-linux-androideabi", }[machine] - self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), - f"{machine=}, {suffix=}") + self.assertEndsWith(suffix, f"-{expected_triplet}.so") @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(suffix.endswith('-darwin.so'), suffix) + self.assertEndsWith(suffix, '-darwin.so') + + def test_always_set_py_debug(self): + self.assertIn('Py_DEBUG', sysconfig.get_config_vars()) + Py_DEBUG = sysconfig.get_config_var('Py_DEBUG') + self.assertIn(Py_DEBUG, (0, 1)) + self.assertEqual(Py_DEBUG, support.Py_DEBUG) + + def test_always_set_py_gil_disabled(self): + self.assertIn('Py_GIL_DISABLED', sysconfig.get_config_vars()) + Py_GIL_DISABLED = sysconfig.get_config_var('Py_GIL_DISABLED') + self.assertIn(Py_GIL_DISABLED, (0, 1)) + self.assertEqual(Py_GIL_DISABLED, support.Py_GIL_DISABLED) + + def test_abiflags(self): + # If this test fails on some platforms, maintainers should update the + # test to make it pass, rather than changing the definition of ABIFLAGS. + self.assertIn('abiflags', sysconfig.get_config_vars()) + self.assertIn('ABIFLAGS', sysconfig.get_config_vars()) + abiflags = sysconfig.get_config_var('abiflags') + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + self.assertIsInstance(abiflags, str) + self.assertIsInstance(ABIFLAGS, str) + self.assertIn(abiflags, ABIFLAGS) + if os.name == 'nt': + self.assertEqual(abiflags, '') + + if not sys.platform.startswith('win'): + valid_abiflags = ('', 't', 'd', 'td') + else: + # Windows uses '_d' rather than 'd'; see also test_abi_debug below + valid_abiflags = ('', 't', '_d', 't_d') + + self.assertIn(ABIFLAGS, valid_abiflags) + + def test_abi_debug(self): + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + if support.Py_DEBUG: + self.assertIn('d', ABIFLAGS) + else: + self.assertNotIn('d', ABIFLAGS) + + # The 'd' flag should always be the last one on Windows. + # On Windows, the debug flag is used differently with a underscore prefix. + # For example, `python{X}.{Y}td` on Unix and `python{X}.{Y}t_d.exe` on Windows. + if support.Py_DEBUG and sys.platform.startswith('win'): + self.assertEndsWith(ABIFLAGS, '_d') + + def test_abi_thread(self): + abi_thread = sysconfig.get_config_var('abi_thread') + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + self.assertIsInstance(abi_thread, str) + if support.Py_GIL_DISABLED: + self.assertEqual(abi_thread, 't') + self.assertIn('t', ABIFLAGS) + else: + self.assertEqual(abi_thread, '') + self.assertNotIn('t', ABIFLAGS) - @unittest.expectedFailure # TODO: RUSTPYTHON @requires_subprocess() - def test_config_vars_depend_on_site_initialization(self): + @unittest.skipIf(os.name == 'nt', 'TODO: RUSTPYTHON; venv creates python.exe but sys.executable is rustpython.exe') + def test_makefile_overwrites_config_vars(self): script = textwrap.dedent(""" - import sysconfig + import sys, sysconfig - config_vars = sysconfig.get_config_vars() + data = { + 'prefix': sys.prefix, + 'exec_prefix': sys.exec_prefix, + 'base_prefix': sys.base_prefix, + 'base_exec_prefix': sys.base_exec_prefix, + 'config_vars': sysconfig.get_config_vars(), + } import json - print(json.dumps(config_vars, indent=2)) + print(json.dumps(data, indent=2)) """) + # We need to run the test inside a virtual environment so that + # sys.prefix/sys.exec_prefix have a different value from the + # prefix/exec_prefix Makefile variables. with self.venv() as venv: - site_config_vars = json.loads(venv.run('-c', script).stdout) - no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(site_config_vars, no_site_config_vars) - # With the site initialization, the virtual environment should be enabled. - self.assertEqual(site_config_vars['base'], venv.prefix) - self.assertEqual(site_config_vars['platbase'], venv.prefix) - #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix - # Without the site initialization, the virtual environment should be disabled. - self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) - self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) - - @unittest.expectedFailure # TODO: RUSTPYTHON - @requires_subprocess() - def test_config_vars_recalculation_after_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig + data = json.loads(venv.run('-c', script).stdout) + + # We expect sysconfig.get_config_vars to correctly reflect sys.prefix/sys.exec_prefix + self.assertEqual(data['prefix'], data['config_vars']['prefix']) + self.assertEqual(data['exec_prefix'], data['config_vars']['exec_prefix']) + # As a sanity check, just make sure sys.prefix/sys.exec_prefix really + # are different from the Makefile values. + # sys.base_prefix/sys.base_exec_prefix should reflect the value of the + # prefix/exec_prefix Makefile variables, so we use them in the comparison. + self.assertNotEqual(data['prefix'], data['base_prefix']) + self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.expectedFailure # TODO: RUSTPYTHON; JSON is generated at build time in CPython + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir - before = sysconfig.get_config_vars() + json_data_path = os.path.join(data_dir, _get_json_data_name()) - import site - site.main() + with open(json_data_path) as f: + json_config_vars = json.load(f) - after = sysconfig.get_config_vars() + system_config_vars = get_config_vars() - import json - print(json.dumps({'before': before, 'after': after}, indent=2)) - """) + # Keys dependent on uncontrollable external context + ignore_keys = {'userbase'} + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} - with self.venv() as venv: - config_vars = json.loads(venv.run('-S', '-c', script).stdout) + for key in ignore_keys: + json_config_vars.pop(key, None) + system_config_vars.pop(key, None) - self.assertNotEqual(config_vars['before'], config_vars['after']) - self.assertEqual(config_vars['after']['base'], venv.prefix) - #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix - #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix + self.assertEqual(system_config_vars, json_config_vars) - @unittest.expectedFailure # TODO: RUSTPYTHON - @requires_subprocess() - def test_paths_depend_on_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig + def test_sysconfig_config_vars_no_prefix_cache(self): + sys.prefix = 'prefix-AAA' + sys.exec_prefix = 'exec-prefix-AAA' - paths = sysconfig.get_paths() + config_vars = sysconfig.get_config_vars() - import json - print(json.dumps(paths, indent=2)) - """) + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) - with self.venv() as venv: - site_paths = json.loads(venv.run('-c', script).stdout) - no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) + sys.prefix = 'prefix-BBB' + sys.exec_prefix = 'exec-prefix-BBB' - self.assertNotEqual(site_paths, no_site_paths) + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) class MakefileTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : /Users/youknowone/Projects/RustPython2/lib/python3.14t/config-3.14t-aarch64-apple-darwin/Makefile @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") @@ -712,6 +792,18 @@ def deprecated(self, removal_version, deprecation_msg=None, error=Exception, err else: return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + def test_expand_makefile_vars(self): + with self.deprecated( + removal_version=(3, 16), + deprecation_msg=( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + ), + error=AttributeError, + error_msg="module 'sysconfig' has no attribute 'expand_makefile_vars'", + ): + sysconfig.expand_makefile_vars('', {}) + def test_is_python_build_check_home(self): with self.deprecated( removal_version=(3, 15), diff --git a/crates/vm/src/stdlib/sysconfig.rs b/crates/vm/src/stdlib/sysconfig.rs index 724e1bcf979..2de27d71ac9 100644 --- a/crates/vm/src/stdlib/sysconfig.rs +++ b/crates/vm/src/stdlib/sysconfig.rs @@ -15,10 +15,9 @@ pub(crate) mod _sysconfig { .unwrap(); vars.set_item("SOABI", vm.ctx.none(), vm).unwrap(); - vars.set_item("Py_GIL_DISABLED", true.to_pyobject(vm), vm) - .unwrap(); - vars.set_item("Py_DEBUG", false.to_pyobject(vm), vm) + vars.set_item("Py_GIL_DISABLED", (1).to_pyobject(vm), vm) .unwrap(); + vars.set_item("Py_DEBUG", (0).to_pyobject(vm), vm).unwrap(); vars } diff --git a/crates/vm/src/stdlib/sysconfigdata.rs b/crates/vm/src/stdlib/sysconfigdata.rs index a10745a8cad..99aab892e1c 100644 --- a/crates/vm/src/stdlib/sysconfigdata.rs +++ b/crates/vm/src/stdlib/sysconfigdata.rs @@ -14,6 +14,14 @@ mod _sysconfigdata { fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { // Set build_time_vars attribute let build_time_vars = build_time_vars(vm); + + // Add runtime-dependent values needed by sysconfig + let paths = &vm.state.config.paths; + build_time_vars.set_item("prefix", paths.prefix.clone().to_pyobject(vm), vm)?; + build_time_vars.set_item("exec_prefix", paths.exec_prefix.clone().to_pyobject(vm), vm)?; + let bindir = format!("{}/bin", &paths.exec_prefix); + build_time_vars.set_item("BINDIR", bindir.to_pyobject(vm), vm)?; + module.set_attr("build_time_vars", build_time_vars, vm)?; // Ensure the module is registered under the platform-specific name @@ -43,6 +51,8 @@ mod _sysconfigdata { "HAVE_GETRANDOM" => 1, // RustPython has no GIL (like free-threaded Python) "Py_GIL_DISABLED" => 1, + "Py_DEBUG" => 0, + "ABIFLAGS" => "t", // Compiler configuration for native extension builds "CC" => "cc", "CXX" => "c++",