Make sys._base_executable internal again and calculate it properly on…

… Windows
python · zooba · Jun 29, 2019 · Jun 26, 2019 · Jun 27, 2019 · Jun 27, 2019
commit 9a744e805ea1a9d9ee9d45c04dfa36e92884e29e
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
@@ -27,6 +27,8 @@ typedef struct _PyPathConfig {
        are ignored when their value are equal to -1 (unset). */
     int isolated;
     int site_import;
+    /* Set when a venv is detected */
+    wchar_t *base_executable;
 } _PyPathConfig;
 
 #define _PyPathConfig_INIT \

diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
@@ -22,7 +22,8 @@
 def _path_eq(p1, p2):
     return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
 
-WINENV = not _path_eq(sys.executable, sys.base_executable)
+WINENV = (hasattr(sys, '_base_executable') and
+          not _path_eq(sys.executable, sys._base_executable))
 
 
 def _close_handles(*handles):
@@ -61,7 +62,7 @@ def __init__(self, process_obj):
         # bpo-35797: When running in a venv, we bypass the redirect
         # executor and launch our base Python.
         if WINENV and _path_eq(python_exe, sys.executable):
-            python_exe = sys.base_executable
+            python_exe = sys._base_executable
             env = os.environ.copy()
             env["__PYVENV_LAUNCHER__"] = sys.executable
         else:

diff --git a/Lib/site.py b/Lib/site.py
@@ -458,7 +458,7 @@ def venv(known_paths):
 
     env = os.environ
     if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
-        executable = sys.base_executable = os.environ['__PYVENV_LAUNCHER__']
+        executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
     elif sys.platform == 'win32':
         executable = sys.executable
         if '__PYVENV_LAUNCHER__' in env:

diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
@@ -25,6 +25,13 @@
 except ImportError:
     ctypes = None
 
+# Platforms that set sys._base_executable can create venvs from within
+# another venv, so no need to skip tests that require venv.create().
+requireVenvCreate = unittest.skipUnless(
+    hasattr(sys, '_base_executable')
+    or sys.prefix == sys.base_prefix,
+    'cannot run venv.create from within a venv on this platform')
+
 def check_output(cmd, encoding=None):
     p = subprocess.Popen(cmd,
         stdout=subprocess.PIPE,
@@ -50,7 +57,7 @@ def setUp(self):
             self.bindir = 'bin'
             self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
             self.include = 'include'
-        executable = sys.base_executable
+        executable = getattr(sys, '_base_executable', sys.executable)
         self.exe = os.path.split(executable)[-1]
 
     def tearDown(self):
@@ -146,6 +153,7 @@ def pip_cmd_checker(cmd):
             with patch('venv.subprocess.check_call', pip_cmd_checker):
                 builder.upgrade_dependencies(fake_context)
 
+    @requireVenvCreate
     def test_prefixes(self):
         """
         Test that the prefix values are as expected.
@@ -281,6 +289,7 @@ def test_symlinking(self):
     # run the test, the pyvenv.cfg in the venv created in the test will
     # point to the venv being used to run the test, and we lose the link
     # to the source build - so Python can't initialise properly.
+    @requireVenvCreate
     def test_executable(self):
         """
         Test that the sys.executable value is as expected.
@@ -324,6 +333,7 @@ def test_unicode_in_batch_file(self):
         )
         self.assertEqual(out.strip(), '0')
 
+    @requireVenvCreate
     def test_multiprocessing(self):
         """
         Test that the multiprocessing is able to spawn.
@@ -343,6 +353,7 @@ def test_multiprocessing(self):
             'pool.terminate()'])
         self.assertEqual(out.strip(), "python".encode())
 
+@requireVenvCreate
 class EnsurePipTest(BaseTest):
     """Test venv module installation of pip."""
     def assert_pip_not_installed(self):

diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
@@ -112,7 +112,7 @@ def create_if_needed(d):
         prompt = self.prompt if self.prompt is not None else context.env_name
         context.prompt = '(%s) ' % prompt
         create_if_needed(env_dir)
-        executable = sys.base_executable
+        executable = getattr(sys, '_base_executable', sys.executable)
         dirname, exename = os.path.split(os.path.abspath(executable))
         context.executable = executable
         context.python_dir = dirname

diff --git a/PC/getpathp.c b/PC/getpathp.c
@@ -537,14 +537,25 @@ get_program_full_path(const PyConfig *config,
     wchar_t program_full_path[MAXPATHLEN+1];
     memset(program_full_path, 0, sizeof(program_full_path));
 
+    if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
+        /* GetModuleFileName should never fail when passed NULL */
+        return _PyStatus_ERR("Cannot determine program path");
+    }
+
     /* The launcher may need to force the executable path to a
      * different environment, so override it here. */
     pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
     if (pyvenv_launcher && pyvenv_launcher[0]) {
+        /* If overridden, preserve the original full path */
+        pathconfig->base_executable = PyMem_RawMalloc(
+            sizeof(wchar_t) * (MAXPATHLEN + 1));
+
+        PyStatus status = canonicalize(pathconfig->base_executable,
+                                       program_full_path);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
         wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
-    } else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
-        /* GetModuleFileName should never fail when passed NULL */
-        return _PyStatus_ERR("Cannot determine program path");
     }
 
     pathconfig->program_full_path = PyMem_RawMalloc(

diff --git a/Python/pathconfig.c b/Python/pathconfig.c
@@ -57,6 +57,7 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->module_search_path);
     CLEAR(config->home);
     CLEAR(config->program_name);
+    CLEAR(config->base_executable);
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -89,6 +90,14 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config)
         status = _PyStatus_NO_MEMORY();
         goto error;
     }
+    if (config->base_executable) {
+        PyMem_RawFree(new_config.base_executable);
+        if (copy_wstr(&new_config.base_executable,
+                      config->base_executable) < 0) {
+            status = _PyStatus_NO_MEMORY();
+            goto error;
+        }
+    }
 
     pathconfig_clear(pathconfig);
     *pathconfig = new_config;
@@ -132,6 +141,7 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config)
     COPY_ATTR(module_search_path);
     COPY_ATTR(program_name);
     COPY_ATTR(home);
+    COPY_ATTR(base_executable);
 
     pathconfig_clear(&_Py_path_config);
     /* Steal new_config strings; don't clear new_config */
@@ -224,6 +234,9 @@ _PyConfig_SetPathConfig(const PyConfig *config)
     if (copy_wstr(&pathconfig.home, config->home) < 0) {
         goto no_memory;
     }
+    if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) {
+        goto no_memory;
+    }
 
     status = _PyPathConfig_SetGlobal(&pathconfig);
     if (_PyStatus_EXCEPTION(status)) {
@@ -321,6 +334,13 @@ config_calculate_pathconfig(PyConfig *config)
         }
     }
 
+    if (config->base_executable == NULL) {
+        if (copy_wstr(&config->base_executable,
+                      pathconfig.base_executable) < 0) {
+            goto no_memory;
+        }
+    }
+
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
     }
@@ -442,6 +462,8 @@ Py_SetPath(const wchar_t *path)
     _Py_path_config.home = NULL;
     new_config.program_name = _Py_path_config.program_name;
     _Py_path_config.program_name = NULL;
+    new_config.base_executable = _Py_path_config.base_executable;
+    _Py_path_config.base_executable = NULL;
 
     pathconfig_clear(&_Py_path_config);
     _Py_path_config = new_config;

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
@@ -2850,7 +2850,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyThreadState *tstate)
     COPY_LIST("path", config->module_search_paths);
 
     SET_SYS_FROM_WSTR("executable", config->executable);
-    SET_SYS_FROM_WSTR("base_executable", config->base_executable);
+    SET_SYS_FROM_WSTR("_base_executable", config->base_executable);
     SET_SYS_FROM_WSTR("prefix", config->prefix);
     SET_SYS_FROM_WSTR("base_prefix", config->base_prefix);
     SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix);