Skip to content

Commit 684a632

Browse files
gaborbernatgaborbernat
authored
Improve discovery on Windows and provide escape hatchet (#2046)
Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>
1 parent 9201a75 commit 684a632

8 files changed

Lines changed: 53 additions & 17 deletions

File tree

‎docs/changelog/1986.bugfix.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
On Windows python ``3.7+`` distributions where the exe shim is missing fallback to the old ways - by :user:`gaborbernat`.

‎docs/changelog/2046.bugfix.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
When discovering interpreters on Windows, via the PEP-514, prefer ``PythonCore`` releases over other ones. virtualenv
2+
is used via pip mostly by this distribution, so prefer it over other such as conda - by :user:`gaborbernat`.

‎docs/changelog/2046.feature.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The builtin discovery takes now a ``--try-first-with`` argument and is first attempted as valid interpreters. One can
2+
use this to force discovery of a given python executable when the discovery order/mechanism raises errors -
3+
by :user:`gaborbernat`.

‎src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,27 @@ def setup_meta(cls, interpreter):
5555
def sources(cls, interpreter):
5656
for src in super(CPython3Windows, cls).sources(interpreter):
5757
yield src
58-
if not cls.venv_37p(interpreter):
58+
if not cls.has_shim(interpreter):
5959
for src in cls.include_dll_and_pyd(interpreter):
6060
yield src
6161

62-
@staticmethod
63-
def venv_37p(interpreter):
64-
return interpreter.version_info.minor >= 7
62+
@classmethod
63+
def has_shim(cls, interpreter):
64+
return interpreter.version_info.minor >= 7 and cls.shim(interpreter) is not None
65+
66+
@classmethod
67+
def shim(cls, interpreter):
68+
shim = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe"
69+
if shim.exists():
70+
return shim
71+
return None
6572

6673
@classmethod
6774
def host_python(cls, interpreter):
68-
if cls.venv_37p(interpreter):
75+
if cls.has_shim(interpreter):
6976
# starting with CPython 3.7 Windows ships with a venvlauncher.exe that avoids the need for dll/pyd copies
7077
# it also means the wrapper must be copied to avoid bugs such as https://bugs.python.org/issue42013
71-
return Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe"
78+
return cls.shim(interpreter)
7279
return super(CPython3Windows, cls).host_python(interpreter)
7380

7481
@classmethod

‎src/virtualenv/discovery/builtin.py‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ def __init__(self, options):
1717
super(Builtin, self).__init__(options)
1818
self.python_spec = options.python if options.python else [sys.executable]
1919
self.app_data = options.app_data
20+
self.try_first_with = options.try_first_with
2021

2122
@classmethod
2223
def add_parser_arguments(cls, parser):
@@ -31,10 +32,19 @@ def add_parser_arguments(cls, parser):
3132
help="interpreter based on what to create environment (path/identifier) "
3233
"- by default use the interpreter where the tool is installed - first found wins",
3334
)
35+
parser.add_argument(
36+
"--try-first-with",
37+
dest="try_first_with",
38+
metavar="py_exe",
39+
type=str,
40+
action="append",
41+
default=[],
42+
help="try first these interpreters before starting the discovery",
43+
)
3444

3545
def run(self):
3646
for python_spec in self.python_spec:
37-
result = get_interpreter(python_spec, self.app_data)
47+
result = get_interpreter(python_spec, self.try_first_with, self.app_data)
3848
if result is not None:
3949
return result
4050
return None
@@ -47,11 +57,11 @@ def __unicode__(self):
4757
return "{} discover of python_spec={!r}".format(self.__class__.__name__, spec)
4858

4959

50-
def get_interpreter(key, app_data=None):
60+
def get_interpreter(key, try_first_with, app_data=None):
5161
spec = PythonSpec.from_string_spec(key)
5262
logging.info("find interpreter for spec %r", spec)
5363
proposed_paths = set()
54-
for interpreter, impl_must_match in propose_interpreters(spec, app_data):
64+
for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data):
5565
key = interpreter.system_executable, impl_must_match
5666
if key in proposed_paths:
5767
continue
@@ -62,7 +72,17 @@ def get_interpreter(key, app_data=None):
6272
proposed_paths.add(key)
6373

6474

65-
def propose_interpreters(spec, app_data):
75+
def propose_interpreters(spec, try_first_with, app_data):
76+
# 0. try with first
77+
for py_exe in try_first_with:
78+
path = os.path.abspath(py_exe)
79+
try:
80+
os.lstat(path) # Windows Store Python does not work with os.path.exists, but does for os.lstat
81+
except OSError:
82+
pass
83+
else:
84+
yield PythonInfo.from_exe(os.path.abspath(path), app_data), True
85+
6686
# 1. if it's a path and exists
6787
if spec.path is not None:
6888
try:

‎src/virtualenv/discovery/windows/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,11 @@ def propose_interpreters(spec, cache_dir):
1313
# see if PEP-514 entries are good
1414

1515
# start with higher python versions in an effort to use the latest version available
16+
# and prefer PythonCore over conda pythons (as virtualenv is mostly used by non conda tools)
1617
existing = list(discover_pythons())
17-
existing.sort(key=lambda i: tuple(-1 if j is None else j for j in i[1:4]), reverse=True)
18+
existing.sort(
19+
key=lambda i: tuple(-1 if j is None else j for j in i[1:4]) + (1 if i[0] == "PythonCore" else 0,), reverse=True
20+
)
1821

1922
for name, major, minor, arch, exe, _ in existing:
2023
# pre-filter

‎tests/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ def temp_app_data(monkeypatch, tmp_path):
367367
@pytest.fixture(scope="session")
368368
def cross_python(is_inside_ci, session_app_data):
369369
spec = str(2 if sys.version_info[0] == 3 else 3)
370-
interpreter = get_interpreter(spec, session_app_data)
370+
interpreter = get_interpreter(spec, [], session_app_data)
371371
if interpreter is None:
372372
msg = "could not find {}".format(spec)
373373
if is_inside_ci:

‎tests/unit/discovery/test_discovery.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,14 @@ def test_discovery_via_path(monkeypatch, case, tmp_path, caplog, session_app_dat
3636
(target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes())
3737
new_path = os.pathsep.join([str(target)] + os.environ.get(str("PATH"), str("")).split(os.pathsep))
3838
monkeypatch.setenv(str("PATH"), new_path)
39-
interpreter = get_interpreter(core)
39+
interpreter = get_interpreter(core, [])
4040

4141
assert interpreter is not None
4242

4343

4444
def test_discovery_via_path_not_found(tmp_path, monkeypatch):
4545
monkeypatch.setenv(str("PATH"), str(tmp_path))
46-
interpreter = get_interpreter(uuid4().hex)
46+
interpreter = get_interpreter(uuid4().hex, [])
4747
assert interpreter is None
4848

4949

@@ -52,13 +52,13 @@ def test_relative_path(tmp_path, session_app_data, monkeypatch):
5252
cwd = sys_executable.parents[1]
5353
monkeypatch.chdir(str(cwd))
5454
relative = str(sys_executable.relative_to(cwd))
55-
result = get_interpreter(relative, session_app_data)
55+
result = get_interpreter(relative, [], session_app_data)
5656
assert result is not None
5757

5858

5959
def test_discovery_fallback_fail(session_app_data, caplog):
6060
caplog.set_level(logging.DEBUG)
61-
builtin = Builtin(Namespace(app_data=session_app_data, python=["magic-one", "magic-two"]))
61+
builtin = Builtin(Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"]))
6262

6363
result = builtin.run()
6464
assert result is None
@@ -68,7 +68,7 @@ def test_discovery_fallback_fail(session_app_data, caplog):
6868

6969
def test_discovery_fallback_ok(session_app_data, caplog):
7070
caplog.set_level(logging.DEBUG)
71-
builtin = Builtin(Namespace(app_data=session_app_data, python=["magic-one", sys.executable]))
71+
builtin = Builtin(Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable]))
7272

7373
result = builtin.run()
7474
assert result is not None, caplog.text

0 commit comments

Comments
 (0)