Skip to content

Commit 0c50b8c

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45211: Remember the stdlib dir during startup. (pythongh-28586)
During runtime startup we figure out the stdlib dir but currently throw that information away. This change preserves it and exposes it via PyConfig.stdlib_dir, _Py_GetStdlibDir(), and sys._stdlib_dir. https://bugs.python.org/issue45211
1 parent 8497514 commit 0c50b8c

10 files changed

Lines changed: 93 additions & 3 deletions

File tree

‎Include/cpython/initconfig.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ typedef struct PyConfig {
184184
/* --- Path configuration outputs ----------- */
185185
int module_search_paths_set;
186186
PyWideStringList module_search_paths;
187+
wchar_t *stdlib_dir;
187188
wchar_t *executable;
188189
wchar_t *base_executable;
189190
wchar_t *prefix;

‎Include/internal/pycore_pathconfig.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ typedef struct _PyPathConfig {
1313
wchar_t *program_full_path;
1414
wchar_t *prefix;
1515
wchar_t *exec_prefix;
16+
wchar_t *stdlib_dir;
1617
/* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
1718
wchar_t *module_search_path;
1819
/* Python program name */

‎Include/internal/pycore_pylifecycle.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig(
122122
const PyConfig *config,
123123
const struct _PyArgv *args);
124124

125+
PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void);
125126

126127
PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
127128

‎Lib/test/test_embed.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
406406
'module_search_paths': GET_DEFAULT_CONFIG,
407407
'module_search_paths_set': 1,
408408
'platlibdir': sys.platlibdir,
409+
'stdlib_dir': GET_DEFAULT_CONFIG,
409410

410411
'site_import': 1,
411412
'bytes_warning': 0,
@@ -515,6 +516,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
515516
'exec_prefix',
516517
'program_name',
517518
'home',
519+
'stdlib_dir',
518520
# program_full_path and module_search_path are copied indirectly from
519521
# the core configuration in check_path_config().
520522
]
@@ -1142,6 +1144,9 @@ def test_init_setpath(self):
11421144
'base_prefix': '',
11431145
'exec_prefix': '',
11441146
'base_exec_prefix': '',
1147+
# The current getpath.c doesn't determine the stdlib dir
1148+
# in this case.
1149+
'stdlib_dir': '',
11451150
}
11461151
self.default_program_name(config)
11471152
env = {'TESTPATH': os.path.pathsep.join(paths)}
@@ -1162,6 +1167,9 @@ def test_init_setpath_config(self):
11621167
'base_prefix': '',
11631168
'exec_prefix': '',
11641169
'base_exec_prefix': '',
1170+
# The current getpath.c doesn't determine the stdlib dir
1171+
# in this case.
1172+
'stdlib_dir': '',
11651173
# overriden by PyConfig
11661174
'program_name': 'conf_program_name',
11671175
'base_executable': 'conf_executable',
@@ -1251,6 +1259,7 @@ def test_init_setpythonhome(self):
12511259
'exec_prefix': exec_prefix,
12521260
'base_exec_prefix': exec_prefix,
12531261
'pythonpath_env': paths_str,
1262+
'stdlib_dir': home,
12541263
}
12551264
self.default_program_name(config)
12561265
env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
@@ -1288,6 +1297,9 @@ def test_init_pybuilddir(self):
12881297
'base_executable': executable,
12891298
'executable': executable,
12901299
'module_search_paths': module_search_paths,
1300+
# The current getpath.c doesn't determine the stdlib dir
1301+
# in this case.
1302+
'stdlib_dir': None,
12911303
}
12921304
env = self.copy_paths_by_env(config)
12931305
self.check_all_configs("test_init_compat_config", config,
@@ -1345,6 +1357,7 @@ def test_init_pyvenv_cfg(self):
13451357
if MS_WINDOWS:
13461358
config['base_prefix'] = pyvenv_home
13471359
config['prefix'] = pyvenv_home
1360+
config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
13481361

13491362
ver = sys.version_info
13501363
dll = f'python{ver.major}'
@@ -1353,6 +1366,10 @@ def test_init_pyvenv_cfg(self):
13531366
dll += '.DLL'
13541367
dll = os.path.join(os.path.dirname(executable), dll)
13551368
path_config['python3_dll'] = dll
1369+
else:
1370+
# The current getpath.c doesn't determine the stdlib dir
1371+
# in this case.
1372+
config['stdlib_dir'] = None
13561373

13571374
env = self.copy_paths_by_env(config)
13581375
self.check_all_configs("test_init_compat_config", config,

‎Lib/test/test_sys.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from test.support import os_helper
1414
from test.support.script_helper import assert_python_ok, assert_python_failure
1515
from test.support import threading_helper
16+
from test.support import import_helper
1617
import textwrap
1718
import unittest
1819
import warnings
@@ -994,6 +995,15 @@ def test_module_names(self):
994995
for name in sys.stdlib_module_names:
995996
self.assertIsInstance(name, str)
996997

998+
def test_stdlib_dir(self):
999+
os = import_helper.import_fresh_module('os')
1000+
marker = getattr(os, '__file__', None)
1001+
if marker and not os.path.exists(marker):
1002+
marker = None
1003+
expected = os.path.dirname(marker) if marker else None
1004+
actual = sys._stdlib_dir
1005+
self.assertEqual(actual, expected)
1006+
9971007

9981008
@test.support.cpython_only
9991009
class UnraisableHookTest(unittest.TestCase):

‎Modules/getpath.c‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1492,6 +1492,16 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
14921492
}
14931493
}
14941494

1495+
if (pathconfig->stdlib_dir == NULL) {
1496+
if (calculate->prefix_found) {
1497+
/* This must be done *before* calculate_set_prefix() is called. */
1498+
pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix);
1499+
if (pathconfig->stdlib_dir == NULL) {
1500+
return _PyStatus_NO_MEMORY();
1501+
}
1502+
}
1503+
}
1504+
14951505
if (pathconfig->prefix == NULL) {
14961506
status = calculate_set_prefix(calculate, pathconfig);
14971507
if (_PyStatus_EXCEPTION(status)) {

‎PC/getpathp.c‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@
116116
* with a semicolon separated path prior to calling Py_Initialize.
117117
*/
118118

119+
#define STDLIB_SUBDIR L"lib"
120+
119121
#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
120122

121123

@@ -293,12 +295,12 @@ search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path)
293295
wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix);
294296
/* We initialize with the longest possible path, in case it doesn't fit.
295297
This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */
296-
join(stdlibdir, L"lib");
298+
join(stdlibdir, STDLIB_SUBDIR);
297299
do {
298300
assert(stdlibdir[wcslen(prefix)] == SEP);
299301
/* Due to reduce() and our initial value, this result
300302
is guaranteed to fit. */
301-
wcscpy(&stdlibdir[wcslen(prefix) + 1], L"lib");
303+
wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR);
302304
if (is_stdlibdir(stdlibdir)) {
303305
return 1;
304306
}
@@ -1013,6 +1015,12 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
10131015
}
10141016

10151017
done:
1018+
if (pathconfig->stdlib_dir == NULL) {
1019+
pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR);
1020+
if (pathconfig->stdlib_dir == NULL) {
1021+
return _PyStatus_NO_MEMORY();
1022+
}
1023+
}
10161024
if (pathconfig->prefix == NULL) {
10171025
pathconfig->prefix = _PyMem_RawWcsdup(prefix);
10181026
if (pathconfig->prefix == NULL) {

‎Python/initconfig.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,7 @@ PyConfig_Clear(PyConfig *config)
669669
_PyWideStringList_Clear(&config->xoptions);
670670
_PyWideStringList_Clear(&config->module_search_paths);
671671
config->module_search_paths_set = 0;
672+
CLEAR(config->stdlib_dir);
672673

673674
CLEAR(config->executable);
674675
CLEAR(config->base_executable);
@@ -909,6 +910,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
909910
COPY_WSTRLIST(xoptions);
910911
COPY_WSTRLIST(module_search_paths);
911912
COPY_ATTR(module_search_paths_set);
913+
COPY_WSTR_ATTR(stdlib_dir);
912914

913915
COPY_WSTR_ATTR(executable);
914916
COPY_WSTR_ATTR(base_executable);
@@ -1015,6 +1017,7 @@ _PyConfig_AsDict(const PyConfig *config)
10151017
SET_ITEM_WSTR(home);
10161018
SET_ITEM_INT(module_search_paths_set);
10171019
SET_ITEM_WSTRLIST(module_search_paths);
1020+
SET_ITEM_WSTR(stdlib_dir);
10181021
SET_ITEM_WSTR(executable);
10191022
SET_ITEM_WSTR(base_executable);
10201023
SET_ITEM_WSTR(prefix);
@@ -1318,6 +1321,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
13181321
// Path configuration output
13191322
GET_UINT(module_search_paths_set);
13201323
GET_WSTRLIST(module_search_paths);
1324+
GET_WSTR_OPT(stdlib_dir);
13211325
GET_WSTR_OPT(executable);
13221326
GET_WSTR_OPT(base_executable);
13231327
GET_WSTR_OPT(prefix);
@@ -3094,6 +3098,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
30943098
PySys_WriteStderr(" environment = %i\n", config->use_environment);
30953099
PySys_WriteStderr(" user site = %i\n", config->user_site_directory);
30963100
PySys_WriteStderr(" import site = %i\n", config->site_import);
3101+
DUMP_CONFIG("stdlib dir", stdlib_dir);
30973102
#undef DUMP_CONFIG
30983103

30993104
#define DUMP_SYS(NAME) \

‎Python/pathconfig.c‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ pathconfig_clear(_PyPathConfig *config)
5454
CLEAR(config->program_full_path);
5555
CLEAR(config->prefix);
5656
CLEAR(config->exec_prefix);
57+
CLEAR(config->stdlib_dir);
5758
CLEAR(config->module_search_path);
5859
CLEAR(config->program_name);
5960
CLEAR(config->home);
@@ -83,6 +84,7 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
8384
COPY_ATTR(prefix);
8485
COPY_ATTR(exec_prefix);
8586
COPY_ATTR(module_search_path);
87+
COPY_ATTR(stdlib_dir);
8688
COPY_ATTR(program_name);
8789
COPY_ATTR(home);
8890
#ifdef MS_WINDOWS
@@ -167,6 +169,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
167169
COPY_CONFIG(program_full_path, executable);
168170
COPY_CONFIG(prefix, prefix);
169171
COPY_CONFIG(exec_prefix, exec_prefix);
172+
COPY_CONFIG(stdlib_dir, stdlib_dir);
170173
COPY_CONFIG(program_name, program_name);
171174
COPY_CONFIG(home, home);
172175
#ifdef MS_WINDOWS
@@ -218,6 +221,7 @@ _PyPathConfig_AsDict(void)
218221
SET_ITEM_STR(prefix);
219222
SET_ITEM_STR(exec_prefix);
220223
SET_ITEM_STR(module_search_path);
224+
SET_ITEM_STR(stdlib_dir);
221225
SET_ITEM_STR(program_name);
222226
SET_ITEM_STR(home);
223227
#ifdef MS_WINDOWS
@@ -311,6 +315,7 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
311315
312316
- exec_prefix
313317
- module_search_path
318+
- stdlib_dir
314319
- prefix
315320
- program_full_path
316321
@@ -401,6 +406,7 @@ config_init_pathconfig(PyConfig *config, int compute_path_config)
401406
COPY_ATTR(program_full_path, executable);
402407
COPY_ATTR(prefix, prefix);
403408
COPY_ATTR(exec_prefix, exec_prefix);
409+
COPY_ATTR(stdlib_dir, stdlib_dir);
404410

405411
#undef COPY_ATTR
406412

@@ -486,16 +492,25 @@ Py_SetPath(const wchar_t *path)
486492

487493
PyMem_RawFree(_Py_path_config.prefix);
488494
PyMem_RawFree(_Py_path_config.exec_prefix);
495+
PyMem_RawFree(_Py_path_config.stdlib_dir);
489496
PyMem_RawFree(_Py_path_config.module_search_path);
490497

491498
_Py_path_config.prefix = _PyMem_RawWcsdup(L"");
492499
_Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
500+
// XXX Copy this from the new module_search_path?
501+
if (_Py_path_config.home != NULL) {
502+
_Py_path_config.stdlib_dir = _PyMem_RawWcsdup(_Py_path_config.home);
503+
}
504+
else {
505+
_Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L"");
506+
}
493507
_Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
494508

495509
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
496510

497511
if (_Py_path_config.prefix == NULL
498512
|| _Py_path_config.exec_prefix == NULL
513+
|| _Py_path_config.stdlib_dir == NULL
499514
|| _Py_path_config.module_search_path == NULL)
500515
{
501516
path_out_of_memory(__func__);
@@ -515,10 +530,13 @@ Py_SetPythonHome(const wchar_t *home)
515530

516531
PyMem_RawFree(_Py_path_config.home);
517532
_Py_path_config.home = _PyMem_RawWcsdup(home);
533+
if (_Py_path_config.home != NULL) {
534+
_Py_path_config.stdlib_dir = _PyMem_RawWcsdup(home);
535+
}
518536

519537
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
520538

521-
if (_Py_path_config.home == NULL) {
539+
if (_Py_path_config.home == NULL || _Py_path_config.stdlib_dir == NULL) {
522540
path_out_of_memory(__func__);
523541
}
524542
}
@@ -572,6 +590,17 @@ Py_GetPath(void)
572590
}
573591

574592

593+
wchar_t *
594+
_Py_GetStdlibDir(void)
595+
{
596+
wchar_t *stdlib_dir = _Py_path_config.stdlib_dir;
597+
if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') {
598+
return stdlib_dir;
599+
}
600+
return NULL;
601+
}
602+
603+
575604
wchar_t *
576605
Py_GetPrefix(void)
577606
{

‎Python/sysmodule.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2974,6 +2974,14 @@ _PySys_UpdateConfig(PyThreadState *tstate)
29742974

29752975
SET_SYS("_xoptions", sys_create_xoptions_dict(config));
29762976

2977+
const wchar_t *stdlibdir = _Py_GetStdlibDir();
2978+
if (stdlibdir != NULL) {
2979+
SET_SYS_FROM_WSTR("_stdlib_dir", stdlibdir);
2980+
}
2981+
else {
2982+
PyDict_SetItemString(sysdict, "_stdlib_dir", Py_None);
2983+
}
2984+
29772985
#undef SET_SYS_FROM_WSTR
29782986
#undef COPY_LIST
29792987
#undef COPY_WSTR

0 commit comments

Comments
 (0)