Bundle path handling in ts_utils.paths (#12805)

srittau · web-flow · commit 2d0f6d827752 · 2024-10-17T08:16:10.000+02:00
diff --git a/lib/ts_utils/metadata.py b/lib/ts_utils/metadata.py
@@ -5,7 +5,6 @@
 
 from __future__ import annotations
 
-import os
 import re
 import urllib.parse
 from collections.abc import Mapping
@@ -18,6 +17,7 @@
 from packaging.requirements import Requirement
 from packaging.specifiers import Specifier
 
+from .paths import PYPROJECT_PATH, STUBS_PATH, distribution_path
 from .utils import cache
 
 __all__ = [
@@ -43,20 +43,15 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]:
 
 @cache
 def _get_oldest_supported_python() -> str:
-    with open("pyproject.toml", "rb") as config:
+    with PYPROJECT_PATH.open("rb") as config:
         val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"]
     assert type(val) is str
     return val
 
 
-def stubs_path(distribution: str) -> Path:
-    """Return the path to the directory of a third-party distribution."""
-    return Path("stubs", distribution)
-
-
 def metadata_path(distribution: str) -> Path:
     """Return the path to the METADATA.toml file of a third-party distribution."""
-    return stubs_path(distribution) / "METADATA.toml"
+    return distribution_path(distribution) / "METADATA.toml"
 
 
 @final
@@ -317,7 +312,7 @@ class PackageDependencies(NamedTuple):
 
 @cache
 def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
-    return {read_metadata(typeshed_name).stub_distribution: typeshed_name for typeshed_name in os.listdir("stubs")}
+    return {read_metadata(dir.name).stub_distribution: dir.name for dir in STUBS_PATH.iterdir()}
 
 
 @cache
diff --git a/lib/ts_utils/paths.py b/lib/ts_utils/paths.py
@@ -0,0 +1,39 @@
+from pathlib import Path
+from typing import Final
+
+# TODO: Use base path relative to this file. Currently, ts_utils gets
+# installed into the user's virtual env, so we can't determine the path
+# to typeshed. Installing ts_utils editable would solve that, see
+# https://github.com/python/typeshed/pull/12806.
+TS_BASE_PATH: Final = Path("")
+STDLIB_PATH: Final = TS_BASE_PATH / "stdlib"
+STUBS_PATH: Final = TS_BASE_PATH / "stubs"
+
+PYPROJECT_PATH: Final = TS_BASE_PATH / "pyproject.toml"
+REQUIREMENTS_PATH: Final = TS_BASE_PATH / "requirements-tests.txt"
+
+TESTS_DIR: Final = "@tests"
+TEST_CASES_DIR: Final = "test_cases"
+
+
+def distribution_path(distribution_name: str) -> Path:
+    """Return the path to the directory of a third-party distribution."""
+    return STUBS_PATH / distribution_name
+
+
+def tests_path(distribution_name: str) -> Path:
+    if distribution_name == "stdlib":
+        return STDLIB_PATH / TESTS_DIR
+    else:
+        return STUBS_PATH / distribution_name / TESTS_DIR
+
+
+def test_cases_path(distribution_name: str) -> Path:
+    return tests_path(distribution_name) / TEST_CASES_DIR
+
+
+def allowlists_path(distribution_name: str) -> Path:
+    if distribution_name == "stdlib":
+        return tests_path("stdlib") / "stubtest_allowlists"
+    else:
+        return tests_path(distribution_name)
diff --git a/lib/ts_utils/utils.py b/lib/ts_utils/utils.py
@@ -21,10 +21,9 @@ def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type:
         return text
 
 
-PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
+from .paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, allowlists_path, test_cases_path
 
-STDLIB_PATH = Path("stdlib")
-STUBS_PATH = Path("stubs")
+PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}"
 
 
 # A backport of functools.cache for Python <3.9
@@ -90,16 +89,14 @@ def venv_python(venv_dir: Path) -> Path:
 # ====================================================================
 
 
-REQS_FILE: Final = "requirements-tests.txt"
-
-
 @cache
 def parse_requirements() -> Mapping[str, Requirement]:
     """Return a dictionary of requirements from the requirements file."""
 
-    with open(REQS_FILE, encoding="UTF-8") as requirements_file:
+    with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file:
         stripped_lines = map(strip_comments, requirements_file)
-        requirements = map(Requirement, filter(None, stripped_lines))
+        stripped_more = [li for li in stripped_lines if not li.startswith("-")]
+        requirements = map(Requirement, filter(None, stripped_more))
         return {requirement.name: requirement for requirement in requirements}
 
 
@@ -155,10 +152,6 @@ def _parse_version(v_str: str) -> tuple[int, int]:
 # ====================================================================
 
 
-TESTS_DIR: Final = "@tests"
-TEST_CASES_DIR: Final = "test_cases"
-
-
 class DistributionTests(NamedTuple):
     name: str
     test_cases_path: Path
@@ -179,17 +172,6 @@ def distribution_info(distribution_name: str) -> DistributionTests:
     raise RuntimeError(f"No test cases found for {distribution_name!r}!")
 
 
-def tests_path(distribution_name: str) -> Path:
-    if distribution_name == "stdlib":
-        return STDLIB_PATH / TESTS_DIR
-    else:
-        return STUBS_PATH / distribution_name / TESTS_DIR
-
-
-def test_cases_path(distribution_name: str) -> Path:
-    return tests_path(distribution_name) / TEST_CASES_DIR
-
-
 def get_all_testcase_directories() -> list[DistributionTests]:
     testcase_directories: list[DistributionTests] = []
     for distribution_path in STUBS_PATH.iterdir():
@@ -201,13 +183,6 @@ def get_all_testcase_directories() -> list[DistributionTests]:
     return [distribution_info("stdlib"), *sorted(testcase_directories)]
 
 
-def allowlists_path(distribution_name: str) -> Path:
-    if distribution_name == "stdlib":
-        return tests_path("stdlib") / "stubtest_allowlists"
-    else:
-        return tests_path(distribution_name)
-
-
 def allowlists(distribution_name: str) -> list[str]:
     prefix = "" if distribution_name == "stdlib" else "stubtest_allowlist_"
     version_id = f"py{sys.version_info.major}{sys.version_info.minor}"
diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py
@@ -29,7 +29,8 @@
 from packaging.specifiers import Specifier
 from termcolor import colored
 
-from ts_utils.metadata import StubMetadata, metadata_path, read_metadata, stubs_path
+from ts_utils.metadata import StubMetadata, metadata_path, read_metadata
+from ts_utils.paths import STUBS_PATH, distribution_path
 
 TYPESHED_OWNER = "python"
 TYPESHED_API_URL = f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed"
@@ -463,7 +464,7 @@ async def analyze_diff(
         assert isinstance(json_resp, dict)
     # https://docs.github.com/en/rest/commits/commits#compare-two-commits
     py_files: list[FileInfo] = [file for file in json_resp["files"] if Path(file["filename"]).suffix == ".py"]
-    stub_path = stubs_path(distribution)
+    stub_path = distribution_path(distribution)
     files_in_typeshed = set(stub_path.rglob("*.pyi"))
     py_files_stubbed_in_typeshed = [file for file in py_files if (stub_path / f"{file['filename']}i") in files_in_typeshed]
     return DiffAnalysis(py_files=py_files, py_files_stubbed_in_typeshed=py_files_stubbed_in_typeshed)
@@ -759,7 +760,7 @@ async def main() -> None:
     if args.distributions:
         dists_to_update = args.distributions
     else:
-        dists_to_update = [path.name for path in Path("stubs").iterdir()]
+        dists_to_update = [path.name for path in STUBS_PATH.iterdir()]
 
     if args.action_level > ActionLevel.nothing:
         subprocess.run(["git", "update-index", "--refresh"], capture_output=True)
diff --git a/tests/check_typeshed_structure.py b/tests/check_typeshed_structure.py
@@ -13,17 +13,13 @@
 from pathlib import Path
 
 from ts_utils.metadata import read_metadata
+from ts_utils.paths import REQUIREMENTS_PATH, STDLIB_PATH, STUBS_PATH, TEST_CASES_DIR, TESTS_DIR, tests_path
 from ts_utils.utils import (
-    REQS_FILE,
-    STDLIB_PATH,
-    TEST_CASES_DIR,
-    TESTS_DIR,
     get_all_testcase_directories,
     get_gitignore_spec,
     parse_requirements,
     parse_stdlib_versions_file,
     spec_matches_path,
-    tests_path,
 )
 
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
@@ -66,7 +62,7 @@ def check_stdlib() -> None:
 def check_stubs() -> None:
     """Check that the stubs directory contains only the correct files."""
     gitignore_spec = get_gitignore_spec()
-    for dist in Path("stubs").iterdir():
+    for dist in STUBS_PATH.iterdir():
         if spec_matches_path(gitignore_spec, dist):
             continue
         assert dist.is_dir(), f"Only directories allowed in stubs, got {dist}"
@@ -97,8 +93,8 @@ def check_distutils() -> None:
     def all_relative_paths_in_directory(path: Path) -> set[Path]:
         return {pyi.relative_to(path) for pyi in path.rglob("*.pyi")}
 
-    all_setuptools_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "setuptools", "_distutils"))
-    all_distutils_files = all_relative_paths_in_directory(Path("stubs", "setuptools", "distutils"))
+    all_setuptools_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "setuptools" / "_distutils")
+    all_distutils_files = all_relative_paths_in_directory(STUBS_PATH / "setuptools" / "distutils")
     assert all_setuptools_files and all_distutils_files, "Looks like this test might be out of date!"
     extra_files = all_setuptools_files - all_distutils_files
     joined = "\n".join(f"  * {f}" for f in extra_files)
@@ -164,10 +160,10 @@ def check_requirement_pins() -> None:
     """Check that type checkers and linters are pinned to an exact version."""
     requirements = parse_requirements()
     for package in linters:
-        assert package in requirements, f"type checker/linter '{package}' not found in {REQS_FILE}"
+        assert package in requirements, f"type checker/linter '{package}' not found in {REQUIREMENTS_PATH.name}"
         spec = requirements[package].specifier
-        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQS_FILE}"
-        msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQS_FILE}"
+        assert len(spec) == 1, f"type checker/linter '{package}' has complex specifier in {REQUIREMENTS_PATH.name}"
+        msg = f"type checker/linter '{package}' is not pinned to an exact version in {REQUIREMENTS_PATH.name}"
         assert str(spec).startswith("=="), msg
 
 
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
@@ -22,11 +22,10 @@
 import tomli
 from packaging.requirements import Requirement
 
-from ts_utils.metadata import PackageDependencies, get_recursive_requirements, read_metadata
+from ts_utils.metadata import PackageDependencies, get_recursive_requirements, metadata_path, read_metadata
+from ts_utils.paths import STDLIB_PATH, STUBS_PATH, TESTS_DIR, TS_BASE_PATH, distribution_path
 from ts_utils.utils import (
     PYTHON_VERSION,
-    STDLIB_PATH,
-    TESTS_DIR,
     colored,
     get_gitignore_spec,
     get_mypy_req,
@@ -47,7 +46,7 @@
 
 SUPPORTED_VERSIONS = ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
 SUPPORTED_PLATFORMS = ("linux", "win32", "darwin")
-DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
+DIRECTORIES_TO_TEST = [STDLIB_PATH, STUBS_PATH]
 
 VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
 Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
@@ -170,7 +169,7 @@ class MypyDistConf(NamedTuple):
 
 
 def add_configuration(configurations: list[MypyDistConf], distribution: str) -> None:
-    with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
+    with metadata_path(distribution).open("rb") as f:
         data = tomli.load(f)
 
     # TODO: This could be added to ts_utils.metadata, but is currently unused
@@ -229,7 +228,7 @@ def run_mypy(
             "--platform",
             args.platform,
             "--custom-typeshed-dir",
-            str(Path(__file__).parent.parent),
+            str(TS_BASE_PATH),
             "--strict",
             # Stub completion is checked by pyright (--allow-*-defs)
             "--allow-untyped-defs",
@@ -283,7 +282,7 @@ def add_third_party_files(
         return
     seen_dists.add(distribution)
     seen_dists.update(r.name for r in typeshed_reqs)
-    root = Path("stubs", distribution)
+    root = distribution_path(distribution)
     for name in os.listdir(root):
         if name.startswith("."):
             continue
@@ -319,7 +318,7 @@ def test_third_party_distribution(
         print_error("no files found")
         sys.exit(1)
 
-    mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists)
+    mypypath = os.pathsep.join(str(distribution_path(dist)) for dist in seen_dists)
     if args.verbose:
         print(colored(f"\nMYPYPATH={mypypath}", "blue"))
     result = run_mypy(
@@ -516,16 +515,12 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary:
     distributions_to_check: dict[str, PackageDependencies] = {}
 
     for distribution in sorted(os.listdir("stubs")):
-        distribution_path = Path("stubs", distribution)
+        dist_path = distribution_path(distribution)
 
-        if spec_matches_path(gitignore_spec, distribution_path):
+        if spec_matches_path(gitignore_spec, dist_path):
             continue
 
-        if (
-            distribution_path in args.filter
-            or Path("stubs") in args.filter
-            or any(distribution_path in path.parents for path in args.filter)
-        ):
+        if dist_path in args.filter or STUBS_PATH in args.filter or any(dist_path in path.parents for path in args.filter):
             metadata = read_metadata(distribution)
             if not metadata.requires_python.contains(PYTHON_VERSION):
                 msg = (
diff --git a/tests/regr_test.py b/tests/regr_test.py
@@ -22,9 +22,9 @@
 from typing_extensions import TypeAlias
 
 from ts_utils.metadata import get_recursive_requirements, read_metadata
+from ts_utils.paths import STDLIB_PATH, TEST_CASES_DIR, TS_BASE_PATH, distribution_path
 from ts_utils.utils import (
     PYTHON_VERSION,
-    TEST_CASES_DIR,
     DistributionTests,
     colored,
     distribution_info,
@@ -134,14 +134,14 @@ def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Ver
     # that has only the required stubs copied over.
     new_typeshed = tempdir / TYPESHED
     new_typeshed.mkdir()
-    shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
+    shutil.copytree(STDLIB_PATH, new_typeshed / "stdlib")
     requirements = get_recursive_requirements(package.name)
     # mypy refuses to consider a directory a "valid typeshed directory"
     # unless there's a stubs/mypy-extensions path inside it,
     # so add that to the list of stubs to copy over to the new directory
     typeshed_requirements = [r.name for r in requirements.typeshed_pkgs]
     for requirement in {package.name, *typeshed_requirements, "mypy-extensions"}:
-        shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
+        shutil.copytree(distribution_path(requirement), new_typeshed / "stubs" / requirement)
 
     if requirements.external_pkgs:
         venv_location = str(tempdir / VENV_DIR)
@@ -190,7 +190,7 @@ def run_testcases(
 
     if package.is_stdlib:
         python_exe = sys.executable
-        custom_typeshed = Path(__file__).parent.parent
+        custom_typeshed = TS_BASE_PATH
         flags.append("--no-site-packages")
     else:
         custom_typeshed = tempdir / TYPESHED
diff --git a/tests/runtests.py b/tests/runtests.py
@@ -10,7 +10,8 @@
 from importlib.util import find_spec
 from pathlib import Path
 
-from ts_utils.utils import TEST_CASES_DIR, colored, test_cases_path
+from ts_utils.paths import TEST_CASES_DIR, test_cases_path
+from ts_utils.utils import colored
 
 _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
 _TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json"
diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py
@@ -13,7 +13,8 @@
 import sys
 from pathlib import Path
 
-from ts_utils.utils import allowlist_stubtest_arguments, allowlists_path
+from ts_utils.paths import TS_BASE_PATH, allowlists_path
+from ts_utils.utils import allowlist_stubtest_arguments
 
 
 def run_stubtest(typeshed_dir: Path) -> int:
@@ -57,4 +58,4 @@ def run_stubtest(typeshed_dir: Path) -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(run_stubtest(typeshed_dir=Path(".")))
+    sys.exit(run_stubtest(typeshed_dir=TS_BASE_PATH))
diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py
