diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 30138d3be6..b778ac49a4 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -182,6 +182,7 @@ tasks: platform: rbe_ubuntu1604 test_flags: - "--test_tag_filters=-integration-test,-acceptance-test" + - "--extra_toolchains=@buildkite_config//config:cc-toolchain" integration_test_build_file_generation_ubuntu_minimum_supported_workspace: <<: *minimum_supported_version @@ -515,18 +516,6 @@ tasks: - "bazel run //:os_specific_requirements.update" - "git diff --exit-code" - integration_test_pip_repository_entry_points_macos_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "pip_repository_entry_points: macOS, workspace" - working_directory: tests/integration/pip_repository_entry_points - platform: macos - integration_test_pip_repository_entry_points_windows_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "pip_repository_entry_points: Windows, workspace" - working_directory: tests/integration/pip_repository_entry_points - platform: windows integration_test_ignore_root_user_error_macos_workspace: <<: *reusable_build_test_all diff --git a/.bazelignore b/.bazelignore index 713903d832..95391738c1 100644 --- a/.bazelignore +++ b/.bazelignore @@ -26,4 +26,3 @@ examples/py_proto_library/bazel-py_proto_library tests/integration/compile_pip_requirements/bazel-compile_pip_requirements tests/integration/ignore_root_user_error/bazel-ignore_root_user_error tests/integration/local_toolchains/bazel-local_toolchains -tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points diff --git a/.bazelrc b/.bazelrc index 94cfb93350..07188a9196 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors @@ -17,6 +17,9 @@ test --test_output=errors # Python targets as required. build --incompatible_default_to_explicit_init_py +# Ensure ongoing compatibility with this flag. +common --incompatible_disallow_struct_provider_syntax + # Windows makes use of runfiles for some rules build --enable_runfiles diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3449bcfac0..a7ca2b06c2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,15 +1,11 @@ # NB: Last matching rule takes precedence in CODEOWNERS. -* @rickeylev +* @rickeylev @aignas # Directory containing the Gazelle extension and Go code. /gazelle/ @f0rmiga /examples/build_file_generation/ @f0rmiga -# Toolchains -/python/repositories.bzl @f0rmiga -/python/private/toolchains_repo.bzl @f0rmiga -/python/tests/toolchains/ @f0rmiga - -# pip_parse related code -/python/pip_install/ @hrfuller +# PyPI integration related code +/python/private/pypi/ @groodt +/tests/pypi/ @groodt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..9632f4e3d3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +--- +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directories: + # Maintain dependencies for our tools + - "/docs/sphinx" + - "/tools/publish" + schedule: + interval: "weekly" + open-pull-requests-limit: 3 diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index b0d0cdff1b..429775172e 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 steps: # Checkout the code - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: jpetrucciani/mypy-check@master with: requirements: 1.6.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb23bc8411..96624b347a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Create release archive and notes run: .github/workflows/create_archive_and_notes.sh - name: Publish wheel dist @@ -37,7 +37,7 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: bazel run --stamp --embed_label=${{ github.ref_name }} //python/runfiles:wheel.publish - name: Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: # Use GH feature to populate the changelog automatically generate_release_notes: true diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8d388e2f7a..d37121b0da 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v3 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/BUILD.bazel b/BUILD.bazel index c97f41dee2..038b56a0c6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -40,7 +40,6 @@ filegroup( "internal_setup.bzl", "version.bzl", "//python:distribution", - "//python/pip_install:distribution", "//tools:distribution", "@rules_python_gazelle_plugin//:distribution", ], diff --git a/CHANGELOG.md b/CHANGELOG.md index 9abbd44de0..61df0862dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,13 +25,67 @@ A brief description of the categories of changes: [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x ### Changed -* Nothing yet +* `protobuf`/`com_google_protobuf` dependency bumped to `v24.4` +* (bzlmod): optimize the creation of config settings used in pip to + reduce the total number of targets in the hub repo. +* (toolchains) The exec tools toolchain now finds its interpreter by reusing + the regular interpreter toolchain. This avoids having to duplicate specifying + where the runtime for the exec tools toolchain is. +* (toolchains) ({obj}`//python:autodetecting_toolchain`) is deprecated. It is + replaced by {obj}`//python/runtime_env_toolchains:all`. The old target will be + removed in a future release. ### Fixed -* Nothing yet +* (bzlmod): When using `experimental_index_url` the `all_requirements`, + `all_whl_requirements` and `all_data_requirements` will now only include + common packages that are available on all target platforms. This is to ensure + that packages that are only present for some platforms are pulled only via + the `deps` of the materialized `py_library`. If you would like to include + platform specific packages, using a `select` statement with references to the + specific package will still work (e.g. + ``` + my_attr = all_requirements + select( + { + "@platforms//os:linux": ["@pypi//foo_available_only_on_linux"], + "//conditions:default": [], + } + )`. +* (bzlmod): Targets in `all_requirements` now use the same form as targets returned by the `requirement` macro. +* (rules) Auto exec groups are enabled. This allows actions run by the rules, + such as precompiling, to pick an execution platform separately from what + other toolchains support. +* (providers) {obj}`PyRuntimeInfo` doesn't require passing the + `interpreter_version_info` arg. +* (bzlmod) Correctly pass `isolated`, `quiet` and `timeout` values to `whl_library` + and drop the defaults from the lock file. +* (whl_library) Correctly handle arch-specific dependencies when we encounter a + platform specific wheel and use `experimental_target_platforms`. + Fixes [#1996](https://github.com/bazelbuild/rules_python/issues/1996). +* (rules) The first element of the default outputs is now the executable again. +* (pip) Fixed crash when pypi packages lacked a sha (e.g. yanked packages) + +### Added +* (toolchains) {obj}`//python/runtime_env_toolchains:all`, which is a drop-in + replacement for the "autodetecting" toolchain. + +### Added +* (gazelle) Added new `python_label_convention` and `python_label_normalization` directives. These directive + allows altering default Gazelle label format to third-party dependencies useful for re-using Gazelle plugin + with other rules, including `rules_pycross`. See [#1939](https://github.com/bazelbuild/rules_python/issues/1939). ### Removed -* Nothing yet +* (pip): Removes the `entrypoint` macro that was replaced by `py_console_script_binary` in 0.26.0. + +## [0.33.2] - 2024-06-13 + +[0.33.2]: https://github.com/bazelbuild/rules_python/releases/tag/0.33.2 + +### Fixed +* (toolchains) The {obj}`exec_tools_toolchain_type` is disabled by default. + To enable it, set {obj}`--//python/config_settings:exec_tools_toolchain=enabled`. + This toolchain must be enabled for precompilation to work. This toolchain will + be enabled by default in a future release. + Fixes [#1967](https://github.com/bazelbuild/rules_python/issues/1967). ## [0.33.1] - 2024-06-13 @@ -137,7 +191,7 @@ A brief description of the categories of changes: placeholder, just like the `python_default_visibility` directive does. * (rules) A new bootstrap implementation that doesn't require a system Python is available. It can be enabled by setting - {obj}`--@rules_python//python:config_settings:bootstrap_impl=two_phase`. It + {obj}`--@rules_python//python/config_settings:bootstrap_impl=script`. It will become the default in a subsequent release. ([#691](https://github.com/bazelbuild/rules_python/issues/691)) * (providers) `PyRuntimeInfo` has two new attributes: diff --git a/DEVELOPING.md b/DEVELOPING.md index 3c9e89d1d6..a70d3b171f 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -2,7 +2,7 @@ ## Updating internal dependencies -1. Modify the `./python/pip_install/tools/requirements.txt` file and run: +1. Modify the `./python/private/pypi/requirements.txt` file and run: ``` bazel run //tools/private/update_deps:update_pip_deps ``` diff --git a/MODULE.bazel b/MODULE.bazel index 53d845ed88..b6d198ffc1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,13 +11,11 @@ bazel_dep(name = "platforms", version = "0.0.4") # Those are loaded only when using py_proto_library bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf") -internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps") +internal_deps = use_extension("//python/private:internal_deps.bzl", "internal_deps") use_repo( internal_deps, - "rules_python_internal", - # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' "pypi__build", "pypi__click", "pypi__colorama", @@ -33,7 +31,7 @@ use_repo( "pypi__tomli", "pypi__wheel", "pypi__zipp", - # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' + "rules_python_internal", ) # We need to do another use_extension call to expose the "pythons_hub" @@ -56,7 +54,7 @@ register_toolchains("@pythons_hub//:all") ##################### # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. -pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal") +pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal") pip.parse( hub_name = "rules_python_publish_deps", python_version = "3.11", @@ -79,7 +77,7 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle") dev_pip = use_extension( - "//python/private/bzlmod:pip.bzl", + "//python/private/pypi:pip.bzl", "pip_internal", dev_dependency = True, ) diff --git a/README.md b/README.md index 546af97009..d890d702d6 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,9 @@ Documentation for rules_python is at and i Examples live in the [examples](examples) directory. -Currently, the core rules build into the Bazel binary, and the symbols in this -repository are simple aliases. However, we are migrating the rules to Starlark and removing them from the Bazel binary. Therefore, the future-proof way to depend on Python rules is via this repository. See[`Migrating from the Bundled Rules`](#Migrating-from-the-bundled-rules) below. - -The core rules are stable. Their implementation in Bazel is subject to Bazel's +The core rules are stable. Their implementation is subject to Bazel's [backward compatibility policy](https://docs.bazel.build/versions/master/backward-compatibility.html). -Once migrated to rules_python, they may evolve at a different -rate, but this repository will still follow [semantic versioning](https://semver.org). +This repository aims to follow [semantic versioning](https://semver.org). The Bazel community maintains this repository. Neither Google nor the Bazel team provides support for the code. However, this repository is part of the test suite used to vet new Bazel releases. See [How to contribute](CONTRIBUTING.md) page for information on our development workflow. diff --git a/docs/sphinx/api/python/config_settings/index.md b/docs/sphinx/api/python/config_settings/index.md index 8c23bf6855..50647abb8d 100644 --- a/docs/sphinx/api/python/config_settings/index.md +++ b/docs/sphinx/api/python/config_settings/index.md @@ -10,6 +10,22 @@ Determines the default hermetic Python toolchain version. This can be set to one of the values that `rules_python` maintains. ::: +::::{bzl:flag} exec_tools_toolchain +Determines if the {obj}`exec_tools_toolchain_type` toolchain is enabled. + +:::{note} +* Note that this only affects the rules_python generated toolchains. +::: + +Values: + +* `enabled`: Allow matching of the registered toolchains at build time. +* `disabled`: Prevent the toolchain from being matched at build time. + +:::{versionadded} 0.33.2 +::: +:::: + ::::{bzl:flag} precompile Determines if Python source files should be compiled at build time. @@ -34,6 +50,8 @@ Values: * `force_disabled`: Like `disabled`, except overrides target-level setting. This is useful useful for development, testing enabling precompilation more broadly, or as an escape hatch if build-time compiling is not available. +:::{versionadded} 0.33.0 +::: :::: ::::{bzl:flag} precompile_source_retention @@ -51,9 +69,11 @@ Values: * `omit_source`: Don't include the orignal py source. * `omit_if_generated_source`: Keep the original source if it's a regular source file, but omit it if it's a generated file. +:::{versionadded} 0.33.0 +::: :::: -:::{bzl:flag} precompile_add_to_runfiles +::::{bzl:flag} precompile_add_to_runfiles Determines if a target adds its compiled files to its runfiles. When a target compiles its files, but doesn't add them to its own runfiles, it @@ -66,7 +86,9 @@ Values: runfiles; they are still added to {bzl:obj}`PyInfo.transitive_pyc_files`. See also: {bzl:obj}`py_binary.pyc_collection` attribute. This is useful for allowing incrementally enabling precompilation on a per-binary basis. +:::{versionadded} 0.33.0 ::: +:::: ::::{bzl:flag} pyc_collection Determine if `py_binary` collects transitive pyc files. @@ -78,6 +100,8 @@ This flag is overridden by the target level `pyc_collection` attribute. Values: * `include_pyc`: Include `PyInfo.transitive_pyc_files` as part of the binary. * `disabled`: Don't include `PyInfo.transitive_pyc_files` as part of the binary. +:::{versionadded} 0.33.0 +::: :::: ::::{bzl:flag} py_linux_libc diff --git a/docs/sphinx/api/python/index.md b/docs/sphinx/api/python/index.md index 494e7b4a02..6c794475ac 100644 --- a/docs/sphinx/api/python/index.md +++ b/docs/sphinx/api/python/index.md @@ -26,18 +26,11 @@ provides: ::::{target} autodetecting_toolchain -A simple toolchain that simply uses `python3` from the runtime environment. +Legacy toolchain; despite its name, it doesn't autodetect anything. -Note that this toolchain provides no build-time information, which makes it of -limited utility. +:::{deprecated} 0.34.0 -This is only provided to aid migration off the builtin Bazel toolchain -(`@bazel_tools//python:autodetecting_toolchain`), and is largely only applicable -to WORKSPACE builds. - -:::{deprecated} unspecified - -Switch to using a hermetic toolchain or manual toolchain configuration instead. +Use {obj}`@rules_python//python/runtime_env_toolchain:all` instead. ::: - :::: + diff --git a/docs/sphinx/api/python/runtime_env_toolchains/index.md b/docs/sphinx/api/python/runtime_env_toolchains/index.md new file mode 100644 index 0000000000..ef31f086d7 --- /dev/null +++ b/docs/sphinx/api/python/runtime_env_toolchains/index.md @@ -0,0 +1,38 @@ +:::{default-domain} bzl +::: +:::{bzl:currentfile} //python/runtime_env_toolchain:BUILD.bazel +::: + +# //python/runtime_env_toolchain + +::::{target} all + +A set of toolchains that invoke `python3` from the runtime environment. + +Note that this toolchain provides no build-time information, which makes it of +limited utility. This is because the invocation of `python3` is done when a +program is run, not at build time. + +This is only provided to aid migration off the builtin Bazel toolchain +(`@bazel_tools//python:autodetecting_toolchain`), and is largely only applicable +to WORKSPACE builds. + +To use this target, register it as a toolchain in WORKSPACE or MODULE.bazel: + +::: +register_toolchains("@rules_python//python/runtime_env_toolchains:all") +::: + +The benefit of this target over the legacy targets is this defines additional +toolchain types that rules_python needs. This prevents toolchain resolution from +continuing to search elsewhere (e.g. potentially incurring a download of the +hermetic runtimes when they won't be used). + +:::{deprecated} 0.34.0 + +Switch to using a hermetic toolchain or manual toolchain configuration instead. +::: + +:::{versionadded} 0.34.0 +::: +:::: diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt index d7c90573af..f2f0158f88 100644 --- a/docs/sphinx/requirements.txt +++ b/docs/sphinx/requirements.txt @@ -4,21 +4,21 @@ # # bazel run //docs/sphinx:requirements.update # -absl-py==2.0.0 \ - --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3 \ - --hash=sha256:d9690211c5fcfefcdd1a45470ac2b5c5acd45241c3af71eed96bc5441746c0d5 +absl-py==2.1.0 \ + --hash=sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308 \ + --hash=sha256:7820790efbb316739cde8b4e19357243fc3608a152024288513dd968d7d959ff # via rules_python_docs (docs/sphinx/pyproject.toml) -alabaster==0.7.13 \ - --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \ - --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2 +alabaster==0.7.16 \ + --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ + --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 # via sphinx -babel==2.13.1 \ - --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ - --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed +babel==2.15.0 \ + --hash=sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb \ + --hash=sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413 # via sphinx -certifi==2023.11.17 \ - --hash=sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1 \ - --hash=sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474 +certifi==2024.6.2 \ + --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ + --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 # via requests charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -140,89 +140,89 @@ markdown-it-py==3.0.0 \ # via # mdit-py-plugins # myst-parser -markupsafe==2.1.3 \ - --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ - --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ - --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ - --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ - --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ - --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ - --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ - --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ - --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ - --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ - --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ - --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ - --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ - --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ - --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ - --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ - --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ - --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ - --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ - --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ - --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ - --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ - --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ - --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ - --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ - --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ - --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ - --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ - --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ - --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ - --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ - --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ - --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ - --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ - --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ - --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ - --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ - --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ - --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ - --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ - --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ - --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ - --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ - --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ - --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ - --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ - --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ - --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ - --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ - --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ - --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ - --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ - --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ - --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ - --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ - --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ - --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ - --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ - --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ - --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 +markupsafe==2.1.5 \ + --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ + --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ + --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ + --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ + --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ + --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ + --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ + --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ + --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ + --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ + --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ + --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ + --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ + --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ + --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ + --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ + --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ + --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ + --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ + --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ + --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ + --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ + --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ + --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ + --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ + --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ + --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ + --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ + --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ + --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ + --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ + --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ + --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ + --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ + --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ + --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ + --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ + --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ + --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ + --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ + --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ + --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ + --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ + --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ + --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ + --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ + --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ + --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ + --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ + --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ + --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ + --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ + --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ + --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ + --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ + --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ + --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ + --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ + --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ + --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 # via jinja2 -mdit-py-plugins==0.4.0 \ - --hash=sha256:b51b3bb70691f57f974e257e367107857a93b36f322a9e6d44ca5bf28ec2def9 \ - --hash=sha256:d8ab27e9aed6c38aa716819fedfde15ca275715955f8a185a8e1cf90fb1d2c1b +mdit-py-plugins==0.4.1 \ + --hash=sha256:1020dfe4e6bfc2c79fb49ae4e3f5b297f5ccd20f010187acc52af2921e27dc6a \ + --hash=sha256:834b8ac23d1cd60cec703646ffd22ae97b7955a6d596eb1d304be1e251ae499c # via myst-parser mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -myst-parser==2.0.0 \ - --hash=sha256:7c36344ae39c8e740dad7fdabf5aa6fc4897a813083c6cc9990044eb93656b14 \ - --hash=sha256:ea929a67a6a0b1683cdbe19b8d2e724cd7643f8aa3e7bb18dd65beac3483bead +myst-parser==3.0.1 \ + --hash=sha256:6457aaa33a5d474aca678b8ead9b3dc298e89c68e67012e73146ea6fd54babf1 \ + --hash=sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87 # via rules_python_docs (docs/sphinx/pyproject.toml) -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via # readthedocs-sphinx-ext # sphinx -pygments==2.17.2 \ - --hash=sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c \ - --hash=sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367 +pygments==2.18.0 \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via sphinx pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -254,6 +254,7 @@ pyyaml==6.0.1 \ --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ @@ -276,13 +277,13 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via myst-parser -readthedocs-sphinx-ext==2.2.3 \ - --hash=sha256:6583c26791a5853ee9e57ce9db864e2fb06808ba470f805d74d53fc50811e012 \ - --hash=sha256:e9d911792789b88ae12e2be94d88c619f89a4fa1fe9e42c1505c9930a07163d8 +readthedocs-sphinx-ext==2.2.5 \ + --hash=sha256:ee5fd5b99db9f0c180b2396cbce528aa36671951b9526bb0272dbfce5517bd27 \ + --hash=sha256:f8c56184ea011c972dd45a90122568587cc85b0127bc9cf064d17c68bc809daa # via rules_python_docs (docs/sphinx/pyproject.toml) -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via # readthedocs-sphinx-ext # sphinx @@ -290,34 +291,29 @@ snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a # via sphinx -sphinx==7.2.6 \ - --hash=sha256:1e09160a40b956dc623c910118fa636da93bd3ca0b9876a7b3df90f07d691560 \ - --hash=sha256:9a5160e1ea90688d5963ba09a2dcd8bdd526620edbb65c328728f1b2228d5ab5 +sphinx==7.3.7 \ + --hash=sha256:413f75440be4cacf328f580b4274ada4565fb2187d696a84970c23f77b64d8c3 \ + --hash=sha256:a4a7db75ed37531c05002d56ed6948d4c42f473a36f46e1382b0bd76ca9627bc # via # myst-parser # rules_python_docs (docs/sphinx/pyproject.toml) # sphinx-rtd-theme - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp # sphinxcontrib-jquery - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml sphinx-rtd-theme==2.0.0 \ --hash=sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b \ --hash=sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586 # via rules_python_docs (docs/sphinx/pyproject.toml) -sphinxcontrib-applehelp==1.0.7 \ - --hash=sha256:094c4d56209d1734e7d252f6e0b3ccc090bd52ee56807a5d9315b19c122ab15d \ - --hash=sha256:39fdc8d762d33b01a7d8f026a3b7d71563ea3b72787d5f00ad8465bd9d6dfbfa +sphinxcontrib-applehelp==1.0.8 \ + --hash=sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619 \ + --hash=sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4 # via sphinx -sphinxcontrib-devhelp==1.0.5 \ - --hash=sha256:63b41e0d38207ca40ebbeabcf4d8e51f76c03e78cd61abe118cf4435c73d4212 \ - --hash=sha256:fe8009aed765188f08fcaadbb3ea0d90ce8ae2d76710b7e29ea7d047177dae2f +sphinxcontrib-devhelp==1.0.6 \ + --hash=sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f \ + --hash=sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 \ - --hash=sha256:6c26a118a05b76000738429b724a0568dbde5b72391a688577da08f11891092a \ - --hash=sha256:8001661c077a73c29beaf4a79968d0726103c5605e27db92b9ebed8bab1359e9 +sphinxcontrib-htmlhelp==2.0.5 \ + --hash=sha256:0dc87637d5de53dd5eec3a6a01753b1ccf99494bd756aafecd74b4fa9e729015 \ + --hash=sha256:393f04f112b4d2f53d93448d4bce35842f62b307ccdc549ec1585e950bc35e04 # via sphinx sphinxcontrib-jquery==4.1 \ --hash=sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a \ @@ -327,19 +323,19 @@ sphinxcontrib-jsmath==1.0.1 \ --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 # via sphinx -sphinxcontrib-qthelp==1.0.6 \ - --hash=sha256:62b9d1a186ab7f5ee3356d906f648cacb7a6bdb94d201ee7adf26db55092982d \ - --hash=sha256:bf76886ee7470b934e363da7a954ea2825650013d367728588732c7350f49ea4 +sphinxcontrib-qthelp==1.0.7 \ + --hash=sha256:053dedc38823a80a7209a80860b16b722e9e0209e32fea98c90e4e6624588ed6 \ + --hash=sha256:e2ae3b5c492d58fcbd73281fbd27e34b8393ec34a073c792642cd8e529288182 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 \ - --hash=sha256:0c64ff898339e1fac29abd2bf5f11078f3ec413cfe9c046d3120d7ca65530b54 \ - --hash=sha256:9b36e503703ff04f20e9675771df105e58aa029cfcbc23b8ed716019b7416ae1 +sphinxcontrib-serializinghtml==1.1.10 \ + --hash=sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7 \ + --hash=sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f # via sphinx -typing-extensions==4.9.0 \ - --hash=sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783 \ - --hash=sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via rules_python_docs (docs/sphinx/pyproject.toml) -urllib3==2.1.0 \ - --hash=sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3 \ - --hash=sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54 +urllib3==2.2.2 \ + --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ + --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via requests diff --git a/docs/sphinx/toolchains.md b/docs/sphinx/toolchains.md index e3be22f97b..fac1bfc6b0 100644 --- a/docs/sphinx/toolchains.md +++ b/docs/sphinx/toolchains.md @@ -233,12 +233,12 @@ use `python3` from the environment a binary runs within. This provides extremely limited functionality to the rules (at build time, nothing is knowable about the Python runtime). -Bazel itself automatically registers `@bazel_tools//python:autodetecting_toolchain` +Bazel itself automatically registers `@bazel_tools//tools/python:autodetecting_toolchain` as the lowest priority toolchain. For WORKSPACE builds, if no other toolchain is registered, that toolchain will be used. For bzlmod builds, rules_python automatically registers a higher-priority toolchain; it won't be used unless there is a toolchain misconfiguration somewhere. To aid migration off the Bazel-builtin toolchain, rules_python provides -{obj}`@rules_python//python:autodetecting_toolchain`. This is an equivalent +{obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 0d30161147..e46989e9fd 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -15,7 +15,7 @@ local_path_override( bazel_dep(name = "rules_proto", version = "5.3.0-21.7") # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc) -bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf") # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. diff --git a/examples/multi_python_versions/WORKSPACE b/examples/multi_python_versions/WORKSPACE index f3a69ce769..4f731d95a8 100644 --- a/examples/multi_python_versions/WORKSPACE +++ b/examples/multi_python_versions/WORKSPACE @@ -9,10 +9,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - default_python_version = "3.9" python_register_multi_toolchains( diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index c6465955ba..50bfe9fe8e 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -6,16 +6,54 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library") -all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"] - -all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"} +all_requirements = [ + "@my_project_pip_deps_vendored_certifi//:pkg", + "@my_project_pip_deps_vendored_charset_normalizer//:pkg", + "@my_project_pip_deps_vendored_idna//:pkg", + "@my_project_pip_deps_vendored_requests//:pkg", + "@my_project_pip_deps_vendored_urllib3//:pkg", +] + +all_whl_requirements_by_package = { + "certifi": "@my_project_pip_deps_vendored_certifi//:whl", + "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", + "idna": "@my_project_pip_deps_vendored_idna//:whl", + "requests": "@my_project_pip_deps_vendored_requests//:whl", + "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl", +} all_whl_requirements = all_whl_requirements_by_package.values() -all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"] - -_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] -_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600} +all_data_requirements = [ + "@my_project_pip_deps_vendored_certifi//:data", + "@my_project_pip_deps_vendored_charset_normalizer//:data", + "@my_project_pip_deps_vendored_idna//:data", + "@my_project_pip_deps_vendored_requests//:data", + "@my_project_pip_deps_vendored_urllib3//:data", +] + +_packages = [ + ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), + ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), + ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), + ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), + ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"), +] +_config = { + "download_only": False, + "enable_implicit_namespace_pkgs": False, + "environment": {}, + "envsubst": ["PIP_RETRIES"], + "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], + "isolated": True, + "pip_data_exclude": [], + "python_interpreter": "python3", + "python_interpreter_target": "@python39_host//:python", + "quiet": True, + "repo": "my_project_pip_deps_vendored", + "repo_prefix": "my_project_pip_deps_vendored_", + "timeout": 600, +} _annotations = {} def requirement(name): @@ -30,11 +68,6 @@ def data_requirement(name): def dist_info_requirement(name): return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "dist_info") -def entry_point(pkg, script = None): - if not script: - script = pkg - return "@my_project_pip_deps_vendored_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script - def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` # down to `setuptools`. diff --git a/gazelle/README.md b/gazelle/README.md index bb688b961a..d68b94de26 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -204,7 +204,10 @@ Python-specific directives are as follows: | Appends additional visibility labels to each generated target. This directive can be set multiple times. | | | [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` | | Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. | - +| `# gazelle:python_label_convention` | `$distribution_name$` | +| Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. | +| `# gazelle:python_label_normalization` | `snake_case` | +| Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". | #### Directive: `python_root`: diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index ccabfd2994..542f6dc015 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -116,7 +116,7 @@ def gazelle_python_manifest( attrs = { "env": { "_TEST_MANIFEST": "$(rootpath {})".format(manifest), - "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), + "_TEST_MANIFEST_GENERATOR_HASH": "$(rlocationpath {})".format(manifest_generator_hash), "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), }, "size": "small", @@ -130,7 +130,10 @@ def gazelle_python_manifest( manifest_generator_hash, ], rundir = ".", - deps = [Label("//manifest")], + deps = [ + Label("//manifest"), + Label("@com_github_bazelbuild_rules_go//go/runfiles"), + ], # kwargs could contain test-specific attributes like size or timeout **dict(attrs, **kwargs) ) diff --git a/gazelle/manifest/test/test.go b/gazelle/manifest/test/test.go index 72cb260d4d..506c7d2074 100644 --- a/gazelle/manifest/test/test.go +++ b/gazelle/manifest/test/test.go @@ -26,6 +26,7 @@ import ( "path/filepath" "testing" + "github.com/bazelbuild/rules_go/go/runfiles" "github.com/bazelbuild/rules_python/gazelle/manifest" ) @@ -49,7 +50,12 @@ func TestGazelleManifestIsUpdated(t *testing.T) { t.Fatal("failed to find the Gazelle manifest file integrity") } - manifestGeneratorHashPath := os.Getenv("_TEST_MANIFEST_GENERATOR_HASH") + manifestGeneratorHashPath, err := runfiles.Rlocation( + os.Getenv("_TEST_MANIFEST_GENERATOR_HASH")) + if err != nil { + t.Fatal("failed to resolve runfiles path of manifest: %v", err) + } + manifestGeneratorHash, err := os.Open(manifestGeneratorHashPath) if err != nil { t.Fatalf("opening %q: %v", manifestGeneratorHashPath, err) diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index c35a261366..b82dd81f8f 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -67,6 +67,8 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.DefaultVisibilty, pythonconfig.Visibility, pythonconfig.TestFilePattern, + pythonconfig.LabelConvention, + pythonconfig.LabelNormalization, } } @@ -196,6 +198,23 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { } } config.SetTestFilePattern(globStrings) + case pythonconfig.LabelConvention: + value := strings.TrimSpace(d.Value) + if value == "" { + log.Fatalf("directive '%s' requires a value", pythonconfig.LabelConvention) + } + config.SetLabelConvention(value) + case pythonconfig.LabelNormalization: + switch directiveArg := strings.ToLower(strings.TrimSpace(d.Value)); directiveArg { + case "pep503": + config.SetLabelNormalization(pythonconfig.Pep503LabelNormalizationType) + case "none": + config.SetLabelNormalization(pythonconfig.NoLabelNormalizationType) + case "snake_case": + config.SetLabelNormalization(pythonconfig.SnakeCaseLabelNormalizationType) + default: + config.SetLabelNormalization(pythonconfig.DefaultLabelNormalizationType) + } } } diff --git a/gazelle/python/testdata/annotation_include_dep/__init__.py b/gazelle/python/testdata/annotation_include_dep/__init__.py index 61015346de..a90a1b9f83 100644 --- a/gazelle/python/testdata/annotation_include_dep/__init__.py +++ b/gazelle/python/testdata/annotation_include_dep/__init__.py @@ -1,5 +1,5 @@ -import module1 import foo # third party package +import module1 # gazelle:include_dep //foo/bar:baz # gazelle:include_dep //hello:world,@star_wars//rebel_alliance/luke:skywalker diff --git a/gazelle/python/testdata/directive_python_label_convention/README.md b/gazelle/python/testdata/directive_python_label_convention/README.md new file mode 100644 index 0000000000..8ce0155fb8 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/README.md @@ -0,0 +1,4 @@ +# Directive: `python_label_convention` + +This test case asserts that the `# gazelle:python_label_convention` directive +works as intended when set. \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_convention/WORKSPACE b/gazelle/python/testdata/directive_python_label_convention/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_convention/test.yaml b/gazelle/python/testdata/directive_python_label_convention/test.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.in b/gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.out b/gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.out new file mode 100644 index 0000000000..697a2027a0 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test1_unset/BUILD.out @@ -0,0 +1,11 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "test1_unset", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = [ + "@gazelle_python_test//google_cloud_aiplatform", + "@gazelle_python_test//google_cloud_storage", + ], +) diff --git a/gazelle/python/testdata/directive_python_label_convention/test1_unset/bar.py b/gazelle/python/testdata/directive_python_label_convention/test1_unset/bar.py new file mode 100644 index 0000000000..99a4b1ce95 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test1_unset/bar.py @@ -0,0 +1,6 @@ +from google.cloud import aiplatform, storage + + +def main(): + a = dir(aiplatform) + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_convention/test1_unset/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_convention/test1_unset/gazelle_python.yaml new file mode 100644 index 0000000000..bd5efaba63 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test1_unset/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + google.cloud.aiplatform: google_cloud_aiplatform + google.cloud.storage: google_cloud_storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.in b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.in new file mode 100644 index 0000000000..83ce6af886 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_label_convention :$distribution_name$ \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.out b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.out new file mode 100644 index 0000000000..061c8e5553 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/BUILD.out @@ -0,0 +1,13 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_label_convention :$distribution_name$ + +py_library( + name = "test2_custom_prefix_colon", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = [ + "@gazelle_python_test//:google_cloud_aiplatform", + "@gazelle_python_test//:google_cloud_storage", + ], +) diff --git a/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/bar.py b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/bar.py new file mode 100644 index 0000000000..99a4b1ce95 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/bar.py @@ -0,0 +1,6 @@ +from google.cloud import aiplatform, storage + + +def main(): + a = dir(aiplatform) + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/gazelle_python.yaml new file mode 100644 index 0000000000..bd5efaba63 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_convention/test2_custom_prefix_colon/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + google.cloud.aiplatform: google_cloud_aiplatform + google.cloud.storage: google_cloud_storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/python/testdata/directive_python_label_normalization/README.md b/gazelle/python/testdata/directive_python_label_normalization/README.md new file mode 100644 index 0000000000..a2e18013a8 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/README.md @@ -0,0 +1,4 @@ +# Directive: `python_label_normalization` + +This test case asserts that the `# gazelle:python_label_normalization` directive +works as intended when set. \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_normalization/WORKSPACE b/gazelle/python/testdata/directive_python_label_normalization/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_normalization/test.yaml b/gazelle/python/testdata/directive_python_label_normalization/test.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.in b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.in new file mode 100644 index 0000000000..5f5620a946 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_label_normalization none \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.out b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.out new file mode 100644 index 0000000000..6e707789d1 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/BUILD.out @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_label_normalization none + +py_library( + name = "test1_type_none", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = ["@gazelle_python_test//google.cloud.storage"], +) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/bar.py b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/bar.py new file mode 100644 index 0000000000..8b3839e00a --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/bar.py @@ -0,0 +1,5 @@ +from google.cloud import storage + + +def main(): + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/gazelle_python.yaml new file mode 100644 index 0000000000..5bfada4437 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test1_type_none/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + # Weird google.cloud.storage here on purpose to make normalization apparent + google.cloud.storage: google.cloud.storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.in b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.in new file mode 100644 index 0000000000..a2cca53870 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_label_normalization pep503 \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.out b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.out new file mode 100644 index 0000000000..7a88c8b98e --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/BUILD.out @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_label_normalization pep503 + +py_library( + name = "test2_type_pep503", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = ["@gazelle_python_test//google-cloud-storage"], +) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/bar.py b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/bar.py new file mode 100644 index 0000000000..8b3839e00a --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/bar.py @@ -0,0 +1,5 @@ +from google.cloud import storage + + +def main(): + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/gazelle_python.yaml new file mode 100644 index 0000000000..5bfada4437 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test2_type_pep503/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + # Weird google.cloud.storage here on purpose to make normalization apparent + google.cloud.storage: google.cloud.storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.in b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.in new file mode 100644 index 0000000000..5d1a19a7a4 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_label_normalization snake_case \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.out b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.out new file mode 100644 index 0000000000..77f180c1c7 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/BUILD.out @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_label_normalization snake_case + +py_library( + name = "test3_type_snake_case", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = ["@gazelle_python_test//google_cloud_storage"], +) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/bar.py b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/bar.py new file mode 100644 index 0000000000..8b3839e00a --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/bar.py @@ -0,0 +1,5 @@ +from google.cloud import storage + + +def main(): + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/gazelle_python.yaml new file mode 100644 index 0000000000..5bfada4437 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test3_type_snake_case/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + # Weird google.cloud.storage here on purpose to make normalization apparent + google.cloud.storage: google.cloud.storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.in b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.out b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.out new file mode 100644 index 0000000000..22971937ed --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/BUILD.out @@ -0,0 +1,8 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "test4_unset_defaults_to_snake_case", + srcs = ["bar.py"], + visibility = ["//:__subpackages__"], + deps = ["@gazelle_python_test//google_cloud_storage"], +) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/bar.py b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/bar.py new file mode 100644 index 0000000000..8b3839e00a --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/bar.py @@ -0,0 +1,5 @@ +from google.cloud import storage + + +def main(): + b = dir(storage) diff --git a/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/gazelle_python.yaml b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/gazelle_python.yaml new file mode 100644 index 0000000000..5bfada4437 --- /dev/null +++ b/gazelle/python/testdata/directive_python_label_normalization/test4_unset_defaults_to_snake_case/gazelle_python.yaml @@ -0,0 +1,6 @@ +manifest: + modules_mapping: + # Weird google.cloud.storage here on purpose to make normalization apparent + google.cloud.storage: google.cloud.storage + pip_repository: + name: gazelle_python_test diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index aa9255290c..41a470a940 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -17,6 +17,7 @@ package pythonconfig import ( "fmt" "path" + "regexp" "strings" "github.com/emirpasic/gods/lists/singlylinkedlist" @@ -77,6 +78,13 @@ const ( // TestFilePattern represents the directive that controls which python // files are mapped to `py_test` targets. TestFilePattern = "python_test_file_pattern" + // LabelConvention represents the directive that defines the format of the + // labels to third-party dependencies. + LabelConvention = "python_label_convention" + // LabelNormalization represents the directive that controls how distribution + // names of labels to third-party dependencies are normalized. Supported values + // are 'none', 'pep503' and 'snake_case' (default). See LabelNormalizationType. + LabelNormalization = "python_label_normalization" ) // GenerationModeType represents one of the generation modes for the Python @@ -96,7 +104,8 @@ const ( ) const ( - packageNameNamingConventionSubstitution = "$package_name$" + packageNameNamingConventionSubstitution = "$package_name$" + distributionNameLabelConventionSubstitution = "$distribution_name$" ) const ( @@ -104,6 +113,10 @@ const ( DefaultVisibilityFmtString = "//%s:__subpackages__" // The default globs used to determine pt_test targets. DefaultTestFilePatternString = "*_test.py,test_*.py" + // The default convention of label of third-party dependencies. + DefaultLabelConvention = "$distribution_name$" + // The default normalization applied to distribution names of third-party dependency labels. + DefaultLabelNormalizationType = SnakeCaseLabelNormalizationType ) // defaultIgnoreFiles is the list of default values used in the @@ -112,14 +125,6 @@ var defaultIgnoreFiles = map[string]struct{}{ "setup.py": {}, } -func SanitizeDistribution(distributionName string) string { - sanitizedDistribution := strings.ToLower(distributionName) - sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") - sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, ".", "_") - - return sanitizedDistribution -} - // Configs is an extension of map[string]*Config. It provides finding methods // on top of the mapping. type Configs map[string]*Config @@ -156,8 +161,18 @@ type Config struct { defaultVisibility []string visibility []string testFilePattern []string + labelConvention string + labelNormalization LabelNormalizationType } +type LabelNormalizationType int + +const ( + NoLabelNormalizationType LabelNormalizationType = iota + Pep503LabelNormalizationType + SnakeCaseLabelNormalizationType +) + // New creates a new Config. func New( repoRoot string, @@ -180,6 +195,8 @@ func New( defaultVisibility: []string{fmt.Sprintf(DefaultVisibilityFmtString, "")}, visibility: []string{}, testFilePattern: strings.Split(DefaultTestFilePatternString, ","), + labelConvention: DefaultLabelConvention, + labelNormalization: DefaultLabelNormalizationType, } } @@ -209,6 +226,8 @@ func (c *Config) NewChild() *Config { defaultVisibility: c.defaultVisibility, visibility: c.visibility, testFilePattern: c.testFilePattern, + labelConvention: c.labelConvention, + labelNormalization: c.labelNormalization, } } @@ -263,10 +282,8 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { } else if gazelleManifest.PipRepository != nil { distributionRepositoryName = gazelleManifest.PipRepository.Name } - sanitizedDistribution := SanitizeDistribution(distributionName) - // @// - lbl := label.New(distributionRepositoryName, sanitizedDistribution, sanitizedDistribution) + lbl := currentCfg.FormatThirdPartyDependency(distributionRepositoryName, distributionName) return lbl.String(), true } } @@ -443,3 +460,48 @@ func (c *Config) SetTestFilePattern(patterns []string) { func (c *Config) TestFilePattern() []string { return c.testFilePattern } + +// SetLabelConvention sets the label convention used for third-party dependencies. +func (c *Config) SetLabelConvention(convention string) { + c.labelConvention = convention +} + +// LabelConvention returns the label convention used for third-party dependencies. +func (c *Config) LabelConvention() string { + return c.labelConvention +} + +// SetLabelConvention sets the label normalization applied to distribution names of third-party dependencies. +func (c *Config) SetLabelNormalization(normalizationType LabelNormalizationType) { + c.labelNormalization = normalizationType +} + +// LabelConvention returns the label normalization applied to distribution names of third-party dependencies. +func (c *Config) LabelNormalization() LabelNormalizationType { + return c.labelNormalization +} + +// FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization. +func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label { + conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName) + + var normConventionalDistributionName string + switch norm := c.LabelNormalization(); norm { + case SnakeCaseLabelNormalizationType: + // See /python/private/normalize_name.bzl + normConventionalDistributionName = strings.ToLower(conventionalDistributionName) + normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "_") + normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "_") + case Pep503LabelNormalizationType: + // See https://packaging.python.org/en/latest/specifications/name-normalization/#name-format + normConventionalDistributionName = strings.ToLower(conventionalDistributionName) // ... "should be lowercased" + normConventionalDistributionName = regexp.MustCompile(`[-_.]+`).ReplaceAllString(normConventionalDistributionName, "-") // ... "all runs of the characters ., -, or _ replaced with a single -" + normConventionalDistributionName = strings.Trim(normConventionalDistributionName, "-") // ... "must start and end with a letter or number" + default: + fallthrough + case NoLabelNormalizationType: + normConventionalDistributionName = conventionalDistributionName + } + + return label.New(repositoryName, normConventionalDistributionName, normConventionalDistributionName) +} diff --git a/gazelle/pythonconfig/pythonconfig_test.go b/gazelle/pythonconfig/pythonconfig_test.go index bf31106e1e..7cdb9af1d1 100644 --- a/gazelle/pythonconfig/pythonconfig_test.go +++ b/gazelle/pythonconfig/pythonconfig_test.go @@ -4,20 +4,244 @@ import ( "testing" ) -func TestDistributionSanitizing(t *testing.T) { +func TestFormatThirdPartyDependency(t *testing.T) { + type testInput struct { + RepositoryName string + DistributionName string + LabelNormalization LabelNormalizationType + LabelConvention string + } + tests := map[string]struct { - input string + input testInput want string }{ - "upper case": {input: "DistWithUpperCase", want: "distwithuppercase"}, - "dashes": {input: "dist-with-dashes", want: "dist_with_dashes"}, - "dots": {input: "dist.with.dots", want: "dist_with_dots"}, - "mixed": {input: "To-be.sanitized", want: "to_be_sanitized"}, + "default / upper case": { + input: testInput{ + DistributionName: "DistWithUpperCase", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//distwithuppercase", + }, + "default / dashes": { + input: testInput{ + DistributionName: "dist-with-dashes", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//dist_with_dashes", + }, + "default / repeating dashes inside": { + input: testInput{ + DistributionName: "friendly--bard", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//friendly_bard", + }, + "default / repeating underscores inside": { + input: testInput{ + DistributionName: "hello___something", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello_something", + }, + "default / prefix repeating underscores": { + input: testInput{ + DistributionName: "__hello-something", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello_something", + }, + "default / suffix repeating underscores": { + input: testInput{ + DistributionName: "hello-something___", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello_something", + }, + "default / prefix repeating dashes": { + input: testInput{ + DistributionName: "---hello-something", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello_something", + }, + "default / suffix repeating dashes": { + input: testInput{ + DistributionName: "hello-something----", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello_something", + }, + "default / dots": { + input: testInput{ + DistributionName: "dist.with.dots", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//dist_with_dots", + }, + "default / mixed": { + input: testInput{ + DistributionName: "FrIeNdLy-._.-bArD", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//friendly_bard", + }, + "default / upper case / custom prefix & suffix": { + input: testInput{ + DistributionName: "DistWithUpperCase", + RepositoryName: "pip", + LabelNormalization: DefaultLabelNormalizationType, + LabelConvention: "pReFiX-$distribution_name$-sUfFiX", + }, + want: "@pip//prefix_distwithuppercase_suffix", + }, + "noop normalization / mixed": { + input: testInput{ + DistributionName: "not-TO-be.sanitized", + RepositoryName: "pip", + LabelNormalization: NoLabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//not-TO-be.sanitized", + }, + "noop normalization / mixed / custom prefix & suffix": { + input: testInput{ + DistributionName: "not-TO-be.sanitized", + RepositoryName: "pip", + LabelNormalization: NoLabelNormalizationType, + LabelConvention: "pre___$distribution_name$___fix", + }, + want: "@pip//pre___not-TO-be.sanitized___fix", + }, + "pep503 / upper case": { + input: testInput{ + DistributionName: "DistWithUpperCase", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//distwithuppercase", + }, + "pep503 / underscores": { + input: testInput{ + DistributionName: "dist_with_underscores", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//dist-with-underscores", + }, + "pep503 / repeating dashes inside": { + input: testInput{ + DistributionName: "friendly--bard", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//friendly-bard", + }, + "pep503 / repeating underscores inside": { + input: testInput{ + DistributionName: "hello___something", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello-something", + }, + "pep503 / prefix repeating underscores": { + input: testInput{ + DistributionName: "__hello-something", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello-something", + }, + "pep503 / suffix repeating underscores": { + input: testInput{ + DistributionName: "hello-something___", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello-something", + }, + "pep503 / prefix repeating dashes": { + input: testInput{ + DistributionName: "---hello-something", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello-something", + }, + "pep503 / suffix repeating dashes": { + input: testInput{ + DistributionName: "hello-something----", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//hello-something", + }, + "pep503 / dots": { + input: testInput{ + DistributionName: "dist.with.dots", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//dist-with-dots", + }, + "pep503 / mixed": { + input: testInput{ + DistributionName: "To-be.sanitized", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: DefaultLabelConvention, + }, + want: "@pip//to-be-sanitized", + }, + "pep503 / underscores / custom prefix & suffix": { + input: testInput{ + DistributionName: "dist_with_underscores", + RepositoryName: "pip", + LabelNormalization: Pep503LabelNormalizationType, + LabelConvention: "pre___$distribution_name$___fix", + }, + want: "@pip//pre-dist-with-underscores-fix", + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - got := SanitizeDistribution(tc.input) + c := Config{ + labelNormalization: tc.input.LabelNormalization, + labelConvention: tc.input.LabelConvention, + } + gotLabel := c.FormatThirdPartyDependency(tc.input.RepositoryName, tc.input.DistributionName) + got := gotLabel.String() if tc.want != got { t.Fatalf("expected %q, got %q", tc.want, got) } diff --git a/internal_deps.bzl b/internal_deps.bzl index 8818751644..56962cbd19 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -173,11 +173,10 @@ def rules_python_internal_deps(): http_archive( name = "com_google_protobuf", - sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", - strip_prefix = "protobuf-21.7", + sha256 = "616bb3536ac1fff3fb1a141450fa28b875e985712170ea7f1bfe5e5fc41e2cd8", + strip_prefix = "protobuf-24.4", urls = [ - "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protobuf-24.4.tar.gz", ], ) diff --git a/internal_setup.bzl b/internal_setup.bzl index bb62611213..1967c0e568 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -22,8 +22,8 @@ load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_in load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries") load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility +load("//python/private/pypi:deps.bzl", "pypi_deps") # buildifier: disable=bzl-visibility def rules_python_internal_setup(): """Setup for rules_python tests and tools.""" @@ -31,7 +31,7 @@ def rules_python_internal_setup(): internal_config_repo(name = "rules_python_internal") # Because we don't use the pip_install rule, we have to call this to fetch its deps - pip_install_dependencies() + pypi_deps() bazel_skylib_workspace() diff --git a/proposals/2018-10-25-selecting-between-python-2-and-3.md b/proposals/2018-10-25-selecting-between-python-2-and-3.md deleted file mode 100644 index e731f971ce..0000000000 --- a/proposals/2018-10-25-selecting-between-python-2-and-3.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -title: Selecting Between Python 2 and 3 -status: Accepted -created: 2018-10-25 -updated: 2019-01-11 -authors: - - [brandjon@](https://github.com/brandjon) -reviewers: - - [mrovner@](https://github.com/mrovner) -discussion thread: [bazel #6583](https://github.com/bazelbuild/bazel/issues/6583) ---- - -# Selecting Between Python 2 and 3 - -## Abstract - -The "Python mode" configuration value controls whether Python 2 or Python 3 is used to run Python targets built by Bazel. This design document reviews the existing mechanisms for setting the Python mode (the "tri-state model") and describes a simplified mechanism that should replace it (the "boolean model"). - -Links to Github issues are given where applicable. See also [bazel #6444](https://github.com/bazelbuild/bazel/issues/6444) for a tracking list of Python mode issues. - -Throughout, when we say `py_binary`, we also mean to include `py_test`. - -## Background - -The Python mode controls whether a Python 2 or 3 interpreter is used to run a `py_binary` that is built by Bazel. - -* When no `py_runtime` is supplied (via `--python_top`), the mode should control whether the command `python2` or `python3` is embedded into the generated wrapper script ([bazel #4815](https://github.com/bazelbuild/bazel/issues/4815)). - -* In a future design for a "`py_toolchain`"-type rule, a pair of interpreter targets will be bundled together as a toolchain, and the mode will control which one gets their full path embedded into this script. - -The Python mode is also used to help validate that Python source code annotated with `srcs_version` is used appropriately: If a Python target has the `srcs_version` attribute set to `PY2` or `PY3` rather than to `PY2AND3` (the default), it can only be depended on by targets built in Python 2 or Python 3 mode respectively. - -Whenever the same Bazel target can be built in multiple configurations within a single build, it is necessary to write the output artifacts of different versions of the target to different paths. Otherwise the build fails with an "action conflict" error -- Bazel's way of avoiding a correctness bug. For Python targets, and more broadly for targets that may transitively depend on Python targets, this means that different output path roots must be used for different Python modes. - -## Out-of-scope generalizations - -It is possible to imagine extending the Python mode and `srcs_version` so that it can check for compatibility with minor releases (ex: "Python 3.7"), patch releases ("Python 3.7.1"), alternative interpreters ("CPython" or "PyPy"), and exclude known bad releases. We decline to do so because this treads into generalized constraint checking, which may be better handled in the future by the [platforms and toolchain framework](https://docs.bazel.build/versions/master/toolchains.html). - -Compared to these other kinds of version checks, Python 2 vs. 3 is a more compelling use case to support with dedicated machinery. The incompatibilities between these versions are more severe. In many code bases there is an ongoing effort to migrate from 2 to 3, while in others there exists Python 2 code that will never be migrated and must be supported indefinitely. - -## Tri-state model - -Under the existing tri-state model, the Python mode can take on three values: `PY2`, `PY3`, and `null`. The first two modes can be triggered by the `--force_python` flag on the command line or by the `default_python_version` attribute on `py_binary` rules. The `null` mode is the default state when neither the flag nor `default_python_version` is specified. `select()` expressions can distinguish between these states by using `config_setting`s that test the value of `force_python` (where `null` is matched by `//conditions:default`). - -The Python mode is "sticky"; once it is set to `PY2` or `PY3`, it stays that way for all subsequent targets. For a `py_binary` target, this means that all transitive dependencies of the target are built with the same mode as the target itself. For the `--force_python` flag, this means that if the flag is given, it applies universally to the entire build invocation, regardless of the `default_python_version` attributes of any Python targets (hence the "default" in the attribute's name). - -### Data dependencies - -In principle the Python mode needs to propagate to any `py_library` targets that are transitively in the `deps` attribute. Conceptually, this corresponds to enforcing that a Python binary cannot `import` a module written for a different version of Python than the currently running interpreter. But there is no need to propagate the mode across the `data` attribute, which often corresponds to one Python binary calling another as a separate process. - -In order to facilitate `PY3` binaries that depend on `PY2` ones and vice versa, the tri-state model needs to be modified so that the mode is reset to `null` for `data` attributes ([bazel #6441](https://github.com/bazelbuild/bazel/issues/6441)). But it's not clear exactly which attributes should trigger a reset. For example, suppose a Python source file is generated by a `genrule`: Then the `genrule` shouldn't propagate any Python mode to any of its attributes, even though it appears in the transitive closure of a `py_binary`'s `deps`. One could imagine resetting the mode across every attribute except those in a small whitelist (`deps` of `py_binary`, `py_test`, and `py_library`), but this would require new functionality in Bazel and possibly interact poorly with Starlark-defined rules. - -### Output roots - -Since targets that are built for Python 3 produce different results than those built for Python 2, the outputs for these two configurations must be kept separate in order to avoid action conflicts. Therefore, targets built in `PY3` mode get placed under an output root that includes the string "`-py3`". - -Currently, targets that are built in the `null` mode default to using Python 2. Counterintuitively, there is a subtle distinction between building a target in `null` mode and `PY2` mode: Even though the same interpreter is used for the top-level target, the target's transitive dependencies may behave differently, for instance if a `select()` on `force_python` is used. This means that using both `PY2` and `null` for the same target can result in action conflicts ([bazel #6501](https://github.com/bazelbuild/bazel/issues/6501)). However, due to a bug it is not yet possible to have both `PY2` and `null` modes within the same build invocation. - -Under the tri-state model, the most straightforward solution for these action conflicts is to use a separate "`-py2`" root for `PY2` mode. This would mean that the same target could be built in not two but three different configurations, corresponding to the three different modes, even though there are only two distinct Python versions. A more complicated alternative would be to prohibit `select()` from being able to distinguish `null` from `PY2`, in order to help ensure that building an arbitrary target in both of these modes does not succeed with different results. - -### Libraries at the top level - -Currently the mode is only changed by `--force_python` and by `py_binary`. This means that when you build a `py_library` at the top level (that is, specifying it directly on the build command line) without a `--force_python` flag, the library gets the `null` mode, which means Python 2 by default. This causes an error if the library has `srcs_python` set to `PY3`. This in turn means you cannot run a flagless build command on a wildcard pattern, such as `bazel build :all` or `bazel build ...`, if any of the targets in the package(s) contains a Python 3-only library target. Worse, if there are both a Python 2-only library and a Python 3-only library, even specifying `--force_python` can't make the wildcard build work. - -In the tri-state model, this can be addressed by allowing `py_library` to change the mode from `null` to either `PY2` or `PY3` based on whichever version is compatible with its `srcs_version` attribute. This was a proposed fix for [bazel #1446](https://github.com/bazelbuild/bazel/issues/1446). - -## Boolean model - -Under the boolean model, `null` is eliminated as a valid value for the Python mode. Instead, the mode will immediately default to either `PY2` or `PY3`. The mode is no longer sticky, but changes as needed whenever a new `py_binary` target is reached. - -Since there is no longer a third value corresponding to "uncommitted", a target can no longer tell whether it was set to `PY2` mode explicitly (by a flag or a `py_binary`), or if it was set by default because no mode was specified. The current version will be inspectable using `config_setting` to read a setting whose value is always one of `"PY2"` or `"PY3"`. - -### Data dependencies - -Since `py_binary` will now change the mode as needed, there is no need to explicitly reset the mode to a particular value (`null`) when crossing `data` attributes. Python 3 targets can freely depend on Python 2 targets and vice versa, so long as the dependency is not via the `deps` attribute in a way that violates `srcs_version` validation (see below). - -### Output roots - -Since there are only two modes, there need only be two output roots. This avoids action conflicts without resorting to creating a redundant third output root, or trying to coerce two similar-but-distinct modes to map onto the same output root. - -Since the mode is not being reset across data dependencies, it is possible that compared to the tri-state model, the boolean model causes some data dependencies to be built in two configurations instead of just one. This is considered to be an acceptable tradeoff of the boolean model. Note that there exist other cases where redundant rebuilding occurs regardless of which model we use. - -### Libraries at the top level - -We want to be able to build a `py_library` at the top level without having to specify the correct mode. At the same time, we still want `srcs_version` to validate that a `py_binary` only depends on `py_library`s that are compatible with its mode. The way to achieve this is to move validation from within the `py_library` rule up to the `py_binary` rule. - -We add two new boolean fields to a provider returned by `py_library`. This bools correspond to whether or not there are any Python 2-only and Python 3-only sources (respectively) in the library's transitive closure. It is easy to compute these bits as boolean ORs as the providers are merged. `py_binary` simply checks these bits against its own Python mode. - -It is important that when `py_binary` detects a version conflict, the user is given the label of one or more transitive dependencies that introduced the constraint. There are several ways to implement this, such as: - -- additional provider fields to propagate context to the error message -- an aspect that traverses the dependencies of the `py_binary` -- emitting warning messages at conflicting `py_library` targets - -The choice of which approach to use is outside the scope of this proposal. - -It is possible that a library is only ever used by Python 3 binaries, but when the library is built as part of a `bazel build :all` command it gets the Python 2 mode by default. This happens even if the library is annotated with `srcs_version` set to `PY3`. Generally this should cause no harm aside from some repeated build work. In the future we can add the same version attribute that `py_binary` has to `py_library`, so the target definition can be made unambiguous. - -Aside from failures due to validation, there is currently a bug whereby building a `PY2` library in `PY3` mode can invoke a stub wrapper that fails ([bazel #1393](https://github.com/bazelbuild/bazel/issues/1393)). We will remove the stub and the behavior that attempted to call it. - -## API changes - -The attribute `default_python_version` of `py_binary` is renamed to `python_version`. The flag `--force_python` is renamed to `--python_version`. (An alternative naming scheme would have been to use "python_major_version", but this is more verbose and inconsistent with `srcs_version`.) - -The Python mode becomes "non-sticky" and `srcs_version` validation becomes less strict. Building a `py_library` target directly will not trigger validation. Building a `py_binary` that depends on a `py_library` having an incompatible version will only fail if the dependency occurs via transitive `deps`, and not when it occurs via other paths such as a `data` dep or a `genrule` that produces a source file. - -The `"py"` provider of Python rules gains two new boolean fields, `has_py2_only_sources` and `has_py3_only_sources`. Existing Python rules are updated to set these fields. Dependencies of Python rules that do not have the `"py"` provider, or those fields on that provider, are treated as if the value of the fields is `False`. - -A new `select()`-able target is created at `@bazel_tools//tools/python:python_version` to return the current Python mode. It can be used in the `flag_values` attribute of `config_setting` and always equals either `"PY2"` or `"PY3"`. (In the future this flag may be moved out of `@bazel_tools` and into `bazelbuild/rules_python`. It may also be made into a `build_setting` so that it can replace the native `--python_version` flag.) It is disallowed to use `"python_version"` in a `config_setting`. - -The flag `--host_force_python` is unaffected by this doc, except that it becomes illegal to use it in a `config_setting`. - -## Migration and compatibility - -The rollout and migration of the new features are split into two groups, syntactic and semantic. - -For syntax, the new `--python_version` flag and `python_version` attribute are available immediately, and behave exactly the same as the old flag and attribute. When both the new and old flags are present on the command line, or both the new and old attributes are present on the same target, the new one takes precedence and the old is ignored. The `@bazel_tools//tools/python:python_version` target is also available unconditionally. - -A migration flag `--incompatible_remove_old_python_version_api` makes unavailable the `--force_python` flag and `default_python_version` attribute, and disallows `select()`-ing on `"force_python"` and `"host_force_python"`. - -For semantics, a flag `--incompatible_allow_python_version_transitions` makes Bazel use the new non-sticky version transitions and the deferred `srcs_version` validation. This applies regardless of whether the new or old API is used to specify the Python version. The new `"py"` provider fields are created regardless of which flags are given. - -Migrating for `--incompatible_remove_old_python_version_api` guarantees that the Python version only ever has two possible values. Migrating for `--incompatible_allow_python_version_transitions` enables data dependencies across different versions of Python. It is recommended to do the API migration first in order to avoid action conflicts. - -Strictly speaking, Python 3 support is currently marked "experimental" in documentation, so in theory we may be able to make these changes without introducing new incompatible and experimental flags. However these changes will likely affect many users of the Python rules, so flags would be more user-friendly. Bazel is also transitioning to a policy wherein all experimental APIs must be flag-guarded, regardless of any disclaimers in their documentation. - -## Changelog - -Date | Change ------------- | ------ -2018-10-25 | Initial version -2018-11-02 | Refine migration path -2018-12-17 | Refine plan for `select()` -2018-12-19 | Refine plan for `select()` again -2019-01-10 | Refine migration path -2019-01-11 | Formal approval and update provider fields diff --git a/proposals/2018-11-08-customizing-the-python-stub-template.md b/proposals/2018-11-08-customizing-the-python-stub-template.md deleted file mode 100644 index 5b9d87820e..0000000000 --- a/proposals/2018-11-08-customizing-the-python-stub-template.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Customizing the Python Stub Template -status: Draft, not yet ready for review -created: 2018-11-08 -updated: 2018-11-09 -authors: - - [brandjon@](https://github.com/brandjon) -reviewers: - - [gpshead@](https://github.com/gpshead) -discussion thread: [bazel #137](https://github.com/bazelbuild/bazel/issues/137) ---- - -# Customizing the Python Stub Template - -## Abstract - -This design document proposes a way to use a different Python stub template, so that users can control how the Python interpreter gets invoked to run their targets. - -**Open questions:** It is not currently clear whether the use cases warrant this kind of expressivity, or whether users can get by with smaller, more narrowly focused ways of parameterizing the existing stub template. The exact stub API is also to be determined. - -## Background - -The usual executable artifact of a `py_binary` rule is a Python stub script. This script manipulates the Python environment to set up the module import path and make the runfiles available, before passing control to the underlying user Python program. The stub script is generated from a [stub template](https://github.com/bazelbuild/bazel/blob/ef0024b831a71521390dcb837b24b86485e5998d/src/main/java/com/google/devtools/build/lib/bazel/rules/python/python_stub_template.txt) by [instantiating some placeholders](https://github.com/bazelbuild/bazel/blob/ef0024b831a71521390dcb837b24b86485e5998d/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L152-L159). - -Generally the Python stub and user program is executed using the system Python interpreter of the target platform. Although this is non-hermetic, the details of the interpreter can be reified by a [`py_runtime`](https://docs.bazel.build/versions/master/be/python.html#py_runtime) target. In the future this will allow for platform-aware selection of an appropriate Python interpreter using the [toolchain](https://docs.bazel.build/versions/master/toolchains.html) framework. - -## Proposal - -A new `Label`-valued attribute, `stub_template`, is added to `py_runtime`. This label points to a file; by default it is `//tools/python:python_stub_template.txt`, which is the renamed location of the existing template. The `py_runtime` rule will resolve this label to an `Artifact` and propagate it in a new field of [`BazelPyRuntimeProvider`](https://github.com/bazelbuild/bazel/blob/1f684e1b87cd8881a0a4b33e86ba66743e32d674/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuntimeProvider.java). [`BazelPythonSemantics#createExecutable`](https://github.com/bazelbuild/bazel/blob/ef0024b831a71521390dcb837b24b86485e5998d/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L130) will refer to this `Artifact` instead of retrieving the template as a Java resource file. - -It is not yet decided which template placeholders are specified, or whether the placeholders will remain an experimental API for the moment. - -## Original approach - -An earlier proposed approach (suggested on the discussion thread, and implemented by [fahhem@](https://github.com/fahhem)) was to add the `stub_template` attribute to `py_binary` rather than to `py_runtime`. - -This would make it trivial to customize the stub for an individual Python target without affecting the other targets in the build. This could be useful if there were a one-off target that had special requirements. - -However, the author believes that the stub is more naturally tied to the Python interpreter than to an individual target. Putting the attribute on `py_runtime` makes it easy to affect all Python targets that use the same interpreter. It also allows the same Python target to use different stubs depending on which interpreter it is built for -- for instance, the same target can have different stubs on different platforms. - -If it is necessary to use a custom stub for a particular target, that could still be achieved by making that one target use a different `py_runtime`. This isn't possible at the moment but will be when a `py_toolchain` rule is added. - -## Changelog - -Date | Change ------------- | ------ -2018-11-08 | Initial version diff --git a/proposals/2019-02-12-design-for-a-python-toolchain.md b/proposals/2019-02-12-design-for-a-python-toolchain.md deleted file mode 100644 index 0d45866107..0000000000 --- a/proposals/2019-02-12-design-for-a-python-toolchain.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -title: Design for a Python Toolchain -status: Accepted -created: 2019-02-12 -updated: 2019-02-21 -authors: - - [brandjon@](https://github.com/brandjon) -reviewers: - - [katre@](https://github.com/katre), [mrovner@](https://github.com/mrovner), [nlopezgi@](https://github.com/nlopezgi) -discussion thread: [bazel #7375](https://github.com/bazelbuild/bazel/issues/7375) ---- - -# Design for a Python Toolchain - -## Abstract - -This doc outlines the design of a Python toolchain rule and its associated machinery. Essentially a new `py_runtime_pair` toolchain rule is created to wrap two `py_runtime` targets (one for Python 2 and one for Python 3), thereby making runtimes discoverable via [toolchain resolution](https://docs.bazel.build/versions/master/toolchains.html). This replaces the previous mechanism of explicitly specifying a global runtime via `--python_top` or `--python_path`; those flags are now deprecated. - -The new toolchain-related definitions are implemented in Starlark. A byproduct of this is that the provider type for `py_runtime` is exposed to Starlark. We also add to `py_runtime` an attribute for declaring whether it represents a Python 2 or Python 3 runtime. - -## Motivation - -The goal is to make the native Python rules use the toolchain framework to resolve the Python runtime. Advantages include: - -* allowing each `py_binary` to use a runtime suitable for its target platform - -* allowing Python 2 and Python 3 targets to run in the same build without [hacks](https://github.com/bazelbuild/bazel/issues/4815#issuecomment-460777113) - -* making it easier to run Python-related builds under remote execution - -* adding support for autodetection of available system Python runtimes, without requiring ad hoc rule logic - -* removing `--python_top` and `--python_path` - -* bringing Python in line with other rule sets and Bazel's best practices - -**Non-goal:** This work does not allow individual `py_binary`s to directly name a Python runtime to use. Instead, this information should be worked into either the configuration or a future toolchain constraint system. See the FAQ, below. - -## Design - -### New definitions - -A new [toolchain type](https://docs.bazel.build/versions/master/toolchains.html#writing-rules-that-use-toolchains) is created at `@bazel_tools//tools/python:toolchain_type`. This is the type for toolchains that provide a way to run Python code. - -Toolchain rules of this type are expected to return a [`ToolchainInfo`](https://docs.bazel.build/versions/master/skylark/lib/ToolchainInfo.html) with two fields, `py2_runtime` and `py3_runtime`, each of type `PyRuntimeInfo`. They are used for `PY2` and `PY3` binaries respectively. - -```python -def _some_python_toolchain_impl(ctx): - ... - return [platform_common.ToolchainInfo( - py2_runtime = PyRuntimeInfo(...), - py3_runtime = PyRuntimeInfo(...))] -``` - -If either Python 2 or Python 3 is not provided by the toolchain, the corresponding field may be set to `None`. This is strongly discouraged, as it will prevent any target relying on that toolchain from using that version of Python. Toolchains that do use `None` here should be registered with lower priority than other toolchains, so that they are chosen only as a fallback. - -`PyRuntimeInfo` is the newly-exposed Starlark name of the native provider returned by the [`py_runtime`](https://docs.bazel.build/versions/master/be/python.html#py_runtime) rule. Like `PyInfo`, it is a top-level built-in name. Also like `PyInfo` and the native Python rules, it will eventually be migrated to Starlark and moved out of the Bazel repository. - -A `PyRuntimeInfo` describes either a *platform runtime* or an *in-build runtime*. A platform runtime accesses a system-installed interpreter at a known path, whereas an in-build runtime points to a build target that acts as the interpreter. In both cases, an "interpreter" is really any executable binary or wrapper script that is capable of running a Python script passed on the command line, following the same conventions as the standard CPython interpreter. Note that any platform runtime imposes a requirement on the target platform. Therefore, any toolchain returning such a `PyRuntimeInfo` should include a corresponding target platform constraint, to ensure it cannot be selected for a platform that does not have the interpreter at that path. Even an in-build runtime can require platform constraints, for instance in the case of a wrapper script that invokes the system interpreter. - -We provide two [`constraint_setting`](https://docs.bazel.build/versions/master/be/platform.html#constraint_setting)s to act as a standardized namespace for this kind of platform constraint: `@bazel_tools//tools/python:py2_interpreter_path` and `@bazel_tools//tools/python:py3_interpreter_path`. This doc does not mandate any particular structure for the names of [`constraint_value`](https://docs.bazel.build/versions/master/be/platform.html#constraint_value)s associated with these settings. If a platform does not provide a Python 2 runtime, it should have no constraint value associated with `py2_interpreter_path`, and similarly for Python 3. - -`PyRuntimeInfo` has the following fields, each of which corresponds to an attribute on `py_runtime`. (The last one, `python_version`, is newly added in this doc.) - -* `interpreter_path`: If this is a platform runtime, this field is the absolute filesystem path to the interpreter on the target platform. Otherwise, this is `None`. - -* `interpreter`: If this is an in-build runtime, this field is a `File` representing the interpreter. Otherwise, this is `None`. - -* `files`: If this is an in-build runtime, this field is a depset of `File`s that need to be added to the runfiles of an executable target that uses this toolchain. The value of `interpreter` need not be included in this field. If this is a platform runtime then this field is `None`. - -* `python_version`: Either the string `"PY2"` or `"PY3"`, indicating which version of Python the interpreter referenced by `interpreter_path` or `interpreter` is. - -The constructor of `PyRuntimeInfo` takes each of these fields as keyword arguments. The constructor enforces the invariants about which combinations of fields may be `None`. Fields that are not meaningful may be omitted; e.g. when `interpreter_path` is given, `interpreter` and `files` may be omitted instead of passing `None`. - -It is not possible to directly specify a system command (e.g. `"python"`) in `interpreter_path`. However, this can be done indirectly by creating a wrapper script that invokes the system command, and referencing that script from the `interpreter` field. - -Finally, we define a standard Python toolchain rule implementing the new toolchain type. The rule's name is `py_runtime_pair` and it can be loaded from `@bazel_tools//tools/python:toolchain.bzl`. It has two label-valued attributes, `py2_runtime` and `py3_runtime`, that refer to `py_runtime` targets. - -### Changes to the native Python rules - -The executable Python rules [`py_binary`](https://docs.bazel.build/versions/master/be/python.html#py_binary) and [`py_test`](https://docs.bazel.build/versions/master/be/python.html#py_test) are modified to require the new toolchain type. The Python runtime information is obtained by retrieving a `PyRuntimeInfo` from either the `py2_runtime` or `py3_runtime` field of the toolchain, rather than from `--python_top`. The `python_version` field of the `PyRuntimeInfo` is also checked to ensure that a `py_runtime` didn't accidentally end up in the wrong place. - -Since `--python_top` is no longer read, it is deprecated. Since `--python_path` was only read when no runtime information is available, but the toolchain must always be present, it too is deprecated. - -Implementation wise, the native `PyRuntimeProvider` is turned into the user-visible `PyRuntimeInfo` by adding Starlark API annotations in the usual way (`@SkylarkCallable`, etc.). A previous version of this proposal suggested defining `PyRuntimeInfo` in Starlark underneath `@bazel_tools` and accessing it from the native rules, but this is technically difficult to implement. - -A `python_version` attribute is added to `py_runtime`. It is mandatory and accepts values `"PY2"` and `"PY3"` only. - -As a drive-by cleanup (and non-breaking change), the `files` attribute of `py_runtime` is made optional. For the non-hermetic case, specifying `files` is nonsensical and it is even an error to give it a non-empty value. For the hermetic case, `files` can be useful but is by no means necessary if the interpreter requires no additional in-repo inputs (such as when the "interpreter" is just a wrapper script that dispatches to the platform's system interpreter). - -### Default toolchain - -For convenience, we supply a predefined [toolchain](https://docs.bazel.build/versions/master/be/platform.html#toolchain) of last resort, `@bazel_tools//tools/python:autodetecting_python_toolchain`. This toolchain is registered with lower priority than any user-registered Python toolchain. It simply dispatches to a wrapper script that tries to locate a suitable interpreter from `PATH` at runtime, on a best-effort basis. It has no platform constraints. - -## Example - -Here is a minimal example that defines a platform whose Python interpreters are located under a non-standard path. The example also defines a Python toolchain to accompany this platform. - -```python -# //platform_defs:BUILD - -load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") - -# Constraint values that represent that the system's "python2" and "python3" -# executables are located under /usr/weirdpath. - -constraint_value( - name = "usr_weirdpath_python2", - constraint_setting = "@bazel_tools//tools/python:py2_interpreter_path", -) - -constraint_value( - name = "usr_weirdpath_python3", - constraint_setting = "@bazel_tools//tools/python:py3_interpreter_path", -) - -# A definition of a platform whose Python interpreters are under these paths. - -platform( - name = "my_platform", - constraint_values = [ - ":usr_weirdpath_python2", - ":usr_weirdpath_python3", - ], -) - -# Python runtime definitions that reify these system paths as BUILD targets. - -py_runtime( - name = "my_platform_py2_runtime", - interpreter_path = "/usr/weirdpath/python2", -) - -py_runtime( - name = "my_platform_py3_runtime", - interpreter_path = "/usr/weirdpath/python3", -) - -py_runtime_pair( - name = "my_platform_runtimes", - py2_runtime = ":my_platform_py2_runtime", - py3_runtime = ":my_platform_py3_runtime", -) - -# A toolchain definition to expose these runtimes to toolchain resolution. - -toolchain( - name = "my_platform_python_toolchain", - # Since the Python interpreter is invoked at runtime on the target - # platform, there's no need to specify execution platform constraints here. - target_compatible_with = [ - # Make sure this toolchain is only selected for a target platform that - # advertises that it has interpreters available under /usr/weirdpath. - ":usr_weirdpath_python2", - ":usr_weirdpath_python3", - ], - toolchain = ":my_platform_runtimes", - toolchain_type = "@bazel_tools//tools/python:toolchain_type", -) -``` - -```python -# //pkg:BUILD - -# An ordinary Python target to build. -py_binary( - name = "my_pybin", - srcs = ["my_pybin.py"], - python_version = "PY3", -) -``` - -```python -# WORKSPACE - -# Register the custom Python toolchain so it can be chosen for my_platform. -register_toolchains( - "//platform_defs:my_platform_python_toolchain", -) -``` - -We can then build with - -``` -bazel build //pkg:my_pybin --platforms=//platform_defs:my_platform -``` - -and thanks to toolchain resolution, the resulting executable will automatically know to use the interpreter located at `/usr/weirdpath/python3`. - -If we had not defined a custom toolchain, then we'd be stuck with `autodetecting_python_toolchain`, which would fail at execution time if `/usr/weirdpath` were not on `PATH`. (It would also be slightly slower since it requires an extra invocation of the interpreter at execution time to confirm its version.) - -## Backward compatibility - -The new `@bazel_tools` definitions and the `PyRuntimeInfo` provider are made available immediately. A new flag, `--incompatible_use_python_toolchains`, is created to assist migration. When the flag is enabled, `py_binary` and `py_test` will use the `PyRuntimeInfo` obtained from the toolchain, instead of the one obtained from `--python_top` or the default information in `--python_path`. In addition, when `--incompatible_use_python_toolchains` is enabled it is an error to set the following flags: `--python_top`, `--python_path`, `--python2_path`, `--python3_path`. (The latter two were already deprecated.) These flags will be deleted when the incompatible flag is removed. - -Because of how the toolchain framework is implemented, it is not possible to gate whether a rule requires a toolchain type based on a flag. Therefore `py_binary` and `py_test` are made to require `@bazel_tools//tools/python:toolchain_type` immediately and unconditionally. This may impact how toolchain resolution determines the toolchains and execution platforms for a given build, but should not otherwise cause problems so long as the build uses constraints correctly. - -The new `python_version` attribute is added to `py_runtime` immediately. Its default value is the same as the `python_version` attribute for `py_binary`, i.e. `PY3` if `--incompatible_py3_is_default` is true and `PY2` otherwise. When `--incompatible_use_python_toolchains` is enabled this attribute becomes mandatory. - -## FAQ - -#### How can I force a `py_binary` to use a given runtime, say for a particular minor version of Python? - -This is not directly addressed by this doc. Note that such a system could be used not just for controlling the minor version of the interpreter, but also to choose between different Python implementations (CPython vs PyPy), compilation modes (optimized, debug), an interpreter linked with a pre-selected set of extensions, etc. - -There are two possible designs. - -The first design is to put this information in the configuration, and have the toolchain read the configuration to decide which `PyRuntimeInfo` to return. We'd use Starlark Build Configurations to define a flag to represent the Python minor version, and transition the `py_binary` target's configuration to use this version. This configuration would be inherited by the resolved toolchain just like any other dependency inherits its parents configuration. The toolchain could then use a `select()` on the minor version flag to choose which `py_runtime` to depend on. - -There's one problem: Currently all toolchains are analyzed in the host configuration. It is expected that this will be addressed soon. - -We could even migrate the Python major version to use this approach. Instead of having two different `ToolchainInfo` fields, `py2_runtime` and `py3_runtime`, we'd have a single `py_runtime` field that would be populated with one or the other based on the configuration. (It's still a good idea to keep them as separate attributes in the user-facing toolchain rule, i.e. `py_runtime_pair`, because it's a very common use case to require both major versions of Python in a build. But note that this causes both runtimes to be analyzed as dependencies, even if the whole build uses only one or the other.) - -The second design for controlling what runtime is chosen is to introduce additional constraints on the toolchain, and let toolchain resolution solve the problem. However, currently toolchains only support constraints on the target and execution platforms, and this is not a platform-related constraint. What would be needed is a per-target semantic-level constraint system. - -The second approach has the advantage of allowing individual runtimes to be registered independently, without having to combine them into a massive `select()`. But the first approach is much more feasible to implement in the short-term. - -#### Why `py_runtime_pair` as opposed to some other way of organizing multiple Python runtimes? - -Alternatives might include a dictionary mapping from version identifiers to runtimes, or a list of runtimes paired with additional metadata. - -The `PY2`/`PY3` dichotomy is already baked into the Python rule set and indeed the Python ecosystem at large. Keeping this concept in the toolchain rule serves to complement, rather than complicate, Bazel's existing Python support. - -It will always be possible to add new toolchains, first by extending the schema of the `ToolchainInfo` accepted by the Python rules, and then by defining new user-facing toolchain rules that serve as front-ends for this provider. - -#### Why not split Python 2 and Python 3 into two separate toolchain types? - -The general pattern for rule sets seems to be to have a single toolchain type representing all of a language's concerns. Case in point: The naming convention for toolchain types is to literally name the target "toolchain_type", and let the package path distinguish its label. - -If the way of categorizing Python runtimes changes in the future, it will probably be easier to migrate rules to use a new provider schema than to use a new set of toolchain types. - -#### How does the introduction of new symbols to `@bazel_tools` affect the eventual plan to migrate the Python rules to `bazelbuild/rules_python`? - -The new `PyRuntimeInfo` provider and `py_runtime_pair` rule would have forwarding aliases set up, so they could be accessed both from `@bazel_tools` and `rules_python` during a future migration window. - -Forwarding aliases would also be defined for the toolchain type and the two `constraint_setting`s. Note that aliasing `toolchain_type`s is currently broken ([#7404](https://github.com/bazelbuild/bazel/issues/7404)). - -In the initial implementation of this proposal, the predefined `autodetecting_python_toolchain` will be automatically registered in the user's workspace by Bazel. This follows precedent for other languages with built-in support in Bazel. Once the rules are migrated to `rules_python`, registration will not be automatic; the user will have to explicitly call a configuration helper defined in `rules_python` from their own `WORKSPACE` file. - -## Changelog - -Date | Change ------------- | ------ -2019-02-12 | Initial version -2019-02-14 | Make `PyRuntimeInfo` natively defined -2019-02-15 | Clarify platform runtime vs in-build runtime -2019-02-21 | Formal approval diff --git a/proposals/README.md b/proposals/README.md deleted file mode 100644 index 36a8a0b7d3..0000000000 --- a/proposals/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Python Rules Proposals - -This is an index of all design documents and proposals for Python rules, both in native code (the Bazel binary) and in Starlark (the rules_python repository). Some of these proposals are also hosted in this directory. - -Proposals that impact native code are also indexed by [bazelbuild/proposals](https://github.com/bazelbuild/proposals), and subject to the [Bazel design process](https://bazel.build/designs/index.html). - -Last updated | Status | Title | Author(s) ------------- | ------------- | ------| --------- -2019-02-21 | Accepted | [Design for a Python Toolchain](https://github.com/bazelbuild/rules_python/blob/master/proposals/2019-02-12-design-for-a-python-toolchain.md) | [brandjon@](https://github.com/brandjon) -2018-11-09 | Draft | [Customizing the Python Stub Template](https://github.com/bazelbuild/rules_python/blob/master/proposals/2018-11-08-customizing-the-python-stub-template.md) | [brandjon@](https://github.com/brandjon) -2019-01-11 | Accepted | [Selecting Between Python 2 and 3](https://github.com/bazelbuild/rules_python/blob/master/proposals/2018-10-25-selecting-between-python-2-and-3.md) | [brandjon@](https://github.com/brandjon) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index cbf29964fb..7a69ac8051 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -24,7 +24,6 @@ that @rules_python//python is only concerned with the core rules. """ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//python/private:autodetecting_toolchain.bzl", "define_autodetecting_toolchain") load(":current_py_toolchain.bzl", "current_py_toolchain") package(default_visibility = ["//visibility:public"]) @@ -39,6 +38,7 @@ filegroup( "//python/constraints:distribution", "//python/entry_points:distribution", "//python/extensions:distribution", + "//python/pip_install:distribution", "//python/private:distribution", "//python/runfiles:distribution", ], @@ -96,12 +96,12 @@ bzl_library( name = "pip_bzl", srcs = ["pip.bzl"], deps = [ - "//python/pip_install:pip_repository_bzl", - "//python/pip_install:repositories_bzl", - "//python/pip_install:requirements_bzl", - "//python/private:bzlmod_enabled_bzl", - "//python/private:full_version_bzl", - "//python/private:render_pkg_aliases_bzl", + "//python/private:normalize_name_bzl", + "//python/private/pypi:multi_pip_parse_bzl", + "//python/private/pypi:package_annotation_bzl", + "//python/private/pypi:pip_compile_bzl", + "//python/private/pypi:pip_repository_bzl", + "//python/private/pypi:whl_library_alias_bzl", "//python/private/whl_filegroup:whl_filegroup_bzl", ], ) @@ -210,7 +210,6 @@ bzl_library( srcs = ["repositories.bzl"], deps = [ ":versions_bzl", - "//python/pip_install:repositories_bzl", "//python/private:auth_bzl", "//python/private:bazel_tools_bzl", "//python/private:bzlmod_enabled_bzl", @@ -219,6 +218,7 @@ bzl_library( "//python/private:internal_config_repo_bzl", "//python/private:repo_utils_bzl", "//python/private:toolchains_repo_bzl", + "//python/private/pypi:deps_bzl", ], ) @@ -319,11 +319,16 @@ toolchain_type( # safe if you know for a fact that your build is completely compatible with the # version of the `python` command installed on the target platform. -define_autodetecting_toolchain(name = "autodetecting_toolchain") +alias( + name = "autodetecting_toolchain", + actual = "//python/runtime_env_toolchains:runtime_env_toolchain", + deprecation = "Use //python/runtime_env_toolchains:all instead", +) alias( name = "autodetecting_toolchain_nonstrict", - actual = ":autodetecting_toolchain", + actual = "//python/runtime_env_toolchains:runtime_env_toolchain", + deprecation = "Use //python/runtime_env_toolchains:all instead", ) # ========= Packaging rules ========= diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index e2d2608c7c..f2383d6056 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -2,24 +2,24 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load( "//python/private:flags.bzl", "BootstrapImplFlag", + "ExecToolsToolchainFlag", "PrecompileAddToRunfilesFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", "PycCollectionFlag", ) load( - "//python/private:pip_flags.bzl", - "INTERNAL_FLAGS", + "//python/private/pypi:flags.bzl", "UniversalWhlFlag", "UseWhlFlag", "WhlLibcFlag", + "define_pypi_internal_flags", ) load(":config_settings.bzl", "construct_config_settings") filegroup( name = "distribution", - srcs = glob(["*.bzl"]) + [ - "BUILD.bazel", + srcs = glob(["**"]) + [ "//python/config_settings/private:distribution", ], visibility = ["//python:__pkg__"], @@ -29,11 +29,30 @@ construct_config_settings( name = "construct_config_settings", ) +string_flag( + name = "exec_tools_toolchain", + build_setting_default = ExecToolsToolchainFlag.DISABLED, + values = sorted(ExecToolsToolchainFlag.__members__.values()), + # NOTE: Only public because it is used in py_toolchain_suite from toolchain + # repositories + visibility = ["//visibility:private"], +) + +config_setting( + name = "is_exec_tools_toolchain_enabled", + flag_values = { + "exec_tools_toolchain": ExecToolsToolchainFlag.ENABLED, + }, + # NOTE: Only public because it is used in py_toolchain_suite from toolchain + # repositories + visibility = ["//visibility:public"], +) + string_flag( name = "precompile", build_setting_default = PrecompileFlag.AUTO, values = sorted(PrecompileFlag.__members__.values()), - # NOTE: Only public because its an implicit dependency + # NOTE: Only public because it's an implicit dependency visibility = ["//visibility:public"], ) @@ -41,7 +60,7 @@ string_flag( name = "precompile_source_retention", build_setting_default = PrecompileSourceRetentionFlag.KEEP_SOURCE, values = sorted(PrecompileSourceRetentionFlag.__members__.values()), - # NOTE: Only public because its an implicit dependency + # NOTE: Only public because it's an implicit dependency visibility = ["//visibility:public"], ) @@ -49,7 +68,7 @@ string_flag( name = "precompile_add_to_runfiles", build_setting_default = PrecompileAddToRunfilesFlag.ALWAYS, values = sorted(PrecompileAddToRunfilesFlag.__members__.values()), - # NOTE: Only public because its an implicit dependency + # NOTE: Only public because it's an implicit dependency visibility = ["//visibility:public"], ) @@ -57,7 +76,7 @@ string_flag( name = "pyc_collection", build_setting_default = PycCollectionFlag.DISABLED, values = sorted(PycCollectionFlag.__members__.values()), - # NOTE: Only public because its an implicit dependency + # NOTE: Only public because it's an implicit dependency visibility = ["//visibility:public"], ) @@ -65,7 +84,7 @@ string_flag( name = "bootstrap_impl", build_setting_default = BootstrapImplFlag.SYSTEM_PYTHON, values = sorted(BootstrapImplFlag.__members__.values()), - # NOTE: Only public because its an implicit dependency + # NOTE: Only public because it's an implicit dependency visibility = ["//visibility:public"], ) @@ -88,6 +107,33 @@ string_flag( visibility = ["//visibility:public"], ) +config_setting( + name = "is_pip_whl_auto", + flag_values = { + ":pip_whl": UseWhlFlag.AUTO, + }, + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +config_setting( + name = "is_pip_whl_no", + flag_values = { + ":pip_whl": UseWhlFlag.NO, + }, + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +config_setting( + name = "is_pip_whl_only", + flag_values = { + ":pip_whl": UseWhlFlag.ONLY, + }, + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + string_flag( name = "pip_whl_osx_arch", build_setting_default = UniversalWhlFlag.ARCH, @@ -117,14 +163,6 @@ string_flag( visibility = ["//visibility:public"], ) -# private pip whl related flags. Their values cannot be changed and they -# are an implementation detail of how `pip_config_settings` work. -[ - string_flag( - name = "_internal_pip_" + flag, - build_setting_default = "", - values = [""], - visibility = ["//visibility:public"], - ) - for flag in INTERNAL_FLAGS -] +define_pypi_internal_flags( + name = "define_pypi_internal_flags", +) diff --git a/python/config_settings/private/BUILD.bazel b/python/config_settings/private/BUILD.bazel index aa68c6508c..cb42e4999a 100644 --- a/python/config_settings/private/BUILD.bazel +++ b/python/config_settings/private/BUILD.bazel @@ -1,7 +1,5 @@ filegroup( name = "distribution", - srcs = glob(["*.bzl"]) + [ - "BUILD.bazel", - ], + srcs = glob(["**"]), visibility = ["//python/config_settings:__pkg__"], ) diff --git a/python/entry_points/BUILD.bazel b/python/entry_points/BUILD.bazel index d45fb18825..46dbd9298b 100644 --- a/python/entry_points/BUILD.bazel +++ b/python/entry_points/BUILD.bazel @@ -32,10 +32,6 @@ bzl_library( filegroup( name = "distribution", - srcs = glob([ - "*.bzl", - ]) + [ - "BUILD.bazel", - ], + srcs = glob(["**"]), visibility = ["//python:__subpackages__"], ) diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel index eb095ab746..e8a63d6d5b 100644 --- a/python/extensions/BUILD.bazel +++ b/python/extensions/BUILD.bazel @@ -28,7 +28,7 @@ bzl_library( name = "pip_bzl", srcs = ["pip.bzl"], visibility = ["//:__subpackages__"], - deps = ["//python/private/bzlmod:pip_bzl"], + deps = ["//python/private/pypi:pip_bzl"], ) bzl_library( @@ -36,8 +36,6 @@ bzl_library( srcs = ["python.bzl"], visibility = ["//:__subpackages__"], deps = [ - "//python/private:util_bzl", - "//python/private/bzlmod:bazel_features_bzl", - "//python/private/bzlmod:python_bzl", + "//python/private:python_bzl", ], ) diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index a69ee34ae7..e9d47263d5 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -14,6 +14,6 @@ "pip module extension for use with bzlmod" -load("//python/private/bzlmod:pip.bzl", _pip = "pip") +load("//python/private/pypi:pip.bzl", _pip = "pip") pip = _pip diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 5428b7542e..4148d90877 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -14,6 +14,6 @@ "Python toolchain module extensions for use with bzlmod" -load("//python/private/bzlmod:python.bzl", _python = "python") +load("//python/private:python.bzl", _python = "python") python = _python diff --git a/python/pip.bzl b/python/pip.bzl index c7cdbb2cc5..a1a67200b1 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -19,252 +19,27 @@ symbols should not be used and they are either undocumented here or marked as for internal use only. """ -load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation") -load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") -load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") +load("//python/private/pypi:multi_pip_parse.bzl", _multi_pip_parse = "multi_pip_parse") +load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") +load("//python/private/pypi:pip_compile.bzl", "pip_compile") +load("//python/private/pypi:pip_repository.bzl", "pip_repository") +load("//python/private/pypi:whl_library_alias.bzl", _whl_library_alias = "whl_library_alias") load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup") -compile_pip_requirements = _compile_pip_requirements +compile_pip_requirements = pip_compile package_annotation = _package_annotation pip_parse = pip_repository whl_filegroup = _whl_filegroup -def _multi_pip_parse_impl(rctx): - rules_python = rctx.attr._rules_python_workspace.workspace_name - load_statements = [] - install_deps_calls = [] - process_requirements_calls = [] - for python_version, pypi_repository in rctx.attr.pip_parses.items(): - sanitized_python_version = python_version.replace(".", "_") - load_statement = """\ -load( - "@{pypi_repository}//:requirements.bzl", - _{sanitized_python_version}_install_deps = "install_deps", - _{sanitized_python_version}_all_requirements = "all_requirements", -)""".format( - pypi_repository = pypi_repository, - sanitized_python_version = sanitized_python_version, - ) - load_statements.append(load_statement) - process_requirements_call = """\ -_process_requirements( - pkg_labels = _{sanitized_python_version}_all_requirements, - python_version = "{python_version}", - repo_prefix = "{pypi_repository}_", -)""".format( - pypi_repository = pypi_repository, - python_version = python_version, - sanitized_python_version = sanitized_python_version, - ) - process_requirements_calls.append(process_requirements_call) - install_deps_call = """ _{sanitized_python_version}_install_deps(**whl_library_kwargs)""".format( - sanitized_python_version = sanitized_python_version, - ) - install_deps_calls.append(install_deps_call) - - # NOTE @aignas 2023-10-31: I am not sure it is possible to render aliases - # for all of the packages using the `render_pkg_aliases` function because - # we need to know what the list of packages for each version is and then - # we would be creating directories for each. - macro_tmpl = "@%s_{}//:{}" % rctx.attr.name - - requirements_bzl = """\ -# Generated by python/pip.bzl - -load("@{rules_python}//python:pip.bzl", "whl_library_alias", "pip_utils") -{load_statements} - -_wheel_names = [] -_version_map = dict() -def _process_requirements(pkg_labels, python_version, repo_prefix): - for pkg_label in pkg_labels: - wheel_name = Label(pkg_label).package - if not wheel_name: - # We are dealing with the cases where we don't have aliases. - workspace_name = Label(pkg_label).workspace_name - wheel_name = workspace_name[len(repo_prefix):] - - _wheel_names.append(wheel_name) - if not wheel_name in _version_map: - _version_map[wheel_name] = dict() - _version_map[wheel_name][python_version] = repo_prefix - -{process_requirements_calls} - -def requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "pkg") - -def whl_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "whl") - -def data_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "data") - -def dist_info_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "dist_info") - -def entry_point(pkg, script = None): - fail("Not implemented yet") - -def install_deps(**whl_library_kwargs): -{install_deps_calls} - for wheel_name in _wheel_names: - whl_library_alias( - name = "{name}_" + wheel_name, - wheel_name = wheel_name, - default_version = "{default_version}", - version_map = _version_map[wheel_name], - ) -""".format( - name = rctx.attr.name, - install_deps_calls = "\n".join(install_deps_calls), - load_statements = "\n".join(load_statements), - macro_tmpl = macro_tmpl, - process_requirements_calls = "\n".join(process_requirements_calls), - rules_python = rules_python, - default_version = rctx.attr.default_version, - ) - rctx.file("requirements.bzl", requirements_bzl) - rctx.file("BUILD.bazel", "exports_files(['requirements.bzl'])") - -_multi_pip_parse = repository_rule( - _multi_pip_parse_impl, - attrs = { - "default_version": attr.string(), - "pip_parses": attr.string_dict(), - "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), - }, -) - -def _whl_library_alias_impl(rctx): - rules_python = rctx.attr._rules_python_workspace.workspace_name - if rctx.attr.default_version: - default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] - else: - default_repo_prefix = None - version_map = rctx.attr.version_map.items() - build_content = ["# Generated by python/pip.bzl"] - for alias_name in ["pkg", "whl", "data", "dist_info"]: - build_content.append(_whl_library_render_alias_target( - alias_name = alias_name, - default_repo_prefix = default_repo_prefix, - rules_python = rules_python, - version_map = version_map, - wheel_name = rctx.attr.wheel_name, - )) - rctx.file("BUILD.bazel", "\n".join(build_content)) - -def _whl_library_render_alias_target( - alias_name, - default_repo_prefix, - rules_python, - version_map, - wheel_name): - # The template below adds one @, but under bzlmod, the name - # is canonical, so we have to add a second @. - if BZLMOD_ENABLED: - rules_python = "@" + rules_python - - alias = ["""\ -alias( - name = "{alias_name}", - actual = select({{""".format(alias_name = alias_name)] - for [python_version, repo_prefix] in version_map: - alias.append("""\ - "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( - full_python_version = full_version(python_version), - actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( - repo_prefix = repo_prefix, - wheel_name = wheel_name, - alias_name = alias_name, - ), - rules_python = rules_python, - )) - if default_repo_prefix: - default_actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( - repo_prefix = default_repo_prefix, - wheel_name = wheel_name, - alias_name = alias_name, - ) - alias.append(' "//conditions:default": "{default_actual}",'.format( - default_actual = default_actual, - )) - - alias.append(" },") # Close select expression condition dict - if not default_repo_prefix: - supported_versions = sorted([python_version for python_version, _ in version_map]) - alias.append(' no_match_error="""{}""",'.format( - NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( - supported_versions = ", ".join(supported_versions), - rules_python = rules_python, - ), - )) - alias.append(" ),") # Close the select expression - alias.append(' visibility = ["//visibility:public"],') - alias.append(")") # Close the alias() expression - return "\n".join(alias) - -whl_library_alias = repository_rule( - _whl_library_alias_impl, - attrs = { - "default_version": attr.string( - mandatory = False, - doc = "Optional Python version in major.minor format, e.g. '3.10'." + - "The Python version of the wheel to use when the versions " + - "from `version_map` don't match. This allows the default " + - "(version unaware) rules to match and select a wheel. If " + - "not specified, then the default rules won't be able to " + - "resolve a wheel and an error will occur.", - ), - "version_map": attr.string_dict(mandatory = True), - "wheel_name": attr.string(mandatory = True), - "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), - }, -) - -def multi_pip_parse(name, default_version, python_versions, python_interpreter_target, requirements_lock, **kwargs): - """NOT INTENDED FOR DIRECT USE! - - This is intended to be used by the multi_pip_parse implementation in the template of the - multi_toolchain_aliases repository rule. - - Args: - name: the name of the multi_pip_parse repository. - default_version: the default Python version. - python_versions: all Python toolchain versions currently registered. - python_interpreter_target: a dictionary which keys are Python versions and values are resolved host interpreters. - requirements_lock: a dictionary which keys are Python versions and values are locked requirements files. - **kwargs: extra arguments passed to all wrapped pip_parse. - - Returns: - The internal implementation of multi_pip_parse repository rule. - """ - pip_parses = {} - for python_version in python_versions: - if not python_version in python_interpreter_target: - fail("Missing python_interpreter_target for Python version %s in '%s'" % (python_version, name)) - if not python_version in requirements_lock: - fail("Missing requirements_lock for Python version %s in '%s'" % (python_version, name)) - - pip_parse_name = name + "_" + python_version.replace(".", "_") - pip_parse( - name = pip_parse_name, - python_interpreter_target = python_interpreter_target[python_version], - requirements_lock = requirements_lock[python_version], - **kwargs - ) - pip_parses[python_version] = pip_parse_name - - return _multi_pip_parse( - name = name, - default_version = default_version, - pip_parses = pip_parses, - ) - # Extra utilities visible to rules_python users. pip_utils = struct( normalize_name = normalize_name, ) + +# The following are only exported here because they are used from +# multi_toolchain_aliases repository_rule, not intended for public use. +# +# See ./private/toolchains_repo.bzl +multi_pip_parse = _multi_pip_parse +whl_library_alias = _whl_library_alias diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 91f2ec7b59..683199f807 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -22,93 +22,39 @@ bzl_library( name = "pip_repository_bzl", srcs = ["pip_repository.bzl"], deps = [ - ":repositories_bzl", - "//python:repositories_bzl", - "//python:versions_bzl", - "//python/pip_install/private:generate_group_library_build_bazel_bzl", - "//python/pip_install/private:generate_whl_library_build_bazel_bzl", - "//python/pip_install/private:srcs_bzl", - "//python/private:bzlmod_enabled_bzl", - "//python/private:envsubst_bzl", - "//python/private:normalize_name_bzl", - "//python/private:parse_requirements_bzl", - "//python/private:parse_whl_name_bzl", - "//python/private:patch_whl_bzl", - "//python/private:render_pkg_aliases_bzl", - "//python/private:repo_utils_bzl", - "//python/private:toolchains_repo_bzl", - "//python/private:whl_target_platforms_bzl", - "@bazel_skylib//lib:sets", + "//python/private/pypi:group_library_bzl", + "//python/private/pypi:package_annotation_bzl", + "//python/private/pypi:pip_repository_bzl", + "//python/private/pypi:whl_library_bzl", ], ) bzl_library( name = "requirements_bzl", srcs = ["requirements.bzl"], - deps = [ - ":repositories_bzl", - "//python:defs_bzl", - ], -) - -bzl_library( - name = "requirements_parser_bzl", - srcs = ["requirements_parser.bzl"], + deps = ["//python/private/pypi:pip_compile_bzl"], ) bzl_library( name = "repositories_bzl", srcs = ["repositories.bzl"], deps = [ - "//:version_bzl", - "//python/private:bazel_tools_bzl", - "@bazel_skylib//lib:versions", + "//python/private/pypi:deps_bzl", ], ) filegroup( name = "distribution", - srcs = glob(["*.bzl"]) + [ - "BUILD.bazel", - "pip_repository_requirements.bzl.tmpl", - "//python/pip_install/private:distribution", - "//python/pip_install/tools/dependency_resolver:distribution", - "//python/pip_install/tools/wheel_installer:distribution", - ], - visibility = ["//:__pkg__"], -) - -filegroup( - name = "repositories", - srcs = ["repositories.bzl"], - visibility = ["//tools/private/update_deps:__pkg__"], -) - -filegroup( - name = "requirements_txt", - srcs = ["tools/requirements.txt"], - visibility = ["//tools/private/update_deps:__pkg__"], + srcs = glob(["**"]), + visibility = ["//python:__pkg__"], ) filegroup( name = "bzl", - srcs = glob(["*.bzl"]) + [ - "//python/pip_install/private:bzl_srcs", - ], + srcs = glob(["*.bzl"]), visibility = ["//:__subpackages__"], ) -filegroup( - name = "py_srcs", - srcs = [ - "//python/pip_install/tools/dependency_resolver:py_srcs", - "//python/pip_install/tools/wheel_installer:py_srcs", - "//python/private:repack_whl.py", - "//tools:wheelmaker.py", - ], - visibility = ["//python/pip_install/private:__pkg__"], -) - exports_files( glob(["*.bzl"]), visibility = ["//docs:__pkg__"], diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index d6c8d9149c..18deee1993 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,1099 +14,13 @@ "" -load("@bazel_skylib//lib:sets.bzl", "sets") -load("//python:repositories.bzl", "is_standalone_interpreter") -load("//python:versions.bzl", "WINDOWS_NAME") -load("//python/pip_install:repositories.bzl", "all_requirements") -load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") -load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") -load("//python/private:envsubst.bzl", "envsubst") -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:patch_whl.bzl", "patch_whl") -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") -load("//python/private:toolchains_repo.bzl", "get_host_os_arch") -load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") - -CPPFLAGS = "CPPFLAGS" - -COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" - -_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" - -def _construct_pypath(rctx): - """Helper function to construct a PYTHONPATH. - - Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl. - This allows us to run python code inside repository rule implementations. - - Args: - rctx: Handle to the repository_context. - - Returns: String of the PYTHONPATH. - """ - - separator = ":" if not "windows" in rctx.os.name.lower() else ";" - pypath = separator.join([ - str(rctx.path(entry).dirname) - for entry in rctx.attr._python_path_entries - ]) - return pypath - -def _get_python_interpreter_attr(rctx): - """A helper function for getting the `python_interpreter` attribute or it's default - - Args: - rctx (repository_ctx): Handle to the rule repository context. - - Returns: - str: The attribute value or it's default - """ - if rctx.attr.python_interpreter: - return rctx.attr.python_interpreter - - if "win" in rctx.os.name: - return "python.exe" - else: - return "python3" - -def _resolve_python_interpreter(rctx): - """Helper function to find the python interpreter from the common attributes - - Args: - rctx: Handle to the rule repository context. - - Returns: - `path` object, for the resolved path to the Python interpreter. - """ - python_interpreter = _get_python_interpreter_attr(rctx) - - if rctx.attr.python_interpreter_target != None: - python_interpreter = rctx.path(rctx.attr.python_interpreter_target) - - (os, _) = get_host_os_arch(rctx) - - # On Windows, the symlink doesn't work because Windows attempts to find - # Python DLLs where the symlink is, not where the symlink points. - if os == WINDOWS_NAME: - python_interpreter = python_interpreter.realpath - elif "/" not in python_interpreter: - # It's a plain command, e.g. "python3", to look up in the environment. - found_python_interpreter = rctx.which(python_interpreter) - if not found_python_interpreter: - fail("python interpreter `{}` not found in PATH".format(python_interpreter)) - python_interpreter = found_python_interpreter - else: - python_interpreter = rctx.path(python_interpreter) - return python_interpreter - -def _get_xcode_location_cflags(rctx): - """Query the xcode sdk location to update cflags - - Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. - Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg - otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 - """ - - # Only run on MacOS hosts - if not rctx.os.name.lower().startswith("mac os"): - return [] - - xcode_sdk_location = repo_utils.execute_unchecked( - rctx, - op = "GetXcodeLocation", - arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], - ) - if xcode_sdk_location.return_code != 0: - return [] - - xcode_root = xcode_sdk_location.stdout.strip() - if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): - # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer - # so we need to change the path to to the macos specific tools which are in a different relative - # path than xcode installed command line tools. - xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) - return [ - "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), - ] - -def _get_toolchain_unix_cflags(rctx, python_interpreter): - """Gather cflags from a standalone toolchain for unix systems. - - Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg - otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 - """ - - # Only run on Unix systems - if not rctx.os.name.lower().startswith(("mac os", "linux")): - return [] - - # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, python_interpreter): - return [] - - stdout = repo_utils.execute_checked_stdout( - rctx, - op = "GetPythonVersionForUnixCflags", - arguments = [ - python_interpreter, - "-c", - "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", - ], - ) - _python_version = stdout - include_path = "{}/include/python{}".format( - python_interpreter.dirname, - _python_version, - ) - - return ["-isystem {}".format(include_path)] - -def use_isolated(ctx, attr): - """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. - - Args: - ctx: repository or module context - attr: attributes for the repo rule or tag extension - - Returns: - True if --isolated should be passed - """ - use_isolated = attr.isolated - - # The environment variable will take precedence over the attribute - isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) - if isolated_env != None: - if isolated_env.lower() in ("0", "false"): - use_isolated = False - else: - use_isolated = True - - return use_isolated - -def _parse_optional_attrs(rctx, args, extra_pip_args = None): - """Helper function to parse common attributes of pip_repository and whl_library repository rules. - - This function also serializes the structured arguments as JSON - so they can be passed on the command line to subprocesses. - - Args: - rctx: Handle to the rule repository context. - args: A list of parsed args for the rule. - extra_pip_args: The pip args to pass. - Returns: Augmented args list. - """ - - if use_isolated(rctx, rctx.attr): - args.append("--isolated") - - # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3) - # support rctx.getenv(name, default): When building incrementally, any change to the value of - # the variable named by name will cause this repository to be re-fetched. - if "getenv" in dir(rctx): - getenv = rctx.getenv - else: - getenv = rctx.os.environ.get - - # Check for None so we use empty default types from our attrs. - # Some args want to be list, and some want to be dict. - if extra_pip_args != None: - args += [ - "--extra_pip_args", - json.encode(struct(arg = [ - envsubst(pip_arg, rctx.attr.envsubst, getenv) - for pip_arg in rctx.attr.extra_pip_args - ])), - ] - - if rctx.attr.download_only: - args.append("--download_only") - - if rctx.attr.pip_data_exclude != None: - args += [ - "--pip_data_exclude", - json.encode(struct(arg = rctx.attr.pip_data_exclude)), - ] - - if rctx.attr.enable_implicit_namespace_pkgs: - args.append("--enable_implicit_namespace_pkgs") - - if rctx.attr.environment != None: - args += [ - "--environment", - json.encode(struct(arg = rctx.attr.environment)), - ] - - return args - -def _create_repository_execution_environment(rctx, python_interpreter): - """Create a environment dictionary for processes we spawn with rctx.execute. - - Args: - rctx (repository_ctx): The repository context. - python_interpreter (path): The resolved python interpreter. - Returns: - Dictionary of environment variable suitable to pass to rctx.execute. - """ - - # Gather any available CPPFLAGS values - cppflags = [] - cppflags.extend(_get_xcode_location_cflags(rctx)) - cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) - - env = { - "PYTHONPATH": _construct_pypath(rctx), - CPPFLAGS: " ".join(cppflags), - } - - return env - -_BUILD_FILE_CONTENTS = """\ -package(default_visibility = ["//visibility:public"]) - -# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it -exports_files(["requirements.bzl"]) -""" - -def _pip_repository_impl(rctx): - requirements_by_platform = parse_requirements( - rctx, - requirements_by_platform = rctx.attr.requirements_by_platform, - requirements_linux = rctx.attr.requirements_linux, - requirements_lock = rctx.attr.requirements_lock, - requirements_osx = rctx.attr.requirements_darwin, - requirements_windows = rctx.attr.requirements_windows, - extra_pip_args = rctx.attr.extra_pip_args, - ) - selected_requirements = {} - options = None - repository_platform = host_platform(rctx.os) - for name, requirements in requirements_by_platform.items(): - r = select_requirement( - requirements, - platform = repository_platform, - ) - if not r: - continue - options = options or r.extra_pip_args - selected_requirements[name] = r.requirement_line - - bzl_packages = sorted(selected_requirements.keys()) - - # Normalize cycles first - requirement_cycles = { - name: sorted(sets.to_list(sets.make(deps))) - for name, deps in rctx.attr.experimental_requirement_cycles.items() - } - - # Check for conflicts between cycles _before_ we normalize package names so - # that reported errors use the names the user specified - for i in range(len(requirement_cycles)): - left_group = requirement_cycles.keys()[i] - left_deps = requirement_cycles.values()[i] - for j in range(len(requirement_cycles) - (i + 1)): - right_deps = requirement_cycles.values()[1 + i + j] - right_group = requirement_cycles.keys()[1 + i + j] - for d in left_deps: - if d in right_deps: - fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) - - # And normalize the names as used in the cycle specs - # - # NOTE: We must check that a listed dependency is actually in the actual - # requirements set for the current platform so that we can support cycles in - # platform-conditional requirements. Otherwise we'll blindly generate a - # label referencing a package which may not be installed on the current - # platform. - requirement_cycles = { - normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) - for name, group in requirement_cycles.items() - } - - imports = [ - # NOTE: Maintain the order consistent with `buildifier` - 'load("@rules_python//python:pip.bzl", "pip_utils")', - 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', - ] - - annotations = {} - for pkg, annotation in rctx.attr.annotations.items(): - filename = "{}.annotation.json".format(normalize_name(pkg)) - rctx.file(filename, json.encode_indent(json.decode(annotation))) - annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) - - config = { - "download_only": rctx.attr.download_only, - "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, - "environment": rctx.attr.environment, - "envsubst": rctx.attr.envsubst, - "extra_pip_args": options, - "isolated": use_isolated(rctx, rctx.attr), - "pip_data_exclude": rctx.attr.pip_data_exclude, - "python_interpreter": _get_python_interpreter_attr(rctx), - "quiet": rctx.attr.quiet, - "repo": rctx.attr.name, - "timeout": rctx.attr.timeout, - } - if rctx.attr.use_hub_alias_dependencies: - config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name) - else: - config["repo_prefix"] = "{}_".format(rctx.attr.name) - - if rctx.attr.python_interpreter_target: - config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) - if rctx.attr.experimental_target_platforms: - config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms - - macro_tmpl = "@%s//{}:{}" % rctx.attr.name - - aliases = render_pkg_aliases( - aliases = { - pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] - for pkg in bzl_packages or [] - }, - ) - for path, contents in aliases.items(): - rctx.file(path, contents) - - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - " # %%GROUP_LIBRARY%%": """\ - group_repo = "{name}__groups" - group_library( - name = group_repo, - repo_prefix = "{name}_", - groups = all_requirement_groups, - )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "pkg") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)), - "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({ - p: macro_tmpl.format(p, "whl") - for p in bzl_packages - })), - "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), - "%%CONFIG%%": _format_dict(_repr_dict(config)), - "%%EXTRA_PIP_ARGS%%": json.encode(options), - "%%IMPORTS%%": "\n".join(imports), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%PACKAGES%%": _format_repr_list( - [ - ("{}_{}".format(rctx.attr.name, p), r) - for p, r in sorted(selected_requirements.items()) - ], - ), - }) - - return - -common_env = [ - "RULES_PYTHON_PIP_ISOLATED", - REPO_DEBUG_ENV_VAR, -] - -common_attrs = { - "download_only": attr.bool( - doc = """ -Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of ---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different -platform from the host platform. - """, - ), - "enable_implicit_namespace_pkgs": attr.bool( - default = False, - doc = """ -If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary -and py_test targets must specify either `legacy_create_init=False` or the global Bazel option -`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. - -This option is required to support some packages which cannot handle the conversion to pkg-util style. - """, - ), - "environment": attr.string_dict( - doc = """ -Environment variables to set in the pip subprocess. -Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy` -Note that pip is run with "--isolated" on the CLI so `PIP__` -style env vars are ignored, but env vars that control requests and urllib3 -can be passed. If you need `PIP__`, take a look at `extra_pip_args` -and `envsubst`. - """, - default = {}, - ), - "envsubst": attr.string_list( - mandatory = False, - doc = """\ -A list of environment variables to substitute (e.g. `["PIP_INDEX_URL", -"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args` -using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset) -or `${VARNAME:-default}` (expanding to default if the variable is unset or empty -in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named -here do not cause packages to be re-fetched. Don't fetch different things based -on the value of these variables. -""", - ), - "experimental_requirement_cycles": attr.string_list_dict( - default = {}, - doc = """\ -A mapping of dependency cycle names to a list of requirements which form that cycle. - -Requirements which form cycles will be installed together and taken as -dependencies together in order to ensure that the cycle is always satisified. - -Example: - `sphinx` depends on `sphinxcontrib-serializinghtml` - When listing both as requirements, ala - - ``` - py_binary( - name = "doctool", - ... - deps = [ - "@pypi//sphinx:pkg", - "@pypi//sphinxcontrib_serializinghtml", - ] - ) - ``` - - Will produce a Bazel error such as - - ``` - ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: - //:doctool (...) - @pypi//sphinxcontrib_serializinghtml:pkg (...) - .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) - | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) - | @pypi_sphinx//:pkg (...) - | @pypi_sphinx//:_pkg (...) - `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) - ``` - - Which we can resolve by configuring these two requirements to be installed together as a cycle - - ``` - pip_parse( - ... - experimental_requirement_cycles = { - "sphinx": [ - "sphinx", - "sphinxcontrib-serializinghtml", - ] - }, - ) - ``` - -Warning: - If a dependency participates in multiple cycles, all of those cycles must be - collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed - as two separate cycles. -""", - ), - "experimental_target_platforms": attr.string_list( - default = [], - doc = """\ -A list of platforms that we will generate the conditional dependency graph for -cross platform wheels by parsing the wheel metadata. This will generate the -correct dependencies for packages like `sphinx` or `pylint`, which include -`colorama` when installed and used on Windows platforms. - -An empty list means falling back to the legacy behaviour where the host -platform is the target platform. - -WARNING: It may not work as expected in cases where the python interpreter -implementation that is being used at runtime is different between different platforms. -This has been tested for CPython only. - -For specific target platforms use values of the form `_` where `` -is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, -`aarch64`, `s390x` and `ppc64le`. - -You can also target a specific Python version by using `cp3__`. -If multiple python versions are specified as target platforms, then select statements -of the `lib` and `whl` targets will include usage of version aware toolchain config -settings like `@rules_python//python/config_settings:is_python_3.y`. - -Special values: `host` (for generating deps for the host platform only) and -`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`. - -NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. -""", - ), - "extra_pip_args": attr.string_list( - doc = """Extra arguments to pass on to pip. Must not contain spaces. - -Supports environment variables using the syntax `$VARNAME` or -`${VARNAME}` (expanding to empty string if unset) or -`${VARNAME:-default}` (expanding to default if the variable is unset -or empty in the environment), if `"VARNAME"` is listed in the -`envsubst` attribute. See also `envsubst`. -""", - ), - "isolated": attr.bool( - doc = """\ -Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to -the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used -to control this flag. -""", - default = True, - ), - "pip_data_exclude": attr.string_list( - doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", - ), - "python_interpreter": attr.string( - doc = """\ -The python interpreter to use. This can either be an absolute path or the name -of a binary found on the host's `PATH` environment variable. If no value is set -`python3` is defaulted for Unix systems and `python.exe` for Windows. -""", - # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr` - # default = "python3" - ), - "python_interpreter_target": attr.label( - allow_single_file = True, - doc = """ -If you are using a custom python interpreter built by another repository rule, -use this attribute to specify its BUILD target. This allows pip_repository to invoke -pip using the same interpreter as your toolchain. If set, takes precedence over -python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python". -""", - ), - "quiet": attr.bool( - default = True, - doc = """\ -If True, suppress printing stdout and stderr output to the terminal. - -If you would like to get more diagnostic output, please use: - - RULES_PYTHON_REPO_DEBUG=1 - -or - - RULES_PYTHON_REPO_DEBUG_VERBOSITY= -""", - ), - "repo_prefix": attr.string( - doc = """ -Prefix for the generated packages will be of the form `@//...` - -DEPRECATED. Only left for people who vendor requirements.bzl. -""", - ), - # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute - "timeout": attr.int( - default = 600, - doc = "Timeout (in seconds) on the rule's execution duration.", - ), - "_py_srcs": attr.label_list( - doc = "Python sources used in the repository rule", - allow_files = True, - default = PIP_INSTALL_PY_SRCS, - ), -} - -pip_repository_attrs = { - "annotations": attr.string_dict( - doc = "Optional annotations to apply to packages", - ), - "requirements_by_platform": attr.label_keyed_string_dict( - doc = """\ -The requirements files and the comma delimited list of target platforms as values. - -The keys are the requirement files and the values are comma-separated platform -identifiers. For now we only support `_` values that are present in -`@platforms//os` and `@platforms//cpu` packages respectively. -""", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """\ -A fully resolved 'requirements.txt' pip requirement file containing the -transitive set of your dependencies. If this file is passed instead of -'requirements' no resolve will take place and pip_repository will create -individual repositories for each of your dependencies so that wheels are -fetched/built only for the targets specified by 'build/run/test'. Note that if -your lockfile is platform-dependent, you can use the `requirements_[platform]` -attributes. - -Note, that in general requirements files are compiled for a specific platform, -but sometimes they can work for multiple platforms. `rules_python` right now -supports requirements files that are created for a particular platform without -platform markers. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "use_hub_alias_dependencies": attr.bool( - default = False, - doc = """\ -Controls if the hub alias dependencies are used. If set to true, then the -group_library will be included in the hub repo. - -True will become default in a subsequent release. -""", - ), - "_template": attr.label( - default = ":pip_repository_requirements.bzl.tmpl", - ), -} - -pip_repository_attrs.update(**common_attrs) - -pip_repository = repository_rule( - attrs = pip_repository_attrs, - doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. - -Those dependencies become available in a generated `requirements.bzl` file. -You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. - -In your WORKSPACE file: - -```starlark -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "pypi", - requirements_lock = ":requirements.txt", -) - -load("@pypi//:requirements.bzl", "install_deps") - -install_deps() -``` - -You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following: -- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library` - created after extracting the `PyYAML` package. -- `@pypi//pyyaml:data` points to the extra data included in the package. -- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package. -- `@pypi//pyyaml:whl` points to the wheel file that was extracted. - -```starlark -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - "@pypi//numpy", - "@pypi//requests", - ], -) -``` - -or - -```starlark -load("@pypi//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("numpy"), - requirement("requests"), - ], -) -``` - -In addition to the `requirement` macro, which is used to access the generated `py_library` -target generated from a package's wheel, The generated `requirements.bzl` file contains -functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -```starlark -load("@pypi//:requirements.bzl", "entry_point") - -alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), -) -``` - -Note that for packages whose name and script are the same, only the name of the package -is needed when calling the `entry_point` macro. - -```starlark -load("@pip//:requirements.bzl", "entry_point") - -alias( - name = "flake8", - actual = entry_point("flake8"), -) -``` - -### Vendoring the requirements.bzl file - -In some cases you may not want to generate the requirements.bzl file as a repository rule -while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module -such as a ruleset, you may want to include the requirements.bzl file rather than make your users -install the WORKSPACE setup to generate it. -See https://github.com/bazelbuild/rules_python/issues/608 - -This is the same workflow as Gazelle, which creates `go_repository` rules with -[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) - -To do this, use the "write to source file" pattern documented in -https://blog.aspect.dev/bazel-can-write-to-the-source-folder -to put a copy of the generated requirements.bzl into your project. -Then load the requirements.bzl file directly rather than from the generated repository. -See the example in rules_python/examples/pip_parse_vendored. -""", - implementation = _pip_repository_impl, - environ = common_env, -) - -def _whl_library_impl(rctx): - python_interpreter = _resolve_python_interpreter(rctx) - args = [ - python_interpreter, - "-m", - "python.pip_install.tools.wheel_installer.wheel_installer", - "--requirement", - rctx.attr.requirement, - ] - extra_pip_args = [] - extra_pip_args.extend(rctx.attr.extra_pip_args) - - # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx, python_interpreter) - - whl_path = None - if rctx.attr.whl_file: - whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) - elif rctx.attr.urls: - filename = rctx.attr.filename - urls = rctx.attr.urls - if not filename: - _, _, filename = urls[0].rpartition("/") - - if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")): - if rctx.attr.filename: - msg = "got '{}'".format(filename) - else: - msg = "detected '{}' from url:\n{}".format(filename, urls[0]) - fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg)) - - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - auth = get_auth(rctx, urls), - ) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(rctx.attr.filename) - else: - # It is an sdist and we need to tell PyPI to use a file in this directory - # and not use any indexes. - extra_pip_args.extend(["--no-index", "--find-links", "."]) - - args = _parse_optional_attrs(rctx, args, extra_pip_args) - - if not whl_path: - repo_utils.execute_checked( - rctx, - op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement), - arguments = args, - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) - if not rctx.delete("whl_file.json"): - fail("failed to delete the whl_file.json file") - - if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip - - whl_path = patch_whl( - rctx, - python_interpreter = python_interpreter, - whl_path = whl_path, - patches = patches, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - target_platforms = rctx.attr.experimental_target_platforms - if target_platforms: - parsed_whl = parse_whl_name(whl_path.basename) - if parsed_whl.platform_tag != "any": - # NOTE @aignas 2023-12-04: if the wheel is a platform specific - # wheel, we only include deps for that target platform - target_platforms = [ - p.target_platform - for p in whl_target_platforms( - platform_tag = parsed_whl.platform_tag, - abi_tag = parsed_whl.abi_tag, - ) - ] - - repo_utils.execute_checked( - rctx, - op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), - arguments = args + [ - "--whl-file", - whl_path, - ] + ["--platform={}".format(p) for p in target_platforms], - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - metadata = json.decode(rctx.read("metadata.json")) - rctx.delete("metadata.json") - - entry_points = {} - for item in metadata["entry_points"]: - name = item["name"] - module = item["module"] - attribute = item["attribute"] - - # There is an extreme edge-case with entry_points that end with `.py` - # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 - entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name - entry_point_target_name = ( - _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py - ) - entry_point_script_name = entry_point_target_name + ".py" - - rctx.file( - entry_point_script_name, - _generate_entry_point_contents(module, attribute), - ) - entry_points[entry_point_without_py] = entry_point_script_name - - build_file_contents = generate_whl_library_build_bazel( - dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), - whl_name = whl_path.basename, - dependencies = metadata["deps"], - dependencies_by_platform = metadata["deps_by_platform"], - group_name = rctx.attr.group_name, - group_deps = rctx.attr.group_deps, - data_exclude = rctx.attr.pip_data_exclude, - tags = [ - "pypi_name=" + metadata["name"], - "pypi_version=" + metadata["version"], - ], - entry_points = entry_points, - annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), - ) - rctx.file("BUILD.bazel", build_file_contents) - - return - -def _generate_entry_point_contents( - module, - attribute, - shebang = "#!/usr/bin/env python3"): - """Generate the contents of an entry point script. - - Args: - module (str): The name of the module to use. - attribute (str): The name of the attribute to call. - shebang (str, optional): The shebang to use for the entry point python - file. - - Returns: - str: A string of python code. - """ - contents = """\ -{shebang} -import sys -from {module} import {attribute} -if __name__ == "__main__": - sys.exit({attribute}()) -""".format( - shebang = shebang, - module = module, - attribute = attribute, - ) - return contents - -# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique -whl_library_attrs = dict({ - "annotation": attr.label( - doc = ( - "Optional json encoded file containing annotation to apply to the extracted wheel. " + - "See `package_annotation`" - ), - allow_files = True, - ), - "dep_template": attr.string( - doc = """ -The dep template to use for referencing the dependencies. It should have `{name}` -and `{target}` tokens that will be replaced with the normalized distribution name -and the target that we need respectively. -""", - ), - "filename": attr.string( - doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", - ), - "group_deps": attr.string_list( - doc = "List of dependencies to skip in order to break the cycles within a dependency group.", - default = [], - ), - "group_name": attr.string( - doc = "Name of the group, if any.", - ), - "repo": attr.string( - mandatory = True, - doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", - ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), - "sha256": attr.string( - doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", - ), - "urls": attr.string_list( - doc = """\ -The list of urls of the whl to be downloaded using bazel downloader. Using this -attr makes `extra_pip_args` and `download_only` ignored.""", - ), - "whl_file": attr.label( - doc = "The whl file that should be used instead of downloading or building the whl.", - ), - "whl_patches": attr.label_keyed_string_dict( - doc = """a label-keyed-string dict that has - json.encode(struct([whl_file], patch_strip]) as values. This - is to maintain flexibility and correct bzlmod extension interface - until we have a better way to define whl_library and move whl - patching to a separate place. INTERNAL USE ONLY.""", - ), - "_python_path_entries": attr.label_list( - # Get the root directory of these rules and keep them as a default attribute - # in order to avoid unnecessary repository fetching restarts. - # - # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 - default = [ - Label("//:BUILD.bazel"), - ] + [ - # Includes all the external dependencies from repositories.bzl - Label("@" + repo + "//:BUILD.bazel") - for repo in all_requirements - ], - ), -}, **common_attrs) -whl_library_attrs.update(AUTH_ATTRS) - -whl_library = repository_rule( - attrs = whl_library_attrs, - doc = """ -Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. -Instantiated from pip_repository and inherits config options from there.""", - implementation = _whl_library_impl, - environ = common_env, -) - -def package_annotation( - additive_build_content = None, - copy_files = {}, - copy_executables = {}, - data = [], - data_exclude_glob = [], - srcs_exclude_glob = []): - """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. - - [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md - - Args: - additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package. - copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf] - copy_executables (dict, optional): A mapping of `src` and `out` files for - [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as - executable. - data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target. - data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated - `py_library` target. - srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target. - - Returns: - str: A json encoded string of the provided content. - """ - return json.encode(struct( - additive_build_content = additive_build_content, - copy_files = copy_files, - copy_executables = copy_executables, - data = data, - data_exclude_glob = data_exclude_glob, - srcs_exclude_glob = srcs_exclude_glob, - )) - -def _group_library_impl(rctx): - build_file_contents = generate_group_library_build_bazel( - repo_prefix = rctx.attr.repo_prefix, - groups = rctx.attr.groups, - ) - rctx.file("BUILD.bazel", build_file_contents) - -group_library = repository_rule( - attrs = { - "groups": attr.string_list_dict( - doc = "A mapping of group names to requirements within that group.", - ), - "repo_prefix": attr.string( - doc = "Prefix used for the whl_library created components of each group", - ), - }, - implementation = _group_library_impl, - doc = """ -Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. -This is an implementation detail of dependency groups and should not be used alone. - """, -) - -# pip_repository implementation - -def _format_list(items): - return "[{}]".format(", ".join(items)) - -def _format_repr_list(strings): - return _format_list( - [repr(s) for s in strings], - ) - -def _repr_dict(items): - return {k: repr(v) for k, v in items.items()} - -def _format_dict(items): - return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()]))) +load("//python/private/pypi:group_library.bzl", _group_library = "group_library") +load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") +load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository") +load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") + +# Re-exports for backwards compatibility +group_library = _group_library +pip_repository = _pip_repository +whl_library = _whl_library +package_annotation = _package_annotation diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel deleted file mode 100644 index 887d2d3468..0000000000 --- a/python/pip_install/private/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load(":pip_install_utils.bzl", "srcs_module") - -package(default_visibility = ["//:__subpackages__"]) - -exports_files([ - "srcs.bzl", -]) - -filegroup( - name = "distribution", - srcs = glob(["*"]), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "bzl_srcs", - srcs = glob(["*.bzl"]), -) - -srcs_module( - name = "srcs_module", - srcs = "//python/pip_install:py_srcs", - dest = ":srcs.bzl", -) - -bzl_library( - name = "generate_whl_library_build_bazel_bzl", - srcs = ["generate_whl_library_build_bazel.bzl"], - deps = [ - "//python/private:labels_bzl", - "//python/private:normalize_name_bzl", - ], -) - -bzl_library( - name = "generate_group_library_build_bazel_bzl", - srcs = ["generate_group_library_build_bazel.bzl"], - deps = [ - "//python/private:labels_bzl", - "//python/private:normalize_name_bzl", - ], -) - -bzl_library( - name = "srcs_bzl", - srcs = ["srcs.bzl"], -) diff --git a/python/pip_install/private/pip_install_utils.bzl b/python/pip_install/private/pip_install_utils.bzl deleted file mode 100644 index 488583dcb8..0000000000 --- a/python/pip_install/private/pip_install_utils.bzl +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for `rules_python` pip rules""" - -_SRCS_TEMPLATE = """\ -\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules - -This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please -`bazel run` this target to apply any updates. Note that doing so will discard any local modifications. -"\"\" - -# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the -# sources changed. -PIP_INSTALL_PY_SRCS = [ - {srcs} -] -""" - -def _src_label(file): - dir_path, file_name = file.short_path.rsplit("/", 1) - - return "@rules_python//{}:{}".format( - dir_path, - file_name, - ) - -def _srcs_module_impl(ctx): - srcs = [_src_label(src) for src in ctx.files.srcs] - if not srcs: - fail("`srcs` cannot be empty") - output = ctx.actions.declare_file(ctx.label.name) - - ctx.actions.write( - output = output, - content = _SRCS_TEMPLATE.format( - srcs = "\n ".join(["\"{}\",".format(src) for src in srcs]), - ), - ) - - return DefaultInfo( - files = depset([output]), - ) - -_srcs_module = rule( - doc = "A rule for writing a list of sources to a templated file", - implementation = _srcs_module_impl, - attrs = { - "srcs": attr.label( - doc = "A filegroup of source files", - allow_files = True, - ), - }, -) - -_INSTALLER_TEMPLATE = """\ -#!/bin/bash -set -euo pipefail -cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}" -""" - -def _srcs_updater_impl(ctx): - output = ctx.actions.declare_file(ctx.label.name + ".sh") - target_file = ctx.file.input - dest = ctx.file.dest.short_path - - ctx.actions.write( - output = output, - content = _INSTALLER_TEMPLATE.format( - path = target_file.short_path, - dest = dest, - ), - is_executable = True, - ) - - return DefaultInfo( - files = depset([output]), - runfiles = ctx.runfiles(files = [target_file]), - executable = output, - ) - -_srcs_updater = rule( - doc = "A rule for writing a `srcs.bzl` file back to the repository", - implementation = _srcs_updater_impl, - attrs = { - "dest": attr.label( - doc = "The target file to write the new `input` to.", - allow_single_file = ["srcs.bzl"], - mandatory = True, - ), - "input": attr.label( - doc = "The file to write back to the repository", - allow_single_file = True, - mandatory = True, - ), - }, - executable = True, -) - -def srcs_module(name, dest, **kwargs): - """A helper rule to ensure `pip_repository` rules are always up to date - - Args: - name (str): The name of the sources module - dest (str): The filename the module should be written as in the current package. - **kwargs (dict): Additional keyword arguments - """ - tags = kwargs.pop("tags", []) - - _srcs_module( - name = name, - tags = tags, - **kwargs - ) - - _srcs_updater( - name = name + ".update", - input = name, - dest = dest, - tags = tags, - ) diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl deleted file mode 100644 index e92e49fc5f..0000000000 --- a/python/pip_install/private/srcs.bzl +++ /dev/null @@ -1,18 +0,0 @@ -"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules - -This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please -`bazel run` this target to apply any updates. Note that doing so will discard any local modifications. -""" - -# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the -# sources changed. -PIP_INSTALL_PY_SRCS = [ - "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py", - "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py", - "@rules_python//python/pip_install/tools/wheel_installer:arguments.py", - "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py", - "@rules_python//python/pip_install/tools/wheel_installer:wheel.py", - "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py", - "@rules_python//python/private:repack_whl.py", - "@rules_python//tools:wheelmaker.py", -] diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 3f91860b8b..5231d1f0a1 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -14,131 +14,6 @@ "" -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//python/private/pypi:deps.bzl", "pypi_deps") -_RULE_DEPS = [ - # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' - ( - "pypi__build", - "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", - "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", - ), - ( - "pypi__click", - "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - ), - ( - "pypi__colorama", - "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - ), - ( - "pypi__importlib_metadata", - "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", - "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - ), - ( - "pypi__installer", - "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - ), - ( - "pypi__more_itertools", - "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", - "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", - ), - ( - "pypi__packaging", - "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", - "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - ), - ( - "pypi__pep517", - "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", - "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", - ), - ( - "pypi__pip", - "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - ), - ( - "pypi__pip_tools", - "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", - "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", - ), - ( - "pypi__pyproject_hooks", - "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", - "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", - ), - ( - "pypi__setuptools", - "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl", - "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - ), - ( - "pypi__tomli", - "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - ), - ( - "pypi__wheel", - "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", - "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", - ), - ( - "pypi__zipp", - "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", - "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", - ), - # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' -] - -_GENERIC_WHEEL = """\ -package(default_visibility = ["//visibility:public"]) - -load("@rules_python//python:defs.bzl", "py_library") - -py_library( - name = "lib", - srcs = glob(["**/*.py"]), - data = glob(["**/*"], exclude=[ - # These entries include those put into user-installed dependencies by - # data_exclude in /python/pip_install/tools/bazel.py - # to avoid non-determinism following pip install's behavior. - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created - "**/* *", - "**/*.dist-info/RECORD", - "BUILD", - "WORKSPACE", - ]), - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["."], -) -""" - -# Collate all the repository names so they can be easily consumed -all_requirements = [name for (name, _, _) in _RULE_DEPS] - -def requirement(pkg): - return Label("@pypi__" + pkg + "//:lib") - -def pip_install_dependencies(): - """ - Fetch dependencies these rules depend on. Workspaces that use the pip_parse rule can call this. - """ - for (name, url, sha256) in _RULE_DEPS: - maybe( - http_archive, - name, - url = url, - sha256 = sha256, - type = "zip", - build_file_content = _GENERIC_WHEEL, - ) +pip_install_dependencies = pypi_deps diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 5caf7629f5..6ae3f8fef1 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -14,149 +14,6 @@ """Rules to verify and update pip-compile locked requirements.txt""" -load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") -load("//python/pip_install:repositories.bzl", "requirement") +load("//python/private/pypi:pip_compile.bzl", "pip_compile") -def compile_pip_requirements( - name, - src = None, - extra_args = [], - extra_deps = [], - generate_hashes = True, - py_binary = _py_binary, - py_test = _py_test, - requirements_in = None, - requirements_txt = None, - requirements_darwin = None, - requirements_linux = None, - requirements_windows = None, - visibility = ["//visibility:private"], - tags = None, - **kwargs): - """Generates targets for managing pip dependencies with pip-compile. - - By default this rules generates a filegroup named "[name]" which can be included in the data - of some other compile_pip_requirements rule that references these requirements - (e.g. with `-r ../other/requirements.txt`). - - It also generates two targets for running pip-compile: - - - validate with `bazel test [name]_test` - - update with `bazel run [name].update` - - If you are using a version control system, the requirements.txt generated by this rule should - be checked into it to ensure that all developers/users have the same dependency versions. - - Args: - name: base name for generated targets, typically "requirements". - src: file containing inputs to dependency resolution. If not specified, - defaults to `pyproject.toml`. Supported formats are: - * a requirements text file, usually named `requirements.in` - * A `.toml` file, where the `project.dependencies` list is used as per - [PEP621](https://peps.python.org/pep-0621/). - extra_args: passed to pip-compile. - extra_deps: extra dependencies passed to pip-compile. - generate_hashes: whether to put hashes in the requirements_txt file. - py_binary: the py_binary rule to be used. - py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. Deprecated, use src instead. - requirements_txt: result of "compiling" the requirements.in file. - requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. - requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. - requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. - tags: tagging attribute common to all build rules, passed to both the _test and .update rules. - visibility: passed to both the _test and .update rules. - **kwargs: other bazel attributes passed to the "_test" rule. - """ - if requirements_in and src: - fail("Only one of 'src' and 'requirements_in' attributes can be used") - else: - src = requirements_in or src or "pyproject.toml" - - requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt - - # "Default" target produced by this macro - # Allow a compile_pip_requirements rule to include another one in the data - # for a requirements file that does `-r ../other/requirements.txt` - native.filegroup( - name = name, - srcs = kwargs.pop("data", []) + [requirements_txt], - visibility = visibility, - ) - - data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] - - # Use the Label constructor so this is expanded in the context of the file - # where it appears, which is to say, in @rules_python - pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py") - - loc = "$(rlocationpath {})" - - args = [ - loc.format(src), - loc.format(requirements_txt), - "//%s:%s.update" % (native.package_name(), name), - "--resolver=backtracking", - "--allow-unsafe", - ] - if generate_hashes: - args.append("--generate-hashes") - if requirements_linux: - args.append("--requirements-linux={}".format(loc.format(requirements_linux))) - if requirements_darwin: - args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) - if requirements_windows: - args.append("--requirements-windows={}".format(loc.format(requirements_windows))) - args.extend(extra_args) - - deps = [ - requirement("build"), - requirement("click"), - requirement("colorama"), - requirement("importlib_metadata"), - requirement("more_itertools"), - requirement("packaging"), - requirement("pep517"), - requirement("pip"), - requirement("pip_tools"), - requirement("pyproject_hooks"), - requirement("setuptools"), - requirement("tomli"), - requirement("zipp"), - Label("//python/runfiles:runfiles"), - ] + extra_deps - - tags = tags or [] - tags.append("requires-network") - tags.append("no-remote-exec") - tags.append("no-sandbox") - attrs = { - "args": args, - "data": data, - "deps": deps, - "main": pip_compile, - "srcs": [pip_compile], - "tags": tags, - "visibility": visibility, - } - - # cheap way to detect the bazel version - _bazel_version_4_or_greater = "propeller_optimize" in dir(native) - - # Bazel 4.0 added the "env" attribute to py_test/py_binary - if _bazel_version_4_or_greater: - attrs["env"] = kwargs.pop("env", {}) - - py_binary( - name = name + ".update", - **attrs - ) - - timeout = kwargs.pop("timeout", "short") - - py_test( - name = name + "_test", - timeout = timeout, - # kwargs could contain test-specific attributes like size or timeout - **dict(attrs, **kwargs) - ) +compile_pip_requirements = pip_compile diff --git a/python/pip_install/requirements_parser.bzl b/python/pip_install/requirements_parser.bzl index 3b49fdf181..82ec1b946c 100644 --- a/python/pip_install/requirements_parser.bzl +++ b/python/pip_install/requirements_parser.bzl @@ -14,120 +14,6 @@ "Pip requirements parser for Starlark" -_STATE = struct( - # Consume extraneous whitespace - ConsumeSpace = 0, - # Consume a comment - ConsumeComment = 1, - # Parse the name of a pip package - ParseDependency = 2, - # Parse a full requirement line - ParseRequirement = 3, - # Parse a pip option - ParseOption = 4, -) +load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt") -EOF = {} - -def parse(content): - """A simplistic (and incomplete) pip requirements lockfile parser. - - Parses package names and their full requirement lines, as well pip - options. - - Args: - content: lockfile content as a string - - Returns: - Struct with fields `requirements` and `options`. - - requirements: List of requirements, where each requirement is a 2-element - tuple containing the package name and the requirement line. - E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...] - - options: List of pip option lines - """ - content = content.replace("\r", "") - - result = struct( - requirements = [], - options = [], - ) - state = _STATE.ConsumeSpace - buffer = "" - - inputs = content.elems()[:] - inputs.append(EOF) - - for input in inputs: - if state == _STATE.ConsumeSpace: - (state, buffer) = _handleConsumeSpace(input) - elif state == _STATE.ConsumeComment: - (state, buffer) = _handleConsumeComment(input, buffer, result) - elif state == _STATE.ParseDependency: - (state, buffer) = _handleParseDependency(input, buffer, result) - elif state == _STATE.ParseOption: - (state, buffer) = _handleParseOption(input, buffer, result) - elif state == _STATE.ParseRequirement: - (state, buffer) = _handleParseRequirement(input, buffer, result) - else: - fail("Unknown state %d" % state) - - return result - -def _handleConsumeSpace(input): - if input == EOF: - return (_STATE.ConsumeSpace, "") - if input.isspace(): - return (_STATE.ConsumeSpace, "") - elif input == "#": - return (_STATE.ConsumeComment, "") - elif input == "-": - return (_STATE.ParseOption, input) - - return (_STATE.ParseDependency, input) - -def _handleConsumeComment(input, buffer, result): - if input == "\n": - if len(result.requirements) > 0 and len(result.requirements[-1]) == 1: - result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - elif len(buffer) > 0: - result.options.append(buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - return (_STATE.ConsumeSpace, "") - return (_STATE.ConsumeComment, buffer) - -def _handleParseDependency(input, buffer, result): - if input == EOF: - fail("Enountered unexpected end of file while parsing requirement") - elif input.isspace() or input in [">", "<", "~", "=", ";", "["]: - result.requirements.append((buffer,)) - return (_STATE.ParseRequirement, buffer + input) - - return (_STATE.ParseDependency, buffer + input) - -def _handleParseOption(input, buffer, result): - if input == "\n" and buffer.endswith("\\"): - return (_STATE.ParseOption, buffer[0:-1]) - elif input == " ": - result.options.append(buffer.rstrip("\n")) - return (_STATE.ParseOption, "") - elif input == "\n" or input == EOF: - result.options.append(buffer.rstrip("\n")) - return (_STATE.ConsumeSpace, "") - elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): - return (_STATE.ConsumeComment, buffer) - - return (_STATE.ParseOption, buffer + input) - -def _handleParseRequirement(input, buffer, result): - if input == "\n" and buffer.endswith("\\"): - return (_STATE.ParseRequirement, buffer[0:-1]) - elif input == "\n" or input == EOF: - result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): - return (_STATE.ConsumeComment, buffer) - - return (_STATE.ParseRequirement, buffer + input) +parse = parse_requirements_txt diff --git a/python/pip_install/tools/dependency_resolver/BUILD.bazel b/python/pip_install/tools/dependency_resolver/BUILD.bazel deleted file mode 100644 index c2cfb39509..0000000000 --- a/python/pip_install/tools/dependency_resolver/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -exports_files(["dependency_resolver.py"]) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel deleted file mode 100644 index a396488d3d..0000000000 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ /dev/null @@ -1,91 +0,0 @@ -load("//python:defs.bzl", "py_binary", "py_library", "py_test") -load("//python/pip_install:repositories.bzl", "requirement") - -py_library( - name = "lib", - srcs = [ - "arguments.py", - "namespace_pkgs.py", - "wheel.py", - "wheel_installer.py", - ], - visibility = ["//third_party/rules_pycross/pycross/private:__subpackages__"], - deps = [ - requirement("installer"), - requirement("pip"), - requirement("packaging"), - requirement("setuptools"), - ], -) - -py_binary( - name = "wheel_installer", - srcs = [ - "wheel_installer.py", - ], - deps = [":lib"], -) - -py_test( - name = "arguments_test", - size = "small", - srcs = [ - "arguments_test.py", - ], - deps = [ - ":lib", - ], -) - -py_test( - name = "namespace_pkgs_test", - size = "small", - srcs = [ - "namespace_pkgs_test.py", - ], - deps = [ - ":lib", - ], -) - -py_test( - name = "wheel_test", - size = "small", - srcs = [ - "wheel_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) - -py_test( - name = "wheel_installer_test", - size = "small", - srcs = [ - "wheel_installer_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 422ed9c7c2..c5da4e31e3 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -13,9 +13,11 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_setting") load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") load("//python:versions.bzl", "print_toolchains_checksums") +load(":py_exec_tools_toolchain.bzl", "current_interpreter_executable") load(":stamp.bzl", "stamp_build_setting") package( @@ -27,9 +29,9 @@ licenses(["notice"]) filegroup( name = "distribution", srcs = glob(["**"]) + [ - "//python/private/bzlmod:distribution", "//python/private/common:distribution", "//python/private/proto:distribution", + "//python/private/pypi:distribution", "//python/private/whl_filegroup:distribution", "//tools/build_defs/python/private:distribution", ], @@ -56,9 +58,11 @@ bzl_library( ) bzl_library( - name = "autodetecting_toolchain_bzl", - srcs = ["autodetecting_toolchain.bzl"], + name = "runtime_env_toolchain_bzl", + srcs = ["runtime_env_toolchain.bzl"], deps = [ + ":py_exec_tools_toolchain_bzl", + ":toolchain_types_bzl", "//python:py_runtime_bzl", "//python:py_runtime_pair_bzl", ], @@ -124,61 +128,27 @@ bzl_library( ) bzl_library( - name = "patch_whl_bzl", - srcs = ["patch_whl.bzl"], - deps = [":parse_whl_name_bzl"], -) - -bzl_library( - name = "parse_requirements_bzl", - srcs = ["parse_requirements.bzl"], - deps = [ - ":normalize_name_bzl", - ":pypi_index_sources_bzl", - ":whl_target_platforms_bzl", - "//python/pip_install:requirements_parser_bzl", - ], -) - -bzl_library( - name = "parse_whl_name_bzl", - srcs = ["parse_whl_name.bzl"], -) - -bzl_library( - name = "pip_flags_bzl", - srcs = ["pip_flags.bzl"], + name = "python_bzl", + srcs = ["python.bzl"], deps = [ - ":enum_bzl", - ], -) - -bzl_library( - name = "pip_repo_name_bzl", - srcs = ["pip_repo_name.bzl"], - deps = [ - ":normalize_name_bzl", - ":parse_whl_name_bzl", + ":pythons_hub_bzl", + ":toolchains_repo_bzl", + ":util_bzl", + "//python:repositories_bzl", + "@bazel_features//:features", ], ) bzl_library( - name = "pypi_index_bzl", - srcs = ["pypi_index.bzl"], + name = "pythons_hub_bzl", + srcs = ["pythons_hub.bzl"], deps = [ - ":auth_bzl", - ":normalize_name_bzl", - ":text_util_bzl", - "//python/pip_install:requirements_parser_bzl", - "//python/private/bzlmod:bazel_features_bzl", + ":full_version_bzl", + ":py_toolchain_suite_bzl", + "//python:versions_bzl", ], ) -bzl_library( - name = "pypi_index_sources_bzl", - srcs = ["pypi_index_sources.bzl"], -) - bzl_library( name = "py_cc_toolchain_bzl", srcs = [ @@ -193,6 +163,7 @@ bzl_library( ":py_cc_toolchain_info_bzl", ":rules_cc_srcs_bzl", ":util_bzl", + "@bazel_skylib//rules:common_settings", ], ) @@ -217,7 +188,11 @@ bzl_library( bzl_library( name = "py_exec_tools_toolchain_bzl", srcs = ["py_exec_tools_toolchain.bzl"], - deps = ["//python/private/common:providers_bzl"], + deps = [ + ":toolchain_types_bzl", + "//python/private/common:providers_bzl", + "@bazel_skylib//rules:common_settings", + ], ) bzl_library( @@ -245,6 +220,7 @@ bzl_library( deps = [ "//python:py_runtime_bzl", "//python:py_runtime_info_bzl", + "@bazel_skylib//rules:common_settings", ], ) @@ -283,17 +259,6 @@ bzl_library( srcs = ["register_extension_info.bzl"], ) -bzl_library( - name = "render_pkg_aliases_bzl", - srcs = ["render_pkg_aliases.bzl"], - deps = [ - ":normalize_name_bzl", - ":text_util_bzl", - ":version_label_bzl", - "//python/pip_install/private:generate_group_library_build_bazel_bzl", - ], -) - bzl_library( name = "repo_utils_bzl", srcs = ["repo_utils.bzl"], @@ -338,20 +303,6 @@ bzl_library( srcs = ["version_label.bzl"], ) -bzl_library( - name = "whl_target_platforms_bzl", - srcs = ["whl_target_platforms.bzl"], - visibility = ["//:__subpackages__"], - deps = [ - "parse_whl_name_bzl", - ], -) - -bzl_library( - name = "labels_bzl", - srcs = ["labels.bzl"], -) - # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", @@ -443,6 +394,15 @@ config_setting( }, ) +# This should only be set by analysis tests to expose additional metadata to +# aid testing, so a setting instead of a flag. +bool_setting( + name = "visible_for_testing", + build_setting_default = False, + # This is only because it is an implicit dependency by the toolchains. + visibility = ["//visibility:public"], +) + print_toolchains_checksums(name = "print_toolchains_checksums") # Used for py_console_script_gen rule @@ -463,3 +423,12 @@ py_library( "//tests/entry_points:__pkg__", ], ) + +# The current toolchain's interpreter as an excutable, usable with +# executable=True attributes. +current_interpreter_executable( + name = "current_interpreter_executable", + # Not actually public. Only public because it's an implicit dependency of + # py_exec_tools_toolchain. + visibility = ["//visibility:public"], +) diff --git a/python/private/autodetecting_toolchain.bzl b/python/private/autodetecting_toolchain.bzl deleted file mode 100644 index 55c95699c9..0000000000 --- a/python/private/autodetecting_toolchain.bzl +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Definitions related to the Python toolchain.""" - -load("//python:py_runtime.bzl", "py_runtime") -load("//python:py_runtime_pair.bzl", "py_runtime_pair") - -def define_autodetecting_toolchain(name): - """Defines the autodetecting Python toolchain. - - Args: - name: The name of the toolchain to introduce. Must have value - "autodetecting_toolchain". This param is present only to make the - BUILD file more readable. - """ - if name != "autodetecting_toolchain": - fail("Python autodetecting toolchain must be named " + - "'autodetecting_toolchain'") - - # buildifier: disable=native-py - py_runtime( - name = "_autodetecting_py3_runtime", - interpreter = "//python/private:autodetecting_toolchain_interpreter.sh", - python_version = "PY3", - stub_shebang = "#!/usr/bin/env python3", - visibility = ["//visibility:private"], - ) - - # This is a dummy runtime whose interpreter_path triggers the native rule - # logic to use the legacy behavior on Windows. - # TODO(#7844): Remove this target. - # buildifier: disable=native-py - py_runtime( - name = "_magic_sentinel_runtime", - interpreter_path = "/_magic_pyruntime_sentinel_do_not_use", - python_version = "PY3", - visibility = ["//visibility:private"], - ) - - py_runtime_pair( - name = "_autodetecting_py_runtime_pair", - py3_runtime = select({ - # If we're on windows, inject the sentinel to tell native rule logic - # that we attempted to use the autodetecting toolchain and need to - # switch back to legacy behavior. - # TODO(#7844): Remove this hack. - "@platforms//os:windows": ":_magic_sentinel_runtime", - "//conditions:default": ":_autodetecting_py3_runtime", - }), - visibility = ["//visibility:public"], - ) - - native.toolchain( - name = name, - toolchain = ":_autodetecting_py_runtime_pair", - toolchain_type = ":toolchain_type", - visibility = ["//visibility:public"], - ) diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel deleted file mode 100644 index 3362f34ffd..0000000000 --- a/python/private/bzlmod/BUILD.bazel +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2022 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") - -package(default_visibility = ["//:__subpackages__"]) - -licenses(["notice"]) - -filegroup( - name = "distribution", - srcs = glob(["**"]), - visibility = ["//python/private:__pkg__"], -) - -bzl_library( - name = "pip_bzl", - srcs = ["pip.bzl"], - deps = [ - ":pip_repository_bzl", - "//python/pip_install:pip_repository_bzl", - "//python/private:pypi_index_bzl", - "//python/private:full_version_bzl", - "//python/private:normalize_name_bzl", - "//python/private:parse_requirements_bzl", - "//python/private:parse_whl_name_bzl", - "//python/private:pip_repo_name_bzl", - "//python/private:version_label_bzl", - ":bazel_features_bzl", - ] + [ - "@pythons_hub//:interpreters_bzl", - ] if BZLMOD_ENABLED else [], -) - -bzl_library( - name = "bazel_features_bzl", - deps = ["@bazel_features//:features"], -) - -bzl_library( - name = "pip_repository_bzl", - srcs = ["pip_repository.bzl"], - visibility = ["//:__subpackages__"], - deps = [ - "//python/private:render_pkg_aliases_bzl", - "//python/private:text_util_bzl", - ], -) - -bzl_library( - name = "python_bzl", - srcs = ["python.bzl"], - deps = [ - ":pythons_hub_bzl", - "//python:repositories_bzl", - "//python/private:toolchains_repo_bzl", - ], -) - -bzl_library( - name = "pythons_hub_bzl", - srcs = ["pythons_hub.bzl"], - deps = [ - "//python:versions_bzl", - "//python/private:full_version_bzl", - "//python/private:py_toolchain_suite_bzl", - ], -) diff --git a/python/private/common/BUILD.bazel b/python/private/common/BUILD.bazel index e258f8a5eb..a415e0587e 100644 --- a/python/private/common/BUILD.bazel +++ b/python/private/common/BUILD.bazel @@ -35,6 +35,7 @@ bzl_library( "//python/private:enum_bzl", "//python/private:flags_bzl", "//python/private:reexports_bzl", + "//python/private:rules_cc_srcs_bzl", "@bazel_skylib//rules:common_settings", ], ) diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl index eb70055787..503578b78c 100644 --- a/python/private/common/attributes.bzl +++ b/python/private/common/attributes.bzl @@ -335,7 +335,7 @@ in the resulting output or not. Valid values are: * `inherit`: Inherit the value from the `--precompile_source_retention` flag. * `keep_source`: Include the original Python source. -* `omit_source`: Don't include the orignal py source. +* `omit_source`: Don't include the original py source. * `omit_if_generated_source`: Keep the original source if it's a regular source file, but omit it if it's a generated file. """, @@ -361,6 +361,9 @@ in the resulting output or not. Valid values are: default = "//python/config_settings:precompile_source_retention", providers = [BuildSettingInfo], ), + # Force enabling auto exec groups, see + # https://bazel.build/extending/auto-exec-groups#how-enable-particular-rule + "_use_auto_exec_groups": attr.bool(default = True), }, allow_none = True, ) diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index 0ac9187b79..5559ccd195 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -153,7 +153,9 @@ def create_cc_details_struct( cc_info_for_self_link, cc_info_with_extra_link_time_libraries, extra_runfiles, - cc_toolchain): + cc_toolchain, + feature_config, + **kwargs): """Creates a CcDetails struct. Args: @@ -170,6 +172,12 @@ def create_cc_details_struct( part of `cc_info_with_extra_link_time_libraries`; should be added to runfiles. cc_toolchain: CcToolchain that should be used when building. + feature_config: struct from cc_configure_features(); see + //python/private/common:py_executable.bzl%cc_configure_features. + **kwargs: Additional keys/values to set in the returned struct. This is to + facilitate extensions with less patching. Any added fields should + pick names that are unlikely to collide if the CcDetails API has + additional fields added. Returns: A `CcDetails` struct. @@ -180,6 +188,8 @@ def create_cc_details_struct( cc_info_with_extra_link_time_libraries = cc_info_with_extra_link_time_libraries, extra_runfiles = extra_runfiles, cc_toolchain = cc_toolchain, + feature_config = feature_config, + **kwargs ) def create_executable_result_struct(*, extra_files_to_build, output_groups, extra_runfiles = None): diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl index e1876ff9d3..eb8b910a2e 100644 --- a/python/private/common/providers.bzl +++ b/python/private/common/providers.bzl @@ -41,13 +41,13 @@ def interpreter_version_info_struct_from_dict(info_dict): """Create a struct of interpreter version info from a dict from an attribute. Args: - info_dict: dict of versio info fields. See interpreter_version_info + info_dict: (dict | None) of version info fields. See interpreter_version_info provider field docs. Returns: struct of version info; see interpreter_version_info provider field docs. """ - info_dict = dict(info_dict) # Copy in case the original is frozen + info_dict = dict(info_dict or {}) # Copy in case the original is frozen if info_dict: if not ("major" in info_dict and "minor" in info_dict): fail("interpreter_version_info must have at least two keys, 'major' and 'minor'") diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index ff1f74de99..2b4a9397c8 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -166,9 +166,10 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = main_py = precompile_result.py_to_pyc_map[main_py] direct_pyc_files = depset(precompile_result.pyc_files) - default_outputs = precompile_result.keep_srcs + precompile_result.pyc_files executable = _declare_executable_file(ctx) - default_outputs.append(executable) + default_outputs = [executable] + default_outputs.extend(precompile_result.keep_srcs) + default_outputs.extend(precompile_result.pyc_files) imports = collect_imports(ctx, semantics) @@ -243,6 +244,9 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = def _get_build_info(ctx, cc_toolchain): build_info_files = py_internal.cc_toolchain_build_info_files(cc_toolchain) if cc_helper.is_stamping_enabled(ctx): + # Makes the target depend on BUILD_INFO_KEY, which helps to discover stamped targets + # See b/326620485 for more details. + ctx.version_file # buildifier: disable=no-effect return build_info_files.non_redacted_build_info_files.to_list() else: return build_info_files.redacted_build_info_files.to_list() @@ -551,15 +555,7 @@ def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) share_native_deps = py_internal.share_native_deps(ctx) - cc_feature_config = cc_configure_features( - ctx, - cc_toolchain = cc_details.cc_toolchain, - # See b/171276569#comment18: this feature string is just to allow - # Google's RBE to know the link action is for the Python case so it can - # take special actions (though as of Jun 2022, no special action is - # taken). - extra_features = ["native_deps_link"], - ) + cc_feature_config = cc_details.feature_config if share_native_deps: linked_lib = _create_shared_native_deps_dso( ctx, @@ -918,7 +914,12 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs): **kwargs ) -def cc_configure_features(ctx, *, cc_toolchain, extra_features): +def cc_configure_features( + ctx, + *, + cc_toolchain, + extra_features, + linking_mode = "static_linking_mode"): """Configure C++ features for Python purposes. Args: @@ -926,11 +927,14 @@ def cc_configure_features(ctx, *, cc_toolchain, extra_features): cc_toolchain: The CcToolchain the target is using. extra_features: list of strings; additional features to request be enabled. + linking_mode: str; either "static_linking_mode" or + "dynamic_linking_mode". Specifies the linking mode feature for + C++ linking. Returns: struct of the feature configuration and all requested features. """ - requested_features = ["static_linking_mode"] + requested_features = [linking_mode] requested_features.extend(extra_features) requested_features.extend(ctx.features) if "legacy_whole_archive" not in ctx.disabled_features: diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index 361e075a39..a0cfebad8a 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -560,6 +560,7 @@ def _get_cc_details_for_binary(ctx, extra_deps): extra_runfiles = ctx.runfiles(), # Though the rules require the CcToolchain, it isn't actually used. cc_toolchain = None, + feature_config = None, ) def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl index b927a4fb9a..673beedd2a 100644 --- a/python/private/common/py_library.bzl +++ b/python/private/common/py_library.bzl @@ -64,6 +64,7 @@ def py_library_impl(ctx, *, semantics): """ check_native_allowed(ctx) direct_sources = filter_to_py_srcs(ctx.files.srcs) + precompile_result = semantics.maybe_precompile(ctx, direct_sources) direct_pyc_files = depset(precompile_result.pyc_files) default_outputs = depset(precompile_result.keep_srcs, transitive = [direct_pyc_files]) @@ -102,6 +103,18 @@ def py_library_impl(ctx, *, semantics): create_output_group_info(py_info.transitive_sources, extra_groups = {}), ] +_DEFAULT_PY_LIBRARY_DOC = """ +A library of Python code that can be depended upon. + +Default outputs: +* The input Python sources +* The precompiled artifacts from the sources. + +NOTE: Precompilation affects which of the default outputs are included in the +resulting runfiles. See the precompile-related attributes and flags for +more information. +""" + def create_py_library_rule(*, attrs = {}, **kwargs): """Creates a py_library rule. @@ -111,6 +124,9 @@ def create_py_library_rule(*, attrs = {}, **kwargs): Returns: A rule object """ + + # Within Google, the doc attribute is overridden + kwargs.setdefault("doc", _DEFAULT_PY_LIBRARY_DOC) return rule( attrs = dicts.add(LIBRARY_ATTRS, attrs), toolchains = [ @@ -120,16 +136,5 @@ def create_py_library_rule(*, attrs = {}, **kwargs): # TODO(b/253818097): fragments=py is only necessary so that # RequiredConfigFragmentsTest passes fragments = ["py"], - doc = """ -A library of Python code that can be depended upon. - -Default outputs: -* The input Python sources -* The precompiled artifacts from the sources. - -NOTE: Precompilation affects which of the default outputs are included in the -resulting runfiles. See the precompile-related attributes and flags for -more information. -""", **kwargs ) diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 92b96b3264..0537655a47 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -181,6 +181,14 @@ def construct_config_settings(name = None): # buildifier: disable=function-docs visibility = ["//visibility:public"], ) + native.config_setting( + name = "is_python_version_unset", + flag_values = { + Label("//python/config_settings:python_version"): "", + }, + visibility = ["//visibility:public"], + ) + for version, matching_versions in VERSION_FLAG_VALUES.items(): is_python_config_setting( name = "is_python_{}".format(version), diff --git a/python/private/flags.bzl b/python/private/flags.bzl index d141f72eee..fa31262c94 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -37,6 +37,15 @@ def _precompile_flag_get_effective_value(ctx): value = PrecompileFlag.DISABLED return value +# Determines if the Python exec tools toolchain should be registered. +# buildifier: disable=name-conventions +ExecToolsToolchainFlag = enum( + # Enable registering the exec tools toolchain using the hermetic toolchain. + ENABLED = "enabled", + # Disable registering the exec tools toolchain using the hermetic toolchain. + DISABLED = "disabled", +) + # Determines if Python source files should be compiled at build time. # # NOTE: The flag value is overridden by the target-level attribute, except diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index 7be34aef8f..c37bc351cc 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -92,6 +92,7 @@ def _internal_config_repo_impl(rctx): internal_config_repo = repository_rule( implementation = _internal_config_repo_impl, + configure = True, environ = [_ENABLE_PYSTAR_ENVVAR_NAME], ) diff --git a/python/private/bzlmod/internal_deps.bzl b/python/private/internal_deps.bzl similarity index 79% rename from python/private/bzlmod/internal_deps.bzl rename to python/private/internal_deps.bzl index 62ca71fecc..6ea3fa40c7 100644 --- a/python/private/bzlmod/internal_deps.bzl +++ b/python/private/internal_deps.bzl @@ -9,12 +9,12 @@ "Python toolchain module extension for internal rule use" load("@bazel_skylib//lib:modules.bzl", "modules") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") -load("//python/private:internal_config_repo.bzl", "internal_config_repo") +load("//python/private/pypi:deps.bzl", "pypi_deps") +load(":internal_config_repo.bzl", "internal_config_repo") def _internal_deps(): internal_config_repo(name = "rules_python_internal") - pip_install_dependencies() + pypi_deps() internal_deps = modules.as_extension( _internal_deps, diff --git a/python/private/normalize_name.bzl b/python/private/normalize_name.bzl index aaeca803b9..7898222e41 100644 --- a/python/private/normalize_name.bzl +++ b/python/private/normalize_name.bzl @@ -38,7 +38,6 @@ Also see Python spec on normalizing package names: https://packaging.python.org/en/latest/specifications/name-normalization/ """ -# Keep in sync with ../pip_install/tools/lib/bazel.py def normalize_name(name): """normalize a PyPI package name and return a valid bazel label. diff --git a/python/private/py_cc_toolchain_rule.bzl b/python/private/py_cc_toolchain_rule.bzl index 5d3debb702..1599415ac7 100644 --- a/python/private/py_cc_toolchain_rule.bzl +++ b/python/private/py_cc_toolchain_rule.bzl @@ -18,6 +18,7 @@ NOTE: This is a beta-quality feature. APIs subject to change until https://github.com/bazelbuild/rules_python/issues/824 is considered done. """ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc:defs.bzl", "CcInfo") load(":py_cc_toolchain_info.bzl", "PyCcToolchainInfo") @@ -37,8 +38,12 @@ def _py_cc_toolchain_impl(ctx): ), python_version = ctx.attr.python_version, ) + extra_kwargs = {} + if ctx.attr._visible_for_testing[BuildSettingInfo].value: + extra_kwargs["toolchain_label"] = ctx.label return [platform_common.ToolchainInfo( py_cc_toolchain = py_cc_toolchain, + **extra_kwargs )] py_cc_toolchain = rule( @@ -60,6 +65,9 @@ py_cc_toolchain = rule( doc = "The Major.minor Python version, e.g. 3.11", mandatory = True, ), + "_visible_for_testing": attr.label( + default = "//python/private:visible_for_testing", + ), }, doc = """\ A toolchain for a Python runtime's C/C++ information (e.g. headers) diff --git a/python/private/py_exec_tools_info.bzl b/python/private/py_exec_tools_info.bzl index 3011f531c8..2998543102 100644 --- a/python/private/py_exec_tools_info.bzl +++ b/python/private/py_exec_tools_info.bzl @@ -31,12 +31,6 @@ to setup the runtime environment for the binary. See also: NOTE: What interpreter is used depends on the toolchain constraints. Ensure the proper target constraints are being applied when obtaining this from the toolchain. -""", - "exec_interpreter_version_info": """ -struct of interpreter version info for `exec_interpreter`. Note this -is for the exec interpreter, not the target interpreter. For version information -about the target Python runtime, use the `//python:toolchain_type` toolchain -information. """, "precompiler": """ Optional Target. The tool to use for generating pyc files. If not available, diff --git a/python/private/py_exec_tools_toolchain.bzl b/python/private/py_exec_tools_toolchain.bzl index 6036db4e7c..b3d0fb2634 100644 --- a/python/private/py_exec_tools_toolchain.bzl +++ b/python/private/py_exec_tools_toolchain.bzl @@ -14,34 +14,61 @@ """Rule that defines a toolchain for build tools.""" -load("//python/private/common:providers.bzl", "interpreter_version_info_struct_from_dict") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") load(":py_exec_tools_info.bzl", "PyExecToolsInfo") def _py_exec_tools_toolchain_impl(ctx): - return [platform_common.ToolchainInfo(exec_tools = PyExecToolsInfo( - exec_interpreter = ctx.attr.exec_interpreter, - exec_interpreter_version_info = interpreter_version_info_struct_from_dict( - ctx.attr.exec_interpreter_version_info, + extra_kwargs = {} + if ctx.attr._visible_for_testing[BuildSettingInfo].value: + extra_kwargs["toolchain_label"] = ctx.label + + return [platform_common.ToolchainInfo( + exec_tools = PyExecToolsInfo( + exec_interpreter = ctx.attr.exec_interpreter, + precompiler = ctx.attr.precompiler, ), - precompiler = ctx.attr.precompiler, - ))] + **extra_kwargs + )] py_exec_tools_toolchain = rule( implementation = _py_exec_tools_toolchain_impl, attrs = { "exec_interpreter": attr.label( + default = "//python/private:current_interpreter_executable", cfg = "exec", - allow_files = True, - doc = "See PyExecToolsInfo.exec_interpreter", - executable = True, - ), - "exec_interpreter_version_info": attr.string_dict( - doc = "See PyExecToolsInfo.exec_interpreter_version_info", + doc = "See PyexecToolsInfo.exec_interpreter.", ), "precompiler": attr.label( allow_files = True, cfg = "exec", doc = "See PyExecToolsInfo.precompiler", ), + "_visible_for_testing": attr.label( + default = "//python/private:visible_for_testing", + ), }, ) + +def _current_interpreter_executable_impl(ctx): + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + if runtime.interpreter: + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) + else: + executable = ctx.actions.declare_symlink(ctx.label.name) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + return [ + toolchain, + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files), + ), + ] + +current_interpreter_executable = rule( + implementation = _current_interpreter_executable_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, +) diff --git a/python/private/py_interpreter_program.bzl b/python/private/py_interpreter_program.bzl index 833ea53b1c..cd62a7190d 100644 --- a/python/private/py_interpreter_program.bzl +++ b/python/private/py_interpreter_program.bzl @@ -57,7 +57,7 @@ def _py_interpreter_program_impl(ctx): py_interpreter_program = rule( doc = """ -Binary-like rule that doesn't require a toolchain becaues its part of +Binary-like rule that doesn't require a toolchain because its part of implementing build tools for the toolchain. This rule expects the Python interprter to be externally provided. diff --git a/python/private/py_runtime_pair_rule.bzl b/python/private/py_runtime_pair_rule.bzl index 02f9a5ba89..eb91413563 100644 --- a/python/private/py_runtime_pair_rule.bzl +++ b/python/private/py_runtime_pair_rule.bzl @@ -14,6 +14,7 @@ """Implementation of py_runtime_pair.""" +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python:py_runtime_info.bzl", "PyRuntimeInfo") load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo") load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") @@ -40,9 +41,14 @@ def _py_runtime_pair_impl(ctx): # fail("Using Python 2 is not supported and disabled; see " + # "https://github.com/bazelbuild/bazel/issues/15684") + extra_kwargs = {} + if ctx.attr._visible_for_testing[BuildSettingInfo].value: + extra_kwargs["toolchain_label"] = ctx.label + return [platform_common.ToolchainInfo( py2_runtime = py2_runtime, py3_runtime = py3_runtime, + **extra_kwargs )] def _get_py_runtime_info(target): @@ -85,6 +91,9 @@ The runtime to use for Python 3 targets. Must have `python_version` set to `PY3`. """, ), + "_visible_for_testing": attr.label( + default = "//python/private:visible_for_testing", + ), }, fragments = ["py"], doc = """\ diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl index 174c36f782..564e0ee45d 100644 --- a/python/private/py_toolchain_suite.bzl +++ b/python/private/py_toolchain_suite.bzl @@ -22,6 +22,8 @@ load( "TARGET_TOOLCHAIN_TYPE", ) +_IS_EXEC_TOOLCHAIN_ENABLED = Label("//python/config_settings:is_exec_tools_toolchain_enabled") + def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, flag_values, **kwargs): """For internal use only. @@ -36,7 +38,7 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth """ # We have to use a String value here because bzlmod is passing in a - # string as we cannot have list of bools in build rule attribues. + # string as we cannot have list of bools in build rule attributes. # This if statement does not appear to work unless it is in the # toolchain file. if set_python_version_constraint in ["True", "False"]: @@ -106,8 +108,16 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth user_repository_name = user_repository_name, ), toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, - # The target settings capture the Python version - target_settings = target_settings, + target_settings = select({ + _IS_EXEC_TOOLCHAIN_ENABLED: target_settings, + # Whatever the default is, it has to map to a `config_setting` + # that will never match. Since the default branch is only taken if + # _IS_EXEC_TOOLCHAIN_ENABLED is false, then it will never match + # when later evaluated during toolchain resolution. + # Note that @platforms//:incompatible can't be used here because + # the RHS must be a `config_setting`. + "//conditions:default": [_IS_EXEC_TOOLCHAIN_ENABLED], + }), exec_compatible_with = kwargs.get("target_compatible_with"), ) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel new file mode 100644 index 0000000000..08fb7259ec --- /dev/null +++ b/python/private/pypi/BUILD.bazel @@ -0,0 +1,296 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +filegroup( + name = "distribution", + srcs = glob( + ["**"], + exclude = ["requirements.txt"], + ) + [ + "//python/private/pypi/dependency_resolver:distribution", + "//python/private/pypi/whl_installer:distribution", + ], + visibility = ["//python/private:__pkg__"], +) + +# Filegroup of bzl files that can be used by downstream rules for documentation generation +filegroup( + name = "bzl", + srcs = glob(["**/*.bzl"]), + visibility = [ + "//python/private:__pkg__", + "//tools/private:__pkg__", + ], +) + +filegroup( + name = "requirements_txt", + srcs = ["requirements.txt"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + +# Keep sorted by library name and keep the files named by the main symbol they export + +bzl_library( + name = "attrs_bzl", + srcs = ["attrs.bzl"], +) + +bzl_library( + name = "extension_bzl", + srcs = ["extension.bzl"], + deps = [ + ":attrs_bzl", + ":hub_repository_bzl", + ":parse_requirements_bzl", + ":parse_whl_name_bzl", + ":pip_repository_attrs_bzl", + ":simpleapi_download_bzl", + ":whl_library_bzl", + ":whl_repo_name_bzl", + "//python/private:full_version_bzl", + "//python/private:normalize_name_bzl", + "//python/private:version_label_bzl", + "@bazel_features//:features", + ] + [ + "@pythons_hub//:interpreters_bzl", + ] if BZLMOD_ENABLED else [], +) + +bzl_library( + name = "config_settings_bzl", + srcs = ["config_settings.bzl"], + deps = ["flags_bzl"], +) + +bzl_library( + name = "deps_bzl", + srcs = ["deps.bzl"], + deps = [ + "//python/private:bazel_tools_bzl", + ], +) + +bzl_library( + name = "flags_bzl", + srcs = ["flags.bzl"], + deps = [ + "//python/private:enum_bzl", + "@bazel_skylib//rules:common_settings", + ], +) + +bzl_library( + name = "generate_whl_library_build_bazel_bzl", + srcs = ["generate_whl_library_build_bazel.bzl"], + deps = [ + ":labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "generate_group_library_build_bazel_bzl", + srcs = ["generate_group_library_build_bazel.bzl"], + deps = [ + ":labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "group_library_bzl", + srcs = ["group_library.bzl"], + deps = [ + ":generate_group_library_build_bazel_bzl", + ], +) + +bzl_library( + name = "hub_repository_bzl", + srcs = ["hub_repository.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + ":render_pkg_aliases_bzl", + "//python/private:text_util_bzl", + ], +) + +bzl_library( + name = "index_sources_bzl", + srcs = ["index_sources.bzl"], +) + +bzl_library( + name = "labels_bzl", + srcs = ["labels.bzl"], +) + +bzl_library( + name = "multi_pip_parse_bzl", + srcs = ["multi_pip_parse.bzl"], + deps = ["pip_repository_bzl"], +) + +bzl_library( + name = "package_annotation_bzl", + srcs = ["package_annotation.bzl"], +) + +bzl_library( + name = "parse_requirements_bzl", + srcs = ["parse_requirements.bzl"], + deps = [ + ":index_sources_bzl", + ":parse_requirements_txt_bzl", + ":whl_target_platforms_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "parse_requirements_txt_bzl", + srcs = ["parse_requirements_txt.bzl"], +) + +bzl_library( + name = "parse_simpleapi_html_bzl", + srcs = ["parse_simpleapi_html.bzl"], +) + +bzl_library( + name = "parse_whl_name_bzl", + srcs = ["parse_whl_name.bzl"], +) + +bzl_library( + name = "patch_whl_bzl", + srcs = ["patch_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + "//python/private:repo_utils_bzl", + ], +) + +bzl_library( + name = "pip_bzl", + srcs = ["pip.bzl"], + deps = [ + ":extension_bzl", + ], +) + +bzl_library( + name = "pip_compile_bzl", + srcs = ["pip_compile.bzl"], + deps = [ + ":deps_bzl", + "//python:defs_bzl", + ], +) + +bzl_library( + name = "pip_repository_bzl", + srcs = ["pip_repository.bzl"], + deps = [ + ":attrs_bzl", + ":parse_requirements_bzl", + ":pip_repository_attrs_bzl", + ":render_pkg_aliases_bzl", + "//python/private:normalize_name_bzl", + "//python/private:repo_utils_bzl", + "//python/private:text_util_bzl", + "@bazel_skylib//lib:sets", + ], +) + +bzl_library( + name = "pip_repository_attrs_bzl", + srcs = ["pip_repository_attrs.bzl"], +) + +bzl_library( + name = "render_pkg_aliases_bzl", + srcs = ["render_pkg_aliases.bzl"], + deps = [ + ":generate_group_library_build_bazel_bzl", + ":labels_bzl", + ":parse_whl_name_bzl", + ":whl_target_platforms_bzl", + "//python/private:normalize_name_bzl", + "//python/private:text_util_bzl", + ], +) + +bzl_library( + name = "simpleapi_download_bzl", + srcs = ["simpleapi_download.bzl"], + deps = [ + ":parse_simpleapi_html_bzl", + "//python/private:auth_bzl", + "//python/private:normalize_name_bzl", + "//python/private:text_util_bzl", + "@bazel_features//:features", + ], +) + +bzl_library( + name = "whl_library_alias_bzl", + srcs = ["whl_library_alias.bzl"], + deps = [ + ":render_pkg_aliases_bzl", + "//python/private:full_version_bzl", + ], +) + +bzl_library( + name = "whl_library_bzl", + srcs = ["whl_library.bzl"], + deps = [ + ":attrs_bzl", + ":deps_bzl", + ":generate_whl_library_build_bazel_bzl", + ":parse_whl_name_bzl", + ":patch_whl_bzl", + ":whl_target_platforms_bzl", + "//python:repositories_bzl", + "//python:versions_bzl", + "//python/private:auth_bzl", + "//python/private:envsubst_bzl", + "//python/private:repo_utils_bzl", + "//python/private:toolchains_repo_bzl", + ], +) + +bzl_library( + name = "whl_repo_name_bzl", + srcs = ["whl_repo_name.bzl"], + deps = [ + ":parse_whl_name_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "whl_target_platforms_bzl", + srcs = ["whl_target_platforms.bzl"], + deps = [":parse_whl_name_bzl"], +) diff --git a/python/private/pypi/README.md b/python/private/pypi/README.md new file mode 100644 index 0000000000..6be5703912 --- /dev/null +++ b/python/private/pypi/README.md @@ -0,0 +1,9 @@ +# PyPI integration code + +This code is for integrating with PyPI and other compatible indexes. At the +moment we have code for: +* Downloading packages using `pip` or `repository_ctx.download`. +* Interacting with PyPI compatible indexes via [SimpleAPI] spec. +* Locking a `requirements.in` or [PEP621] compliant `pyproject.toml`. + +[PEP621]: https://peps.python.org/pep-0621/ diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl new file mode 100644 index 0000000000..79ffea54a1 --- /dev/null +++ b/python/private/pypi/attrs.bzl @@ -0,0 +1,224 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"common attributes for whl_library and pip_repository" + +ATTRS = { + "download_only": attr.bool( + doc = """ +Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of +--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different +platform from the host platform. + """, + ), + "enable_implicit_namespace_pkgs": attr.bool( + default = False, + doc = """ +If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary +and py_test targets must specify either `legacy_create_init=False` or the global Bazel option +`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. + +This option is required to support some packages which cannot handle the conversion to pkg-util style. + """, + ), + "environment": attr.string_dict( + doc = """ +Environment variables to set in the pip subprocess. +Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy` +Note that pip is run with "--isolated" on the CLI so `PIP__` +style env vars are ignored, but env vars that control requests and urllib3 +can be passed. If you need `PIP__`, take a look at `extra_pip_args` +and `envsubst`. + """, + default = {}, + ), + "envsubst": attr.string_list( + mandatory = False, + doc = """\ +A list of environment variables to substitute (e.g. `["PIP_INDEX_URL", +"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args` +using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset) +or `${VARNAME:-default}` (expanding to default if the variable is unset or empty +in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named +here do not cause packages to be re-fetched. Don't fetch different things based +on the value of these variables. +""", + ), + "experimental_requirement_cycles": attr.string_list_dict( + default = {}, + doc = """\ +A mapping of dependency cycle names to a list of requirements which form that cycle. + +Requirements which form cycles will be installed together and taken as +dependencies together in order to ensure that the cycle is always satisified. + +Example: + `sphinx` depends on `sphinxcontrib-serializinghtml` + When listing both as requirements, ala + + ``` + py_binary( + name = "doctool", + ... + deps = [ + "@pypi//sphinx:pkg", + "@pypi//sphinxcontrib_serializinghtml", + ] + ) + ``` + + Will produce a Bazel error such as + + ``` + ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: + //:doctool (...) + @pypi//sphinxcontrib_serializinghtml:pkg (...) + .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) + | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) + | @pypi_sphinx//:pkg (...) + | @pypi_sphinx//:_pkg (...) + `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) + ``` + + Which we can resolve by configuring these two requirements to be installed together as a cycle + + ``` + pip_parse( + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, + ) + ``` + +Warning: + If a dependency participates in multiple cycles, all of those cycles must be + collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed + as two separate cycles. +""", + ), + "experimental_target_platforms": attr.string_list( + default = [], + doc = """\ +A list of platforms that we will generate the conditional dependency graph for +cross platform wheels by parsing the wheel metadata. This will generate the +correct dependencies for packages like `sphinx` or `pylint`, which include +`colorama` when installed and used on Windows platforms. + +An empty list means falling back to the legacy behaviour where the host +platform is the target platform. + +WARNING: It may not work as expected in cases where the python interpreter +implementation that is being used at runtime is different between different platforms. +This has been tested for CPython only. + +For specific target platforms use values of the form `_` where `` +is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, +`aarch64`, `s390x` and `ppc64le`. + +You can also target a specific Python version by using `cp3__`. +If multiple python versions are specified as target platforms, then select statements +of the `lib` and `whl` targets will include usage of version aware toolchain config +settings like `@rules_python//python/config_settings:is_python_3.y`. + +Special values: `host` (for generating deps for the host platform only) and +`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`. + +NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. +""", + ), + "extra_pip_args": attr.string_list( + doc = """Extra arguments to pass on to pip. Must not contain spaces. + +Supports environment variables using the syntax `$VARNAME` or +`${VARNAME}` (expanding to empty string if unset) or +`${VARNAME:-default}` (expanding to default if the variable is unset +or empty in the environment), if `"VARNAME"` is listed in the +`envsubst` attribute. See also `envsubst`. +""", + ), + "isolated": attr.bool( + doc = """\ +Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to +the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used +to control this flag. +""", + default = True, + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", + ), + "python_interpreter": attr.string( + doc = """\ +The python interpreter to use. This can either be an absolute path or the name +of a binary found on the host's `PATH` environment variable. If no value is set +`python3` is defaulted for Unix systems and `python.exe` for Windows. +""", + # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr` + # default = "python3" + ), + "python_interpreter_target": attr.label( + allow_single_file = True, + doc = """ +If you are using a custom python interpreter built by another repository rule, +use this attribute to specify its BUILD target. This allows pip_repository to invoke +pip using the same interpreter as your toolchain. If set, takes precedence over +python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python". +""", + ), + "quiet": attr.bool( + default = True, + doc = """\ +If True, suppress printing stdout and stderr output to the terminal. + +If you would like to get more diagnostic output, please use: + + RULES_PYTHON_REPO_DEBUG=1 + +or + + RULES_PYTHON_REPO_DEBUG_VERBOSITY= +""", + ), + # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute + "timeout": attr.int( + default = 600, + doc = "Timeout (in seconds) on the rule's execution duration.", + ), +} + +def use_isolated(ctx, attr): + """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. + + Args: + ctx: repository or module context + attr: attributes for the repo rule or tag extension + + Returns: + True if --isolated should be passed + """ + use_isolated = attr.isolated + + # The environment variable will take precedence over the attribute + isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) + if isolated_env != None: + if isolated_env.lower() in ("0", "false"): + use_isolated = False + else: + use_isolated = True + + return use_isolated diff --git a/python/private/pip_config_settings.bzl b/python/private/pypi/config_settings.bzl similarity index 53% rename from python/private/pip_config_settings.bzl rename to python/private/pypi/config_settings.bzl index 2fe3c87a10..974121782f 100644 --- a/python/private/pip_config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -39,26 +39,21 @@ Note, that here the specialization of musl vs manylinux wheels is the same in order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa. """ -load(":config_settings.bzl", "is_python_config_setting") -load( - ":pip_flags.bzl", - "INTERNAL_FLAGS", - "UniversalWhlFlag", - "UseWhlFlag", - "WhlLibcFlag", -) +load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag") FLAGS = struct( **{ f: str(Label("//python/config_settings:" + f)) for f in [ "python_version", - "pip_whl", "pip_whl_glibc_version", "pip_whl_muslc_version", "pip_whl_osx_arch", "pip_whl_osx_version", "py_linux_libc", + "is_pip_whl_no", + "is_pip_whl_only", + "is_pip_whl_auto", ] } ) @@ -73,7 +68,7 @@ _flags = struct( } ) -def pip_config_settings( +def config_settings( *, python_versions = [], glibc_versions = [], @@ -82,8 +77,7 @@ def pip_config_settings( target_platforms = [], name = None, visibility = None, - alias_rule = None, - config_setting_rule = None): + native = native): """Generate all of the pip config settings. Args: @@ -100,10 +94,9 @@ def pip_config_settings( constraint values for each condition. visibility (list[str], optional): The visibility to be passed to the exposed labels. All other labels will be private. - alias_rule (rule): The alias rule to use for creating the - objects. Can be overridden for unit tests reasons. - config_setting_rule (rule): The config setting rule to use for creating the - objects. Can be overridden for unit tests reasons. + native (struct): The struct containing alias and config_setting rules + to use for creating the objects. Can be overridden for unit tests + reasons. """ glibc_versions = [""] + glibc_versions @@ -114,43 +107,25 @@ def pip_config_settings( for t in target_platforms ] - alias_rule = alias_rule or native.alias - - for version in python_versions: - is_python = "is_python_{}".format(version) - alias_rule( + for python_version in [""] + python_versions: + is_python = "is_python_{}".format(python_version or "version_unset") + native.alias( name = is_python, actual = Label("//python/config_settings:" + is_python), visibility = visibility, ) - for os, cpu in target_platforms: - constraint_values = [] - suffix = "" - if os: - constraint_values.append("@platforms//os:" + os) - suffix += "_" + os - if cpu: - constraint_values.append("@platforms//cpu:" + cpu) - suffix += "_" + cpu - - _sdist_config_setting( - name = "sdist" + suffix, - constraint_values = constraint_values, - visibility = visibility, - config_setting_rule = config_setting_rule, - ) - for python_version in python_versions: - _sdist_config_setting( - name = "cp{}_sdist{}".format(python_version, suffix), - python_version = python_version, - constraint_values = constraint_values, - visibility = visibility, - config_setting_rule = config_setting_rule, - ) - - for python_version in [""] + python_versions: - _whl_config_settings( + for os, cpu in target_platforms: + constraint_values = [] + suffix = "" + if os: + constraint_values.append("@platforms//os:" + os) + suffix += "_" + os + if cpu: + constraint_values.append("@platforms//cpu:" + cpu) + suffix += "_" + cpu + + _dist_config_settings( suffix = suffix, plat_flag_values = _plat_flag_values( os = os, @@ -161,34 +136,45 @@ def pip_config_settings( ), constraint_values = constraint_values, python_version = python_version, + is_python = is_python, visibility = visibility, - config_setting_rule = config_setting_rule, + native = native, ) -def _whl_config_settings(*, suffix, plat_flag_values, **kwargs): - # With the following three we cover different per-version wheels - python_version = kwargs.get("python_version") - py = "cp{}_py".format(python_version) if python_version else "py" - pycp = "cp{}_cp3x".format(python_version) if python_version else "cp3x" - - flag_values = {} - - for n, f in { - "{}_none_any{}".format(py, suffix): None, - "{}3_none_any{}".format(py, suffix): _flags.whl_py3, - "{}3_abi3_any{}".format(py, suffix): _flags.whl_py3_abi3, - "{}_none_any{}".format(pycp, suffix): _flags.whl_pycp3x, - "{}_abi3_any{}".format(pycp, suffix): _flags.whl_pycp3x_abi3, - "{}_cp_any{}".format(pycp, suffix): _flags.whl_pycp3x_abicp, - }.items(): - if f and f in flag_values: - fail("BUG") - elif f: +def _dist_config_settings(*, suffix, plat_flag_values, **kwargs): + flag_values = {_flags.dist: ""} + + # First create an sdist, we will be building upon the flag values, which + # will ensure that each sdist config setting is the least specialized of + # all. However, we need at least one flag value to cover the case where we + # have `sdist` for any platform, hence we have a non-empty `flag_values` + # here. + _dist_config_setting( + name = "sdist{}".format(suffix), + flag_values = flag_values, + is_pip_whl = FLAGS.is_pip_whl_no, + **kwargs + ) + + for name, f in [ + ("py_none", _flags.whl_py2_py3), + ("py3_none", _flags.whl_py3), + ("py3_abi3", _flags.whl_py3_abi3), + ("cp3x_none", _flags.whl_pycp3x), + ("cp3x_abi3", _flags.whl_pycp3x_abi3), + ("cp3x_cp", _flags.whl_pycp3x_abicp), + ]: + if f in flag_values: + # This should never happen as all of the different whls should have + # unique flag values. + fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) + else: flag_values[f] = "" - _whl_config_setting( - name = n, + _dist_config_setting( + name = "{}_any{}".format(name, suffix), flag_values = flag_values, + is_pip_whl = FLAGS.is_pip_whl_only, **kwargs ) @@ -197,22 +183,25 @@ def _whl_config_settings(*, suffix, plat_flag_values, **kwargs): for (suffix, flag_values) in plat_flag_values: flag_values = flag_values | generic_flag_values - for n, f in { - "{}_none_{}".format(py, suffix): _flags.whl_plat, - "{}3_none_{}".format(py, suffix): _flags.whl_plat_py3, - "{}3_abi3_{}".format(py, suffix): _flags.whl_plat_py3_abi3, - "{}_none_{}".format(pycp, suffix): _flags.whl_plat_pycp3x, - "{}_abi3_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abi3, - "{}_cp_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abicp, - }.items(): - if f and f in flag_values: - fail("BUG") - elif f: + for name, f in [ + ("py_none", _flags.whl_plat), + ("py3_none", _flags.whl_plat_py3), + ("py3_abi3", _flags.whl_plat_py3_abi3), + ("cp3x_none", _flags.whl_plat_pycp3x), + ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3), + ("cp3x_cp", _flags.whl_plat_pycp3x_abicp), + ]: + if f in flag_values: + # This should never happen as all of the different whls should have + # unique flag values. + fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) + else: flag_values[f] = "" - _whl_config_setting( - name = n, + _dist_config_setting( + name = "{}_{}".format(name, suffix), flag_values = flag_values, + is_pip_whl = FLAGS.is_pip_whl_only, **kwargs ) @@ -282,85 +271,50 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): return ret -def _whl_config_setting(*, name, flag_values, visibility, config_setting_rule = None, **kwargs): - config_setting_rule = config_setting_rule or _config_setting_or - config_setting_rule( - name = "is_" + name, - flag_values = flag_values | { - FLAGS.pip_whl: UseWhlFlag.ONLY, - }, - default = flag_values | { - _flags.whl_py2_py3: "", - FLAGS.pip_whl: UseWhlFlag.AUTO, - }, - visibility = visibility, - **kwargs - ) - -def _sdist_config_setting(*, name, visibility, config_setting_rule = None, **kwargs): - config_setting_rule = config_setting_rule or _config_setting_or - config_setting_rule( - name = "is_" + name, - flag_values = {FLAGS.pip_whl: UseWhlFlag.NO}, - default = {FLAGS.pip_whl: UseWhlFlag.AUTO}, - visibility = visibility, - **kwargs - ) +def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native = native, **kwargs): + """A macro to create a target that matches is_pip_whl_auto and one more value. -def _config_setting_or(*, name, flag_values, default, visibility, **kwargs): - match_name = "_{}".format(name) - default_name = "_{}_default".format(name) + Args: + name: The name of the public target. + is_pip_whl: The config setting to match in addition to + `is_pip_whl_auto` when evaluating the config setting. + is_python: The python version config_setting to match. + python_version: The python version name. + native (struct): The struct containing alias and config_setting rules + to use for creating the objects. Can be overridden for unit tests + reasons. + **kwargs: The kwargs passed to the config_setting rule. Visibility of + the main alias target is also taken from the kwargs. + """ + _name = "_is_" + name + visibility = kwargs.get("visibility") native.alias( - name = name, + name = "is_cp{}_{}".format(python_version, name) if python_version else "is_{}".format(name), actual = select({ - "//conditions:default": default_name, - match_name: match_name, + # First match by the python version + is_python: _name, + "//conditions:default": is_python, }), visibility = visibility, ) - _config_setting( - name = match_name, - flag_values = flag_values, - visibility = visibility, - **kwargs - ) - _config_setting( - name = default_name, - flag_values = default, + if python_version: + # Reuse the config_setting targets that we use with the default + # `python_version` setting. + return + + config_setting_name = _name + "_setting" + native.config_setting(name = config_setting_name, **kwargs) + + # Next match by the `pip_whl` flag value and then match by the flags that + # are intrinsic to the distribution. + native.alias( + name = _name, + actual = select({ + "//conditions:default": FLAGS.is_pip_whl_auto, + FLAGS.is_pip_whl_auto: config_setting_name, + is_pip_whl: config_setting_name, + }), visibility = visibility, - **kwargs ) - -def _config_setting(python_version = "", **kwargs): - if python_version: - # NOTE @aignas 2024-05-26: with this we are getting about 24k internal - # config_setting targets in our unit tests. Whilst the number of the - # external dependencies does not dictate this number, it does mean that - # bazel will take longer to parse stuff. This would be especially - # noticeable in repos, which use multiple hub repos within a single - # workspace. - # - # A way to reduce the number of targets would be: - # * put them to a central location and teach this code to just alias them, - # maybe we should create a pip_config_settings repo within the pip - # extension, which would collect config settings for all hub_repos. - # * put them in rules_python - this has the drawback of exposing things like - # is_cp3.10_linux and users may start depending upon the naming - # convention and this API is very unstable. - is_python_config_setting( - python_version = python_version, - **kwargs - ) - else: - # We need this to ensure that there are no ambiguous matches when python_version - # is unset, which usually happens when we are not using the python version aware - # rules. - flag_values = kwargs.pop("flag_values", {}) | { - FLAGS.python_version: "", - } - native.config_setting( - flag_values = flag_values, - **kwargs - ) diff --git a/python/private/pypi/dependency_resolver/BUILD.bazel b/python/private/pypi/dependency_resolver/BUILD.bazel new file mode 100644 index 0000000000..9531b55552 --- /dev/null +++ b/python/private/pypi/dependency_resolver/BUILD.bazel @@ -0,0 +1,7 @@ +exports_files(["dependency_resolver.py"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python/private/pypi:__subpackages__"], +) diff --git a/python/pip_install/tools/dependency_resolver/__init__.py b/python/private/pypi/dependency_resolver/__init__.py similarity index 100% rename from python/pip_install/tools/dependency_resolver/__init__.py rename to python/private/pypi/dependency_resolver/__init__.py diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py similarity index 100% rename from python/pip_install/tools/dependency_resolver/dependency_resolver.py rename to python/private/pypi/dependency_resolver/dependency_resolver.py diff --git a/python/private/pypi/deps.bzl b/python/private/pypi/deps.bzl new file mode 100644 index 0000000000..81bef7aaab --- /dev/null +++ b/python/private/pypi/deps.bzl @@ -0,0 +1,143 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +_RULE_DEPS = [ + # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' + ( + "pypi__build", + "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + ), + ( + "pypi__click", + "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + ), + ( + "pypi__colorama", + "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + ), + ( + "pypi__importlib_metadata", + "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + ), + ( + "pypi__installer", + "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + ), + ( + "pypi__more_itertools", + "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + ), + ( + "pypi__packaging", + "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + ), + ( + "pypi__pep517", + "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + ), + ( + "pypi__pip", + "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + ), + ( + "pypi__pip_tools", + "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + ), + ( + "pypi__pyproject_hooks", + "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + ), + ( + "pypi__setuptools", + "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl", + "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", + ), + ( + "pypi__tomli", + "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + ), + ( + "pypi__wheel", + "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + ), + ( + "pypi__zipp", + "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + ), + # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' +] + +_GENERIC_WHEEL = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "lib", + srcs = glob(["**/*.py"]), + data = glob(["**/*"], exclude=[ + # These entries include those put into user-installed dependencies by + # data_exclude to avoid non-determinism. + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created + "**/* *", + "**/*.dist-info/RECORD", + "BUILD", + "WORKSPACE", + ]), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], +) +""" + +# Collate all the repository names so they can be easily consumed +all_requirements = [name for (name, _, _) in _RULE_DEPS] + +def requirement(pkg): + return Label("@pypi__" + pkg + "//:lib") + +def pypi_deps(): + """ + Fetch dependencies these rules depend on. Workspaces that use the pip_parse rule can call this. + """ + for (name, url, sha256) in _RULE_DEPS: + maybe( + http_archive, + name, + url = url, + sha256 = sha256, + type = "zip", + build_file_content = _GENERIC_WHEEL, + ) diff --git a/python/private/bzlmod/pip.bzl b/python/private/pypi/extension.bzl similarity index 96% rename from python/private/bzlmod/pip.bzl rename to python/private/pypi/extension.bzl index 122debd2a2..6aafc71831 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/pypi/extension.bzl @@ -16,22 +16,19 @@ load("@bazel_features//:features.bzl", "bazel_features") load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load( - "//python/pip_install:pip_repository.bzl", - "pip_repository_attrs", - "use_isolated", - "whl_library", -) load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:pip_repo_name.bzl", "pip_repo_name") -load("//python/private:pypi_index.bzl", "simpleapi_download") -load("//python/private:render_pkg_aliases.bzl", "whl_alias") load("//python/private:repo_utils.bzl", "repo_utils") load("//python/private:version_label.bzl", "version_label") -load(":pip_repository.bzl", "pip_repository") +load(":attrs.bzl", "use_isolated") +load(":hub_repository.bzl", "hub_repository") +load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":render_pkg_aliases.bzl", "whl_alias") +load(":simpleapi_download.bzl", "simpleapi_download") +load(":whl_library.bzl", "whl_library") +load(":whl_repo_name.bzl", "whl_repo_name") def _parse_version(version): major, _, version = version.partition(".") @@ -100,7 +97,7 @@ You cannot use both the additive_build_content and additive_build_content_file a whl_mods = whl_mods, ) -def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): +def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache, exposed_packages): logger = repo_utils.logger(module_ctx) python_interpreter_target = pip_attr.python_interpreter_target is_hub_reproducible = True @@ -239,12 +236,18 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s quiet = (pip_attr.quiet, True), timeout = (pip_attr.timeout, 600), ) - whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) + whl_library_args.update({ + k: v + for k, (v, default) in maybe_args_with_default.items() + if v != default + }) if get_index_urls: # TODO @aignas 2024-05-26: move to a separate function found_something = False + is_exposed = False for requirement in requirements: + is_exposed = is_exposed or requirement.is_exposed for distribution in requirement.whls + [requirement.sdist]: if not distribution: # sdist may be None @@ -264,7 +267,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # This is no-op because pip is not used to download the wheel. whl_library_args.pop("download_only", None) - repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256) + repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) whl_library_args["requirement"] = requirement.srcs.requirement whl_library_args["urls"] = [distribution.url] whl_library_args["sha256"] = distribution.sha256 @@ -289,6 +292,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) if found_something: + if is_exposed: + exposed_packages.setdefault(hub_name, {})[whl_name] = None continue requirement = select_requirement( @@ -430,6 +435,7 @@ def _pip_impl(module_ctx): # Where hub, whl, and pip are the repo names hub_whl_map = {} hub_group_map = {} + exposed_packages = {} simpleapi_cache = {} is_extension_reproducible = True @@ -469,11 +475,11 @@ def _pip_impl(module_ctx): else: pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache, exposed_packages) is_extension_reproducible = is_extension_reproducible and is_hub_reproducible for hub_name, whl_map in hub_whl_map.items(): - pip_repository( + hub_repository( name = hub_name, repo_name = hub_name, whl_map = { @@ -481,6 +487,7 @@ def _pip_impl(module_ctx): for key, value in whl_map.items() }, default_version = _major_minor_version(DEFAULT_PYTHON_VERSION), + packages = sorted(exposed_packages.get(hub_name, {})), groups = hub_group_map.get(hub_name), ) @@ -614,16 +621,9 @@ A dict of labels to wheel names that is typically generated by the whl_modificat The labels are JSON config files describing the modifications. """, ), - }, **pip_repository_attrs) + }, **ATTRS) attrs.update(AUTH_ATTRS) - # Like the pip_repository rule, we end up setting this manually so - # don't allow users to override it. - attrs.pop("repo_prefix") - - # annotations has been replaced with whl_modifications in bzlmod - attrs.pop("annotations") - return attrs def _whl_mod_attrs(): @@ -711,7 +711,7 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) -pip = module_extension( +pypi = module_extension( doc = """\ This extension is used to make dependencies from pip available. @@ -755,7 +755,7 @@ extension. }, ) -pip_internal = module_extension( +pypi_internal = module_extension( doc = """\ This extension is used to make dependencies from pypi available. diff --git a/python/private/pip_flags.bzl b/python/private/pypi/flags.bzl similarity index 85% rename from python/private/pip_flags.bzl rename to python/private/pypi/flags.bzl index c8154ff383..1e380625ce 100644 --- a/python/private/pip_flags.bzl +++ b/python/private/pypi/flags.bzl @@ -18,7 +18,8 @@ NOTE: The transitive loads of this should be kept minimal. This avoids loading unnecessary files when all that are needed are flag definitions. """ -load(":enum.bzl", "enum") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("//python/private:enum.bzl", "enum") # Determines if we should use whls for third party # @@ -54,6 +55,7 @@ WhlLibcFlag = enum( ) INTERNAL_FLAGS = [ + "dist", "whl_plat", "whl_plat_py3", "whl_plat_py3_abi3", @@ -67,3 +69,12 @@ INTERNAL_FLAGS = [ "whl_pycp3x_abi3", "whl_pycp3x_abicp", ] + +def define_pypi_internal_flags(name): + for flag in INTERNAL_FLAGS: + string_flag( + name = "_internal_pip_" + flag, + build_setting_default = "", + values = [""], + visibility = ["//visibility:public"], + ) diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl similarity index 99% rename from python/pip_install/private/generate_group_library_build_bazel.bzl rename to python/private/pypi/generate_group_library_build_bazel.bzl index 5fa93e22b7..54da066b42 100644 --- a/python/pip_install/private/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -14,15 +14,15 @@ """Generate the BUILD.bazel contents for a repo defined by a group_library.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/private:labels.bzl", + ":labels.bzl", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:text_util.bzl", "render") _PRELUDE = """\ load("@rules_python//python:defs.bzl", "py_library") diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl similarity index 99% rename from python/pip_install/private/generate_whl_library_build_bazel.bzl rename to python/private/pypi/generate_whl_library_build_bazel.bzl index f3ddd3bcab..d25f73a049 100644 --- a/python/pip_install/private/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -14,8 +14,10 @@ """Generate the BUILD.bazel contents for a repo defined by a whl_library.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/private:labels.bzl", + ":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "PY_LIBRARY_IMPL_LABEL", @@ -24,8 +26,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:text_util.bzl", "render") _COPY_FILE_TEMPLATE = """\ copy_file( diff --git a/python/private/pypi/group_library.bzl b/python/private/pypi/group_library.bzl new file mode 100644 index 0000000000..ff800e2f18 --- /dev/null +++ b/python/private/pypi/group_library.bzl @@ -0,0 +1,40 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""group_library implementation for WORKSPACE setups.""" + +load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") + +def _group_library_impl(rctx): + build_file_contents = generate_group_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + groups = rctx.attr.groups, + ) + rctx.file("BUILD.bazel", build_file_contents) + +group_library = repository_rule( + attrs = { + "groups": attr.string_list_dict( + doc = "A mapping of group names to requirements within that group.", + ), + "repo_prefix": attr.string( + doc = "Prefix used for the whl_library created components of each group", + ), + }, + implementation = _group_library_impl, + doc = """ +Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. +This is an implementation detail of dependency groups and should not be used alone. + """, +) diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/pypi/hub_repository.bzl similarity index 71% rename from python/private/bzlmod/pip_repository.bzl rename to python/private/pypi/hub_repository.bzl index 0f962031d6..f589dd4744 100644 --- a/python/private/bzlmod/pip_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -14,12 +14,12 @@ "" +load("//python/private:text_util.bzl", "render") load( - "//python/private:render_pkg_aliases.bzl", + ":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases", "whl_alias", ) -load("//python/private:text_util.bzl", "render") _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) @@ -28,8 +28,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ -def _pip_repository_impl(rctx): - bzl_packages = rctx.attr.whl_map.keys() +def _impl(rctx): + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: [whl_alias(**v) for v in json.decode(values)] @@ -55,7 +55,7 @@ def _pip_repository_impl(rctx): for p in bzl_packages ]), "%%ALL_REQUIREMENTS%%": render.list([ - macro_tmpl.format(p, p) + macro_tmpl.format(p, "pkg") for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ @@ -63,38 +63,41 @@ def _pip_repository_impl(rctx): for p in bzl_packages }), "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.repo_name, }) -pip_repository_attrs = { - "default_version": attr.string( - mandatory = True, - doc = """\ +hub_repository = repository_rule( + attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ This is the default python version in the format of X.Y. This should match what is setup by the 'python' extension using the 'is_default = True' setting.""", - ), - "groups": attr.string_list_dict( - mandatory = False, - ), - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_map": attr.string_dict( - mandatory = True, - doc = """\ + ), + "groups": attr.string_list_dict( + mandatory = False, + ), + "packages": attr.string_list( + mandatory = False, + doc = """\ +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +""", + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_dict( + mandatory = True, + doc = """\ The wheel map where values are json.encoded strings of the whl_map constructed in the pip.parse tag class. """, - ), - "_template": attr.label( - default = ":requirements.bzl.tmpl", - ), -} - -pip_repository = repository_rule( - attrs = pip_repository_attrs, + ), + "_template": attr.label( + default = ":requirements.bzl.tmpl.bzlmod", + ), + }, doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_repository_impl, + implementation = _impl, ) diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi/index_sources.bzl similarity index 98% rename from python/private/pypi_index_sources.bzl rename to python/private/pypi/index_sources.bzl index 470a8c9f5a..21660141db 100644 --- a/python/private/pypi_index_sources.bzl +++ b/python/private/pypi/index_sources.bzl @@ -16,7 +16,7 @@ A file that houses private functions used in the `bzlmod` extension with the same name. """ -def get_simpleapi_sources(line): +def index_sources(line): """Get PyPI sources from a requirements.txt line. We interpret the spec described in diff --git a/python/private/labels.bzl b/python/private/pypi/labels.bzl similarity index 100% rename from python/private/labels.bzl rename to python/private/pypi/labels.bzl diff --git a/python/private/pypi/multi_pip_parse.bzl b/python/private/pypi/multi_pip_parse.bzl new file mode 100644 index 0000000000..fe9e2db82d --- /dev/null +++ b/python/private/pypi/multi_pip_parse.bzl @@ -0,0 +1,160 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A pip_parse implementation for version aware toolchains in WORKSPACE.""" + +load(":pip_repository.bzl", pip_parse = "pip_repository") + +def _multi_pip_parse_impl(rctx): + rules_python = rctx.attr._rules_python_workspace.workspace_name + load_statements = [] + install_deps_calls = [] + process_requirements_calls = [] + for python_version, pypi_repository in rctx.attr.pip_parses.items(): + sanitized_python_version = python_version.replace(".", "_") + load_statement = """\ +load( + "@{pypi_repository}//:requirements.bzl", + _{sanitized_python_version}_install_deps = "install_deps", + _{sanitized_python_version}_all_requirements = "all_requirements", +)""".format( + pypi_repository = pypi_repository, + sanitized_python_version = sanitized_python_version, + ) + load_statements.append(load_statement) + process_requirements_call = """\ +_process_requirements( + pkg_labels = _{sanitized_python_version}_all_requirements, + python_version = "{python_version}", + repo_prefix = "{pypi_repository}_", +)""".format( + pypi_repository = pypi_repository, + python_version = python_version, + sanitized_python_version = sanitized_python_version, + ) + process_requirements_calls.append(process_requirements_call) + install_deps_call = """ _{sanitized_python_version}_install_deps(**whl_library_kwargs)""".format( + sanitized_python_version = sanitized_python_version, + ) + install_deps_calls.append(install_deps_call) + + # NOTE @aignas 2023-10-31: I am not sure it is possible to render aliases + # for all of the packages using the `render_pkg_aliases` function because + # we need to know what the list of packages for each version is and then + # we would be creating directories for each. + macro_tmpl = "@%s_{}//:{}" % rctx.attr.name + + requirements_bzl = """\ +# Generated by python/pip.bzl + +load("@{rules_python}//python:pip.bzl", "whl_library_alias", "pip_utils") +{load_statements} + +_wheel_names = [] +_version_map = dict() +def _process_requirements(pkg_labels, python_version, repo_prefix): + for pkg_label in pkg_labels: + wheel_name = Label(pkg_label).package + if not wheel_name: + # We are dealing with the cases where we don't have aliases. + workspace_name = Label(pkg_label).workspace_name + wheel_name = workspace_name[len(repo_prefix):] + + _wheel_names.append(wheel_name) + if not wheel_name in _version_map: + _version_map[wheel_name] = dict() + _version_map[wheel_name][python_version] = repo_prefix + +{process_requirements_calls} + +def requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "pkg") + +def whl_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "whl") + +def data_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "data") + +def dist_info_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "dist_info") + +def install_deps(**whl_library_kwargs): +{install_deps_calls} + for wheel_name in _wheel_names: + whl_library_alias( + name = "{name}_" + wheel_name, + wheel_name = wheel_name, + default_version = "{default_version}", + version_map = _version_map[wheel_name], + ) +""".format( + name = rctx.attr.name, + install_deps_calls = "\n".join(install_deps_calls), + load_statements = "\n".join(load_statements), + macro_tmpl = macro_tmpl, + process_requirements_calls = "\n".join(process_requirements_calls), + rules_python = rules_python, + default_version = rctx.attr.default_version, + ) + rctx.file("requirements.bzl", requirements_bzl) + rctx.file("BUILD.bazel", "exports_files(['requirements.bzl'])") + +_multi_pip_parse = repository_rule( + _multi_pip_parse_impl, + attrs = { + "default_version": attr.string(), + "pip_parses": attr.string_dict(), + "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), + }, +) + +def multi_pip_parse(name, default_version, python_versions, python_interpreter_target, requirements_lock, **kwargs): + """NOT INTENDED FOR DIRECT USE! + + This is intended to be used by the multi_pip_parse implementation in the template of the + multi_toolchain_aliases repository rule. + + Args: + name: the name of the multi_pip_parse repository. + default_version: the default Python version. + python_versions: all Python toolchain versions currently registered. + python_interpreter_target: a dictionary which keys are Python versions and values are resolved host interpreters. + requirements_lock: a dictionary which keys are Python versions and values are locked requirements files. + **kwargs: extra arguments passed to all wrapped pip_parse. + + Returns: + The internal implementation of multi_pip_parse repository rule. + """ + pip_parses = {} + for python_version in python_versions: + if not python_version in python_interpreter_target: + fail("Missing python_interpreter_target for Python version %s in '%s'" % (python_version, name)) + if not python_version in requirements_lock: + fail("Missing requirements_lock for Python version %s in '%s'" % (python_version, name)) + + pip_parse_name = name + "_" + python_version.replace(".", "_") + pip_parse( + name = pip_parse_name, + python_interpreter_target = python_interpreter_target[python_version], + requirements_lock = requirements_lock[python_version], + **kwargs + ) + pip_parses[python_version] = pip_parse_name + + return _multi_pip_parse( + name = name, + default_version = default_version, + pip_parses = pip_parses, + ) diff --git a/python/private/pypi/package_annotation.bzl b/python/private/pypi/package_annotation.bzl new file mode 100644 index 0000000000..4a54703ac4 --- /dev/null +++ b/python/private/pypi/package_annotation.bzl @@ -0,0 +1,49 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Package annotation API for WORKSPACE setups.""" + +def package_annotation( + additive_build_content = None, + copy_files = {}, + copy_executables = {}, + data = [], + data_exclude_glob = [], + srcs_exclude_glob = []): + """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md + + Args: + additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package. + copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf] + copy_executables (dict, optional): A mapping of `src` and `out` files for + [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as + executable. + data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target. + data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated + `py_library` target. + srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target. + + Returns: + str: A json encoded string of the provided content. + """ + return json.encode(struct( + additive_build_content = additive_build_content, + copy_files = copy_files, + copy_executables = copy_executables, + data = data, + data_exclude_glob = data_exclude_glob, + srcs_exclude_glob = srcs_exclude_glob, + )) diff --git a/python/private/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl similarity index 93% rename from python/private/parse_requirements.bzl rename to python/private/pypi/parse_requirements.bzl index cb5024c841..d52180c009 100644 --- a/python/private/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -26,9 +26,9 @@ file for the host platform to be backwards compatible with the legacy behavior. """ -load("//python/pip_install:requirements_parser.bzl", "parse") -load(":normalize_name.bzl", "normalize_name") -load(":pypi_index_sources.bzl", "get_simpleapi_sources") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":index_sources.bzl", "index_sources") +load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms") # This includes the vendored _translate_cpu and _translate_os from @@ -181,6 +181,8 @@ def parse_requirements( * srcs: The Simple API downloadable source list. * requirement_line: The original requirement line. * target_platforms: The list of target platforms that this package is for. + * is_exposed: A boolean if the package should be exposed via the hub + repository. The second element is extra_pip_args should be passed to `whl_library`. """ @@ -266,12 +268,14 @@ def parse_requirements( for p in DEFAULT_PLATFORMS if p not in configured_platforms ] + for p in plats: + configured_platforms[p] = file contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information - # needed for the whl_libary declarations later. - parse_result = parse(contents) + # needed for the whl_library declarations later. + parse_result = parse_requirements_txt(contents) # Replicate a surprising behavior that WORKSPACE builds allowed: # Defining a repo with the same name multiple times, but only the last @@ -317,7 +321,7 @@ def parse_requirements( (requirement_line, ",".join(extra_pip_args)), struct( distribution = distribution, - srcs = get_simpleapi_sources(requirement_line), + srcs = index_sources(requirement_line), requirement_line = requirement_line, target_platforms = [], extra_pip_args = extra_pip_args, @@ -344,6 +348,19 @@ def parse_requirements( ret = {} for whl_name, reqs in requirements_by_platform.items(): + requirement_target_platforms = {} + for r in reqs.values(): + for p in r.target_platforms: + requirement_target_platforms[p] = None + + is_exposed = len(requirement_target_platforms) == len(configured_platforms) + if not is_exposed and logger: + logger.debug(lambda: "Package {} will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + whl_name, + sorted(requirement_target_platforms), + sorted(configured_platforms), + )) + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): whls, sdist = _add_dists( r, @@ -362,6 +379,7 @@ def parse_requirements( download = r.download, whls = whls, sdist = sdist, + is_exposed = is_exposed, ), ) @@ -449,7 +467,7 @@ def _add_dists(requirement, index_urls, python_version, logger = None): continue if logger: - logger.warn("Could not find a whl or an sdist with sha256={}".format(sha256)) + logger.warn(lambda: "Could not find a whl or an sdist with sha256={}".format(sha256)) yanked = {} for dist in whls + [sdist]: diff --git a/python/private/pypi/parse_requirements_txt.bzl b/python/private/pypi/parse_requirements_txt.bzl new file mode 100644 index 0000000000..6f51d034da --- /dev/null +++ b/python/private/pypi/parse_requirements_txt.bzl @@ -0,0 +1,133 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pip requirements parser for Starlark.""" + +_STATE = struct( + # Consume extraneous whitespace + ConsumeSpace = 0, + # Consume a comment + ConsumeComment = 1, + # Parse the name of a pip package + ParseDependency = 2, + # Parse a full requirement line + ParseRequirement = 3, + # Parse a pip option + ParseOption = 4, +) + +EOF = {} + +def parse_requirements_txt(content): + """A simplistic (and incomplete) pip requirements lockfile parser. + + Parses package names and their full requirement lines, as well pip + options. + + Args: + content: lockfile content as a string + + Returns: + Struct with fields `requirements` and `options`. + + requirements: List of requirements, where each requirement is a 2-element + tuple containing the package name and the requirement line. + E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...] + + options: List of pip option lines + """ + content = content.replace("\r", "") + + result = struct( + requirements = [], + options = [], + ) + state = _STATE.ConsumeSpace + buffer = "" + + inputs = content.elems()[:] + inputs.append(EOF) + + for input in inputs: + if state == _STATE.ConsumeSpace: + (state, buffer) = _handleConsumeSpace(input) + elif state == _STATE.ConsumeComment: + (state, buffer) = _handleConsumeComment(input, buffer, result) + elif state == _STATE.ParseDependency: + (state, buffer) = _handleParseDependency(input, buffer, result) + elif state == _STATE.ParseOption: + (state, buffer) = _handleParseOption(input, buffer, result) + elif state == _STATE.ParseRequirement: + (state, buffer) = _handleParseRequirement(input, buffer, result) + else: + fail("Unknown state %d" % state) + + return result + +def _handleConsumeSpace(input): + if input == EOF: + return (_STATE.ConsumeSpace, "") + if input.isspace(): + return (_STATE.ConsumeSpace, "") + elif input == "#": + return (_STATE.ConsumeComment, "") + elif input == "-": + return (_STATE.ParseOption, input) + + return (_STATE.ParseDependency, input) + +def _handleConsumeComment(input, buffer, result): + if input == "\n": + if len(result.requirements) > 0 and len(result.requirements[-1]) == 1: + result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + elif len(buffer) > 0: + result.options.append(buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + return (_STATE.ConsumeSpace, "") + return (_STATE.ConsumeComment, buffer) + +def _handleParseDependency(input, buffer, result): + if input == EOF: + fail("Enountered unexpected end of file while parsing requirement") + elif input.isspace() or input in [">", "<", "~", "=", ";", "["]: + result.requirements.append((buffer,)) + return (_STATE.ParseRequirement, buffer + input) + + return (_STATE.ParseDependency, buffer + input) + +def _handleParseOption(input, buffer, result): + if input == "\n" and buffer.endswith("\\"): + return (_STATE.ParseOption, buffer[0:-1]) + elif input == " ": + result.options.append(buffer.rstrip("\n")) + return (_STATE.ParseOption, "") + elif input == "\n" or input == EOF: + result.options.append(buffer.rstrip("\n")) + return (_STATE.ConsumeSpace, "") + elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): + return (_STATE.ConsumeComment, buffer) + + return (_STATE.ParseOption, buffer + input) + +def _handleParseRequirement(input, buffer, result): + if input == "\n" and buffer.endswith("\\"): + return (_STATE.ParseRequirement, buffer[0:-1]) + elif input == "\n" or input == EOF: + result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): + return (_STATE.ConsumeComment, buffer) + + return (_STATE.ParseRequirement, buffer + input) diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl new file mode 100644 index 0000000000..f7cd032aca --- /dev/null +++ b/python/private/pypi/parse_simpleapi_html.bzl @@ -0,0 +1,106 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parse SimpleAPI HTML in Starlark. +""" + +def parse_simpleapi_html(*, url, content): + """Get the package URLs for given shas by parsing the Simple API HTML. + + Args: + url(str): The URL that the HTML content can be downloaded from. + content(str): The Simple API HTML content. + + Returns: + A list of structs with: + * filename: The filename of the artifact. + * url: The URL to download the artifact. + * sha256: The sha256 of the artifact. + * metadata_sha256: The whl METADATA sha256 if we can download it. If this is + present, then the 'metadata_url' is also present. Defaults to "". + * metadata_url: The URL for the METADATA if we can download it. Defaults to "". + """ + sdists = {} + whls = {} + lines = content.split("= (2, 0): + # We don't expect to have version 2.0 here, but have this check in place just in case. + # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api + fail("Unsupported API version: {}".format(api_version)) + + for line in lines[1:]: + dist_url, _, tail = line.partition("#sha256=") + sha256, _, tail = tail.partition("\"") + + # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api + yanked = "data-yanked" in line + + maybe_metadata, _, tail = tail.partition(">") + filename, _, tail = tail.partition("<") + + metadata_sha256 = "" + metadata_url = "" + for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]: + metadata_marker = metadata_marker + "=\"sha256=" + if metadata_marker in maybe_metadata: + # Implement https://peps.python.org/pep-0714/ + _, _, tail = maybe_metadata.partition(metadata_marker) + metadata_sha256, _, _ = tail.partition("\"") + metadata_url = dist_url + ".metadata" + break + + if filename.endswith(".whl"): + whls[sha256] = struct( + filename = filename, + url = _absolute_url(url, dist_url), + sha256 = sha256, + metadata_sha256 = metadata_sha256, + metadata_url = _absolute_url(url, metadata_url), + yanked = yanked, + ) + else: + sdists[sha256] = struct( + filename = filename, + url = _absolute_url(url, dist_url), + sha256 = sha256, + metadata_sha256 = "", + metadata_url = "", + yanked = yanked, + ) + + return struct( + sdists = sdists, + whls = whls, + ) + +def _absolute_url(index_url, candidate): + if not candidate.startswith(".."): + return candidate + + candidate_parts = candidate.split("..") + last = candidate_parts[-1] + for _ in range(len(candidate_parts) - 1): + index_url, _, _ = index_url.rstrip("/").rpartition("/") + + return "{}/{}".format(index_url, last.strip("/")) diff --git a/python/private/parse_whl_name.bzl b/python/private/pypi/parse_whl_name.bzl similarity index 100% rename from python/private/parse_whl_name.bzl rename to python/private/pypi/parse_whl_name.bzl diff --git a/python/private/patch_whl.bzl b/python/private/pypi/patch_whl.bzl similarity index 87% rename from python/private/patch_whl.bzl rename to python/private/pypi/patch_whl.bzl index 9e3119f744..c2c633da7f 100644 --- a/python/private/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -27,7 +27,8 @@ other patches ensures that the users have overview on exactly what has changed within the wheel. """ -load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:repo_utils.bzl", "repo_utils") +load(":parse_whl_name.bzl", "parse_whl_name") _rules_python_root = Label("//:BUILD.bazel") @@ -40,7 +41,7 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): whl_path: The whl file name to be patched. patches: a label-keyed-int dict that has the patch files as keys and the patch_strip as the value. - **kwargs: extras passed to rctx.execute. + **kwargs: extras passed to repo_utils.execute_checked. Returns: value of the repackaging action. @@ -75,11 +76,12 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): record_patch = rctx.path("RECORD.patch") - result = rctx.execute( - [ + repo_utils.execute_checked( + rctx, + arguments = [ python_interpreter, "-m", - "python.private.repack_whl", + "python.private.pypi.repack_whl", "--record-patch", record_patch, whl_input, @@ -91,16 +93,6 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): **kwargs ) - if result.return_code: - fail( - "repackaging .whl {whl} failed: with exit code '{return_code}':\n{stdout}\n\nstderr:\n{stderr}".format( - whl = whl_input.basename, - stdout = result.stdout, - stderr = result.stderr, - return_code = result.return_code, - ), - ) - if record_patch.exists: record_patch_contents = rctx.read(record_patch) warning_msg = """WARNING: the resultant RECORD file of the patch wheel is different diff --git a/python/private/pypi/pip.bzl b/python/private/pypi/pip.bzl new file mode 100644 index 0000000000..cb8e111e0e --- /dev/null +++ b/python/private/pypi/pip.bzl @@ -0,0 +1,20 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"pip module extensions for use with bzlmod." + +load("//python/private/pypi:extension.bzl", "pypi", "pypi_internal") + +pip = pypi +pip_internal = pypi_internal diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl new file mode 100644 index 0000000000..7389e72120 --- /dev/null +++ b/python/private/pypi/pip_compile.bzl @@ -0,0 +1,167 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Rules to verify and update pip-compile locked requirements.txt. + +NOTE @aignas 2024-06-23: We are using the implementation specific name here to +make it possible to have multiple tools inside the `pypi` directory +""" + +load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") +load(":deps.bzl", "requirement") + +def pip_compile( + name, + src = None, + extra_args = [], + extra_deps = [], + generate_hashes = True, + py_binary = _py_binary, + py_test = _py_test, + requirements_in = None, + requirements_txt = None, + requirements_darwin = None, + requirements_linux = None, + requirements_windows = None, + visibility = ["//visibility:private"], + tags = None, + **kwargs): + """Generates targets for managing pip dependencies with pip-compile. + + By default this rules generates a filegroup named "[name]" which can be included in the data + of some other compile_pip_requirements rule that references these requirements + (e.g. with `-r ../other/requirements.txt`). + + It also generates two targets for running pip-compile: + + - validate with `bazel test [name]_test` + - update with `bazel run [name].update` + + If you are using a version control system, the requirements.txt generated by this rule should + be checked into it to ensure that all developers/users have the same dependency versions. + + Args: + name: base name for generated targets, typically "requirements". + src: file containing inputs to dependency resolution. If not specified, + defaults to `pyproject.toml`. Supported formats are: + * a requirements text file, usually named `requirements.in` + * A `.toml` file, where the `project.dependencies` list is used as per + [PEP621](https://peps.python.org/pep-0621/). + extra_args: passed to pip-compile. + extra_deps: extra dependencies passed to pip-compile. + generate_hashes: whether to put hashes in the requirements_txt file. + py_binary: the py_binary rule to be used. + py_test: the py_test rule to be used. + requirements_in: file expressing desired dependencies. Deprecated, use src instead. + requirements_txt: result of "compiling" the requirements.in file. + requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. + requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. + requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. + tags: tagging attribute common to all build rules, passed to both the _test and .update rules. + visibility: passed to both the _test and .update rules. + **kwargs: other bazel attributes passed to the "_test" rule. + """ + if requirements_in and src: + fail("Only one of 'src' and 'requirements_in' attributes can be used") + else: + src = requirements_in or src or "pyproject.toml" + + requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt + + # "Default" target produced by this macro + # Allow a compile_pip_requirements rule to include another one in the data + # for a requirements file that does `-r ../other/requirements.txt` + native.filegroup( + name = name, + srcs = kwargs.pop("data", []) + [requirements_txt], + visibility = visibility, + ) + + data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + + # Use the Label constructor so this is expanded in the context of the file + # where it appears, which is to say, in @rules_python + pip_compile = Label("//python/private/pypi/dependency_resolver:dependency_resolver.py") + + loc = "$(rlocationpath {})" + + args = [ + loc.format(src), + loc.format(requirements_txt), + "//%s:%s.update" % (native.package_name(), name), + "--resolver=backtracking", + "--allow-unsafe", + ] + if generate_hashes: + args.append("--generate-hashes") + if requirements_linux: + args.append("--requirements-linux={}".format(loc.format(requirements_linux))) + if requirements_darwin: + args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) + if requirements_windows: + args.append("--requirements-windows={}".format(loc.format(requirements_windows))) + args.extend(extra_args) + + deps = [ + requirement("build"), + requirement("click"), + requirement("colorama"), + requirement("importlib_metadata"), + requirement("more_itertools"), + requirement("packaging"), + requirement("pep517"), + requirement("pip"), + requirement("pip_tools"), + requirement("pyproject_hooks"), + requirement("setuptools"), + requirement("tomli"), + requirement("zipp"), + Label("//python/runfiles:runfiles"), + ] + extra_deps + + tags = tags or [] + tags.append("requires-network") + tags.append("no-remote-exec") + tags.append("no-sandbox") + attrs = { + "args": args, + "data": data, + "deps": deps, + "main": pip_compile, + "srcs": [pip_compile], + "tags": tags, + "visibility": visibility, + } + + # cheap way to detect the bazel version + _bazel_version_4_or_greater = "propeller_optimize" in dir(native) + + # Bazel 4.0 added the "env" attribute to py_test/py_binary + if _bazel_version_4_or_greater: + attrs["env"] = kwargs.pop("env", {}) + + py_binary( + name = name + ".update", + **attrs + ) + + timeout = kwargs.pop("timeout", "short") + + py_test( + name = name + "_test", + timeout = timeout, + # kwargs could contain test-specific attributes like size or timeout + **dict(attrs, **kwargs) + ) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl new file mode 100644 index 0000000000..a22f4d9d2c --- /dev/null +++ b/python/private/pypi/pip_repository.bzl @@ -0,0 +1,327 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") +load("//python/private:text_util.bzl", "render") +load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") + +def _get_python_interpreter_attr(rctx): + """A helper function for getting the `python_interpreter` attribute or it's default + + Args: + rctx (repository_ctx): Handle to the rule repository context. + + Returns: + str: The attribute value or it's default + """ + if rctx.attr.python_interpreter: + return rctx.attr.python_interpreter + + if "win" in rctx.os.name: + return "python.exe" + else: + return "python3" + +def use_isolated(ctx, attr): + """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. + + Args: + ctx: repository or module context + attr: attributes for the repo rule or tag extension + + Returns: + True if --isolated should be passed + """ + use_isolated = attr.isolated + + # The environment variable will take precedence over the attribute + isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) + if isolated_env != None: + if isolated_env.lower() in ("0", "false"): + use_isolated = False + else: + use_isolated = True + + return use_isolated + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _pip_repository_impl(rctx): + requirements_by_platform = parse_requirements( + rctx, + requirements_by_platform = rctx.attr.requirements_by_platform, + requirements_linux = rctx.attr.requirements_linux, + requirements_lock = rctx.attr.requirements_lock, + requirements_osx = rctx.attr.requirements_darwin, + requirements_windows = rctx.attr.requirements_windows, + extra_pip_args = rctx.attr.extra_pip_args, + ) + selected_requirements = {} + options = None + repository_platform = host_platform(rctx.os) + for name, requirements in requirements_by_platform.items(): + r = select_requirement( + requirements, + platform = repository_platform, + ) + if not r: + continue + options = options or r.extra_pip_args + selected_requirements[name] = r.requirement_line + + bzl_packages = sorted(selected_requirements.keys()) + + # Normalize cycles first + requirement_cycles = { + name: sorted(sets.to_list(sets.make(deps))) + for name, deps in rctx.attr.experimental_requirement_cycles.items() + } + + # Check for conflicts between cycles _before_ we normalize package names so + # that reported errors use the names the user specified + for i in range(len(requirement_cycles)): + left_group = requirement_cycles.keys()[i] + left_deps = requirement_cycles.values()[i] + for j in range(len(requirement_cycles) - (i + 1)): + right_deps = requirement_cycles.values()[1 + i + j] + right_group = requirement_cycles.keys()[1 + i + j] + for d in left_deps: + if d in right_deps: + fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) + + # And normalize the names as used in the cycle specs + # + # NOTE: We must check that a listed dependency is actually in the actual + # requirements set for the current platform so that we can support cycles in + # platform-conditional requirements. Otherwise we'll blindly generate a + # label referencing a package which may not be installed on the current + # platform. + requirement_cycles = { + normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + for name, group in requirement_cycles.items() + } + + imports = [ + # NOTE: Maintain the order consistent with `buildifier` + 'load("@rules_python//python:pip.bzl", "pip_utils")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', + ] + + annotations = {} + for pkg, annotation in rctx.attr.annotations.items(): + filename = "{}.annotation.json".format(normalize_name(pkg)) + rctx.file(filename, json.encode_indent(json.decode(annotation))) + annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) + + config = { + "download_only": rctx.attr.download_only, + "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, + "environment": rctx.attr.environment, + "envsubst": rctx.attr.envsubst, + "extra_pip_args": options, + "isolated": use_isolated(rctx, rctx.attr), + "pip_data_exclude": rctx.attr.pip_data_exclude, + "python_interpreter": _get_python_interpreter_attr(rctx), + "quiet": rctx.attr.quiet, + "repo": rctx.attr.name, + "timeout": rctx.attr.timeout, + } + if rctx.attr.use_hub_alias_dependencies: + config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name) + else: + config["repo_prefix"] = "{}_".format(rctx.attr.name) + + if rctx.attr.python_interpreter_target: + config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.experimental_target_platforms: + config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms + + macro_tmpl = "@%s//{}:{}" % rctx.attr.name + + aliases = render_pkg_aliases( + aliases = { + pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] + for pkg in bzl_packages or [] + }, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + " # %%GROUP_LIBRARY%%": """\ + group_repo = "{name}__groups" + group_library( + name = group_repo, + repo_prefix = "{name}_", + groups = all_requirement_groups, + )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", + "%%ALL_DATA_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "pkg") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENT_GROUPS%%": render.dict(requirement_cycles), + "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ + p: macro_tmpl.format(p, "whl") + for p in bzl_packages + }), + "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))), + "%%CONFIG%%": render.dict(dict(sorted(config.items()))), + "%%EXTRA_PIP_ARGS%%": json.encode(options), + "%%IMPORTS%%": "\n".join(imports), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + "%%PACKAGES%%": render.list( + [ + ("{}_{}".format(rctx.attr.name, p), r) + for p, r in sorted(selected_requirements.items()) + ], + ), + }) + + return + +pip_repository = repository_rule( + attrs = dict( + annotations = attr.string_dict( + doc = "Optional annotations to apply to packages", + ), + _template = attr.label( + default = ":requirements.bzl.tmpl.workspace", + ), + **ATTRS + ), + doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. + +Those dependencies become available in a generated `requirements.bzl` file. +You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. + +In your WORKSPACE file: + +```starlark +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "pypi", + requirements_lock = ":requirements.txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() +``` + +You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following: +- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library` + created after extracting the `PyYAML` package. +- `@pypi//pyyaml:data` points to the extra data included in the package. +- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package. +- `@pypi//pyyaml:whl` points to the wheel file that was extracted. + +```starlark +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + "@pypi//numpy", + "@pypi//requests", + ], +) +``` + +or + +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + requirement("numpy"), + requirement("requests"), + ], +) +``` + +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```starlark +load("@pypi//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), +) +``` + +Note that for packages whose name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```starlark +load("@pip//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` + +### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. +""", + implementation = _pip_repository_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/pip_repository_attrs.bzl b/python/private/pypi/pip_repository_attrs.bzl new file mode 100644 index 0000000000..23000869e9 --- /dev/null +++ b/python/private/pypi/pip_repository_attrs.bzl @@ -0,0 +1,73 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common attributes between bzlmod pip.parse and workspace pip_parse. + +A common attributes shared between bzlmod and workspace implementations +stored in a separate file to avoid unnecessary refetching of the +repositories.""" + +load(":attrs.bzl", COMMON_ATTRS = "ATTRS") + +ATTRS = { + "requirements_by_platform": attr.label_keyed_string_dict( + doc = """\ +The requirements files and the comma delimited list of target platforms as values. + +The keys are the requirement files and the values are comma-separated platform +identifiers. For now we only support `_` values that are present in +`@platforms//os` and `@platforms//cpu` packages respectively. +""", + ), + "requirements_darwin": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Mac OS", + ), + "requirements_linux": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Linux", + ), + "requirements_lock": attr.label( + allow_single_file = True, + doc = """\ +A fully resolved 'requirements.txt' pip requirement file containing the +transitive set of your dependencies. If this file is passed instead of +'requirements' no resolve will take place and pip_repository will create +individual repositories for each of your dependencies so that wheels are +fetched/built only for the targets specified by 'build/run/test'. Note that if +your lockfile is platform-dependent, you can use the `requirements_[platform]` +attributes. + +Note, that in general requirements files are compiled for a specific platform, +but sometimes they can work for multiple platforms. `rules_python` right now +supports requirements files that are created for a particular platform without +platform markers. +""", + ), + "requirements_windows": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Windows", + ), + "use_hub_alias_dependencies": attr.bool( + default = False, + doc = """\ +Controls if the hub alias dependencies are used. If set to true, then the +group_library will be included in the hub repo. + +True will become default in a subsequent release. +""", + ), +} + +ATTRS.update(**COMMON_ATTRS) diff --git a/python/private/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl similarity index 97% rename from python/private/render_pkg_aliases.bzl rename to python/private/pypi/render_pkg_aliases.bzl index a37c32882e..eb907fee0f 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -16,8 +16,10 @@ This is used in bzlmod and non-bzlmod setups.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/pip_install/private:generate_group_library_build_bazel.bzl", + ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility load( @@ -29,9 +31,7 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -load(":text_util.bzl", "render") load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ @@ -309,7 +309,7 @@ def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwarg aliases = config_setting_aliases, **kwargs ) - contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions) + contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions) return contents def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): @@ -398,12 +398,12 @@ def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): ret.extend(versioned.values()) return ret -def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): +def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): return """\ -load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") +load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") -pip_config_settings( - name = "pip_config_settings", +config_settings( + name = "config_settings", glibc_versions = {glibc_versions}, muslc_versions = {muslc_versions}, osx_versions = {osx_versions}, @@ -544,7 +544,7 @@ def get_filename_config_settings( suffixes = target_platforms else: prefixes = ["{}_{}".format(py, abi)] - suffixes = _whl_config_setting_sufixes( + suffixes = _whl_config_setting_suffixes( platform_tag = parsed.platform_tag, glibc_versions = glibc_versions, muslc_versions = muslc_versions, @@ -578,7 +578,7 @@ def get_filename_config_settings( else: return [":is_{}".format(p) for p in prefixes], setting_supported_versions -def _whl_config_setting_sufixes( +def _whl_config_setting_suffixes( platform_tag, glibc_versions, muslc_versions, diff --git a/python/private/repack_whl.py b/python/private/pypi/repack_whl.py similarity index 100% rename from python/private/repack_whl.py rename to python/private/pypi/repack_whl.py diff --git a/python/private/bzlmod/requirements.bzl.tmpl b/python/private/pypi/requirements.bzl.tmpl.bzlmod similarity index 57% rename from python/private/bzlmod/requirements.bzl.tmpl rename to python/private/pypi/requirements.bzl.tmpl.bzlmod index b99322dd96..ba227aeb2d 100644 --- a/python/private/bzlmod/requirements.bzl.tmpl +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -24,23 +24,3 @@ def data_requirement(name): def dist_info_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - actual_script = script or pkg - - fail("""Please replace this instance of entry_point with the following: - -``` -load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") - -py_console_script_binary( - name = "{pkg}", - pkg = "@%%NAME%%//{pkg}",{script} -) -``` -""".format( - pkg = pip_utils.normalize_name(pkg), - script = "" if not script else "\n script = \"%s\"," % actual_script, - )) diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/private/pypi/requirements.bzl.tmpl.workspace similarity index 92% rename from python/pip_install/pip_repository_requirements.bzl.tmpl rename to python/private/pypi/requirements.bzl.tmpl.workspace index 07b4b08148..2f4bcd6916 100644 --- a/python/pip_install/pip_repository_requirements.bzl.tmpl +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -29,11 +29,6 @@ def data_requirement(name): def dist_info_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") -def entry_point(pkg, script = None): - if not script: - script = pkg - return "@%%NAME%%_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script - def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` # down to `setuptools`. diff --git a/python/pip_install/tools/requirements.txt b/python/private/pypi/requirements.txt similarity index 100% rename from python/pip_install/tools/requirements.txt rename to python/private/pypi/requirements.txt diff --git a/python/private/pypi_index.bzl b/python/private/pypi/simpleapi_download.bzl similarity index 67% rename from python/private/pypi_index.bzl rename to python/private/pypi/simpleapi_download.bzl index 64d908e32b..b258fef07a 100644 --- a/python/private/pypi_index.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -17,9 +17,10 @@ A file that houses private functions used in the `bzlmod` extension with the sam """ load("@bazel_features//:features.bzl", "bazel_features") -load(":auth.bzl", "get_auth") -load(":envsubst.bzl", "envsubst") -load(":normalize_name.bzl", "normalize_name") +load("//python/private:auth.bzl", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":parse_simpleapi_html.bzl", "parse_simpleapi_html") def simpleapi_download(ctx, *, attr, cache, parallel_download = True): """Download Simple API HTML. @@ -71,7 +72,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): success = False for index_url in index_urls: - result = read_simple_api( + result = _read_simpleapi( ctx = ctx, url = "{}/{}/".format( index_url_overrides.get(pkg_normalized, index_url).rstrip("/"), @@ -122,7 +123,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): return contents -def read_simple_api(ctx, url, attr, cache, **download_kwargs): +def _read_simpleapi(ctx, url, attr, cache, **download_kwargs): """Read SimpleAPI. Args: @@ -195,98 +196,9 @@ def _read_index_result(ctx, result, output, url, cache, cache_key): content = ctx.read(output) - output = parse_simple_api_html(url = url, content = content) + output = parse_simpleapi_html(url = url, content = content) if output: cache.setdefault(cache_key, output) return struct(success = True, output = output, cache_key = cache_key) else: return struct(success = False) - -def parse_simple_api_html(*, url, content): - """Get the package URLs for given shas by parsing the Simple API HTML. - - Args: - url(str): The URL that the HTML content can be downloaded from. - content(str): The Simple API HTML content. - - Returns: - A list of structs with: - * filename: The filename of the artifact. - * url: The URL to download the artifact. - * sha256: The sha256 of the artifact. - * metadata_sha256: The whl METADATA sha256 if we can download it. If this is - present, then the 'metadata_url' is also present. Defaults to "". - * metadata_url: The URL for the METADATA if we can download it. Defaults to "". - """ - sdists = {} - whls = {} - lines = content.split("= (2, 0): - # We don't expect to have version 2.0 here, but have this check in place just in case. - # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api - fail("Unsupported API version: {}".format(api_version)) - - for line in lines[1:]: - dist_url, _, tail = line.partition("#sha256=") - sha256, _, tail = tail.partition("\"") - - # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api - yanked = "data-yanked" in line - - maybe_metadata, _, tail = tail.partition(">") - filename, _, tail = tail.partition("<") - - metadata_sha256 = "" - metadata_url = "" - for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]: - metadata_marker = metadata_marker + "=\"sha256=" - if metadata_marker in maybe_metadata: - # Implement https://peps.python.org/pep-0714/ - _, _, tail = maybe_metadata.partition(metadata_marker) - metadata_sha256, _, _ = tail.partition("\"") - metadata_url = dist_url + ".metadata" - break - - if filename.endswith(".whl"): - whls[sha256] = struct( - filename = filename, - url = _absolute_url(url, dist_url), - sha256 = sha256, - metadata_sha256 = metadata_sha256, - metadata_url = _absolute_url(url, metadata_url), - yanked = yanked, - ) - else: - sdists[sha256] = struct( - filename = filename, - url = _absolute_url(url, dist_url), - sha256 = sha256, - metadata_sha256 = "", - metadata_url = "", - yanked = yanked, - ) - - return struct( - sdists = sdists, - whls = whls, - ) - -def _absolute_url(index_url, candidate): - if not candidate.startswith(".."): - return candidate - - candidate_parts = candidate.split("..") - last = candidate_parts[-1] - for _ in range(len(candidate_parts) - 1): - index_url, _, _ = index_url.rstrip("/").rpartition("/") - - return "{}/{}".format(index_url, last.strip("/")) diff --git a/python/private/pypi/whl_installer/BUILD.bazel b/python/private/pypi/whl_installer/BUILD.bazel new file mode 100644 index 0000000000..58231ceb04 --- /dev/null +++ b/python/private/pypi/whl_installer/BUILD.bazel @@ -0,0 +1,36 @@ +load("//python:defs.bzl", "py_binary", "py_library") +load("//python/private/pypi:deps.bzl", "requirement") + +py_library( + name = "lib", + srcs = [ + "arguments.py", + "namespace_pkgs.py", + "wheel.py", + "wheel_installer.py", + ], + visibility = [ + "//tests:__subpackages__", + "//third_party/rules_pycross/pycross/private:__subpackages__", + ], + deps = [ + requirement("installer"), + requirement("pip"), + requirement("packaging"), + requirement("setuptools"), + ], +) + +py_binary( + name = "wheel_installer", + srcs = [ + "wheel_installer.py", + ], + deps = [":lib"], +) + +filegroup( + name = "distribution", + srcs = glob(["*"]), + visibility = ["//python/private/pypi:__subpackages__"], +) diff --git a/python/pip_install/tools/wheel_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py similarity index 98% rename from python/pip_install/tools/wheel_installer/arguments.py rename to python/private/pypi/whl_installer/arguments.py index 71133c29ca..173d3a39a3 100644 --- a/python/pip_install/tools/wheel_installer/arguments.py +++ b/python/private/pypi/whl_installer/arguments.py @@ -17,7 +17,7 @@ import pathlib from typing import Any, Dict, Set -from python.pip_install.tools.wheel_installer import wheel +from python.private.pypi.whl_installer import wheel def parser(**kwargs: Any) -> argparse.ArgumentParser: diff --git a/python/pip_install/tools/wheel_installer/namespace_pkgs.py b/python/private/pypi/whl_installer/namespace_pkgs.py similarity index 100% rename from python/pip_install/tools/wheel_installer/namespace_pkgs.py rename to python/private/pypi/whl_installer/namespace_pkgs.py diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/private/pypi/whl_installer/wheel.py similarity index 96% rename from python/pip_install/tools/wheel_installer/wheel.py rename to python/private/pypi/whl_installer/wheel.py index 3d6780de9a..c167df9193 100644 --- a/python/pip_install/tools/wheel_installer/wheel.py +++ b/python/private/pypi/whl_installer/wheel.py @@ -342,7 +342,12 @@ def __init__( self.name: str = Deps._normalize(name) self._platforms: Set[Platform] = platforms or set() self._target_versions = {p.minor_version for p in platforms or {}} - self._add_version_select = platforms and len(self._target_versions) > 2 + self._default_minor_version = None + if platforms and len(self._target_versions) > 2: + # TODO @aignas 2024-06-23: enable this to be set via a CLI arg + # for being more explicit. + self._default_minor_version = host_interpreter_minor_version() + if None in self._target_versions and len(self._target_versions) > 2: raise ValueError( f"all python versions need to be specified explicitly, got: {platforms}" @@ -540,21 +545,21 @@ def _add_req(self, req: Requirement, extras: Set[str]) -> None: ): continue - if match_arch and self._add_version_select: + if match_arch and self._default_minor_version: self._add(req.name, plat) - if plat.minor_version == host_interpreter_minor_version(): + if plat.minor_version == self._default_minor_version: self._add(req.name, Platform(plat.os, plat.arch)) elif match_arch: - self._add(req.name, plat) - elif match_os and self._add_version_select: + self._add(req.name, Platform(plat.os, plat.arch)) + elif match_os and self._default_minor_version: self._add(req.name, Platform(plat.os, minor_version=plat.minor_version)) - if plat.minor_version == host_interpreter_minor_version(): + if plat.minor_version == self._default_minor_version: self._add(req.name, Platform(plat.os)) elif match_os: self._add(req.name, Platform(plat.os)) - elif match_version and self._add_version_select: + elif match_version and self._default_minor_version: self._add(req.name, Platform(minor_version=plat.minor_version)) - if plat.minor_version == host_interpreter_minor_version(): + if plat.minor_version == self._default_minor_version: self._add(req.name, Platform()) elif match_version: self._add(req.name, None) diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py similarity index 98% rename from python/pip_install/tools/wheel_installer/wheel_installer.py rename to python/private/pypi/whl_installer/wheel_installer.py index 801ef959f0..ef8181c30d 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/private/pypi/whl_installer/wheel_installer.py @@ -27,7 +27,7 @@ from pip._vendor.packaging.utils import canonicalize_name -from python.pip_install.tools.wheel_installer import arguments, namespace_pkgs, wheel +from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel def _configure_reproducible_wheels() -> None: diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl new file mode 100644 index 0000000000..77cbd4e29b --- /dev/null +++ b/python/private/pypi/whl_library.bzl @@ -0,0 +1,509 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("//python:repositories.bzl", "is_standalone_interpreter") +load("//python:versions.bzl", "WINDOWS_NAME") +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load("//python/private:toolchains_repo.bzl", "get_host_os_arch") +load(":attrs.bzl", "ATTRS", "use_isolated") +load(":deps.bzl", "all_requirements") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":patch_whl.bzl", "patch_whl") +load(":whl_target_platforms.bzl", "whl_target_platforms") + +_CPPFLAGS = "CPPFLAGS" +_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + +def _construct_pypath(rctx): + """Helper function to construct a PYTHONPATH. + + Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl. + This allows us to run python code inside repository rule implementations. + + Args: + rctx: Handle to the repository_context. + + Returns: String of the PYTHONPATH. + """ + + separator = ":" if not "windows" in rctx.os.name.lower() else ";" + pypath = separator.join([ + str(rctx.path(entry).dirname) + for entry in rctx.attr._python_path_entries + ]) + return pypath + +def _get_python_interpreter_attr(rctx): + """A helper function for getting the `python_interpreter` attribute or it's default + + Args: + rctx (repository_ctx): Handle to the rule repository context. + + Returns: + str: The attribute value or it's default + """ + if rctx.attr.python_interpreter: + return rctx.attr.python_interpreter + + if "win" in rctx.os.name: + return "python.exe" + else: + return "python3" + +def _resolve_python_interpreter(rctx): + """Helper function to find the python interpreter from the common attributes + + Args: + rctx: Handle to the rule repository context. + + Returns: + `path` object, for the resolved path to the Python interpreter. + """ + python_interpreter = _get_python_interpreter_attr(rctx) + + if rctx.attr.python_interpreter_target != None: + python_interpreter = rctx.path(rctx.attr.python_interpreter_target) + + (os, _) = get_host_os_arch(rctx) + + # On Windows, the symlink doesn't work because Windows attempts to find + # Python DLLs where the symlink is, not where the symlink points. + if os == WINDOWS_NAME: + python_interpreter = python_interpreter.realpath + elif "/" not in python_interpreter: + # It's a plain command, e.g. "python3", to look up in the environment. + found_python_interpreter = rctx.which(python_interpreter) + if not found_python_interpreter: + fail("python interpreter `{}` not found in PATH".format(python_interpreter)) + python_interpreter = found_python_interpreter + else: + python_interpreter = rctx.path(python_interpreter) + return python_interpreter + +def _get_xcode_location_cflags(rctx): + """Query the xcode sdk location to update cflags + + Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + + # Only run on MacOS hosts + if not rctx.os.name.lower().startswith("mac os"): + return [] + + xcode_sdk_location = repo_utils.execute_unchecked( + rctx, + op = "GetXcodeLocation", + arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], + ) + if xcode_sdk_location.return_code != 0: + return [] + + xcode_root = xcode_sdk_location.stdout.strip() + if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) + return [ + "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), + ] + +def _get_toolchain_unix_cflags(rctx, python_interpreter): + """Gather cflags from a standalone toolchain for unix systems. + + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + + # Only run on Unix systems + if not rctx.os.name.lower().startswith(("mac os", "linux")): + return [] + + # Only update the location when using a standalone toolchain. + if not is_standalone_interpreter(rctx, python_interpreter): + return [] + + stdout = repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonVersionForUnixCflags", + arguments = [ + python_interpreter, + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", + ], + ) + _python_version = stdout + include_path = "{}/include/python{}".format( + python_interpreter.dirname, + _python_version, + ) + + return ["-isystem {}".format(include_path)] + +def _parse_optional_attrs(rctx, args, extra_pip_args = None): + """Helper function to parse common attributes of pip_repository and whl_library repository rules. + + This function also serializes the structured arguments as JSON + so they can be passed on the command line to subprocesses. + + Args: + rctx: Handle to the rule repository context. + args: A list of parsed args for the rule. + extra_pip_args: The pip args to pass. + Returns: Augmented args list. + """ + + if use_isolated(rctx, rctx.attr): + args.append("--isolated") + + # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3) + # support rctx.getenv(name, default): When building incrementally, any change to the value of + # the variable named by name will cause this repository to be re-fetched. + if "getenv" in dir(rctx): + getenv = rctx.getenv + else: + getenv = rctx.os.environ.get + + # Check for None so we use empty default types from our attrs. + # Some args want to be list, and some want to be dict. + if extra_pip_args != None: + args += [ + "--extra_pip_args", + json.encode(struct(arg = [ + envsubst(pip_arg, rctx.attr.envsubst, getenv) + for pip_arg in rctx.attr.extra_pip_args + ])), + ] + + if rctx.attr.download_only: + args.append("--download_only") + + if rctx.attr.pip_data_exclude != None: + args += [ + "--pip_data_exclude", + json.encode(struct(arg = rctx.attr.pip_data_exclude)), + ] + + if rctx.attr.enable_implicit_namespace_pkgs: + args.append("--enable_implicit_namespace_pkgs") + + if rctx.attr.environment != None: + args += [ + "--environment", + json.encode(struct(arg = rctx.attr.environment)), + ] + + return args + +def _create_repository_execution_environment(rctx, python_interpreter): + """Create a environment dictionary for processes we spawn with rctx.execute. + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + Returns: + Dictionary of environment variable suitable to pass to rctx.execute. + """ + + # Gather any available CPPFLAGS values + cppflags = [] + cppflags.extend(_get_xcode_location_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) + + env = { + "PYTHONPATH": _construct_pypath(rctx), + _CPPFLAGS: " ".join(cppflags), + } + + return env + +def _whl_library_impl(rctx): + python_interpreter = _resolve_python_interpreter(rctx) + args = [ + python_interpreter, + "-m", + "python.private.pypi.whl_installer.wheel_installer", + "--requirement", + rctx.attr.requirement, + ] + extra_pip_args = [] + extra_pip_args.extend(rctx.attr.extra_pip_args) + + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter) + + whl_path = None + if rctx.attr.whl_file: + whl_path = rctx.path(rctx.attr.whl_file) + + # Simulate the behaviour where the whl is present in the current directory. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + elif rctx.attr.urls: + filename = rctx.attr.filename + urls = rctx.attr.urls + if not filename: + _, _, filename = urls[0].rpartition("/") + + if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")): + if rctx.attr.filename: + msg = "got '{}'".format(filename) + else: + msg = "detected '{}' from url:\n{}".format(filename, urls[0]) + fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg)) + + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + auth = get_auth(rctx, urls), + ) + + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + + if filename.endswith(".whl"): + whl_path = rctx.path(rctx.attr.filename) + else: + # It is an sdist and we need to tell PyPI to use a file in this directory + # and not use any indexes. + extra_pip_args.extend(["--no-index", "--find-links", "."]) + + args = _parse_optional_attrs(rctx, args, extra_pip_args) + + if not whl_path: + repo_utils.execute_checked( + rctx, + op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement), + arguments = args, + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + whl_path = patch_whl( + rctx, + op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement), + python_interpreter = python_interpreter, + whl_path = whl_path, + patches = patches, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + target_platforms = rctx.attr.experimental_target_platforms + if target_platforms: + parsed_whl = parse_whl_name(whl_path.basename) + if parsed_whl.platform_tag != "any": + # NOTE @aignas 2023-12-04: if the wheel is a platform specific + # wheel, we only include deps for that target platform + target_platforms = [ + p.target_platform + for p in whl_target_platforms( + platform_tag = parsed_whl.platform_tag, + abi_tag = parsed_whl.abi_tag, + ) + ] + + repo_utils.execute_checked( + rctx, + op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), + arguments = args + [ + "--whl-file", + whl_path, + ] + ["--platform={}".format(p) for p in target_platforms], + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + metadata = json.decode(rctx.read("metadata.json")) + rctx.delete("metadata.json") + + # NOTE @aignas 2024-06-22: this has to live on until we stop supporting + # passing `twine` as a `:pkg` library via the `WORKSPACE` builds. + # + # See ../../packaging.bzl line 190 + entry_points = {} + for item in metadata["entry_points"]: + name = item["name"] + module = item["module"] + attribute = item["attribute"] + + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) + entry_point_script_name = entry_point_target_name + ".py" + + rctx.file( + entry_point_script_name, + _generate_entry_point_contents(module, attribute), + ) + entry_points[entry_point_without_py] = entry_point_script_name + + build_file_contents = generate_whl_library_build_bazel( + dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), + whl_name = whl_path.basename, + dependencies = metadata["deps"], + dependencies_by_platform = metadata["deps_by_platform"], + group_name = rctx.attr.group_name, + group_deps = rctx.attr.group_deps, + data_exclude = rctx.attr.pip_data_exclude, + tags = [ + "pypi_name=" + metadata["name"], + "pypi_version=" + metadata["version"], + ], + entry_points = entry_points, + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + ) + rctx.file("BUILD.bazel", build_file_contents) + + return + +def _generate_entry_point_contents( + module, + attribute, + shebang = "#!/usr/bin/env python3"): + """Generate the contents of an entry point script. + + Args: + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + contents = """\ +{shebang} +import sys +from {module} import {attribute} +if __name__ == "__main__": + sys.exit({attribute}()) +""".format( + shebang = shebang, + module = module, + attribute = attribute, + ) + return contents + +# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique +whl_library_attrs = dict({ + "annotation": attr.label( + doc = ( + "Optional json encoded file containing annotation to apply to the extracted wheel. " + + "See `package_annotation`" + ), + allow_files = True, + ), + "dep_template": attr.string( + doc = """ +The dep template to use for referencing the dependencies. It should have `{name}` +and `{target}` tokens that will be replaced with the normalized distribution name +and the target that we need respectively. +""", + ), + "filename": attr.string( + doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "repo": attr.string( + mandatory = True, + doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", + ), + "repo_prefix": attr.string( + doc = """ +Prefix for the generated packages will be of the form `@//...` + +DEPRECATED. Only left for people who vendor requirements.bzl. +""", + ), + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", + ), + "sha256": attr.string( + doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", + ), + "urls": attr.string_list( + doc = """\ +The list of urls of the whl to be downloaded using bazel downloader. Using this +attr makes `extra_pip_args` and `download_only` ignored.""", + ), + "whl_file": attr.label( + doc = "The whl file that should be used instead of downloading or building the whl.", + ), + "whl_patches": attr.label_keyed_string_dict( + doc = """a label-keyed-string dict that has + json.encode(struct([whl_file], patch_strip]) as values. This + is to maintain flexibility and correct bzlmod extension interface + until we have a better way to define whl_library and move whl + patching to a separate place. INTERNAL USE ONLY.""", + ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_requirements + ], + ), +}, **ATTRS) +whl_library_attrs.update(AUTH_ATTRS) + +whl_library = repository_rule( + attrs = whl_library_attrs, + doc = """ +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. +Instantiated from pip_repository and inherits config options from there.""", + implementation = _whl_library_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_library_alias.bzl b/python/private/pypi/whl_library_alias.bzl new file mode 100644 index 0000000000..263d7ec0e7 --- /dev/null +++ b/python/private/pypi/whl_library_alias.bzl @@ -0,0 +1,99 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""whl_library aliases for multi_pip_parse.""" + +load("//python/private:full_version.bzl", "full_version") +load(":render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") + +def _whl_library_alias_impl(rctx): + rules_python = rctx.attr._rules_python_workspace.workspace_name + if rctx.attr.default_version: + default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] + else: + default_repo_prefix = None + version_map = rctx.attr.version_map.items() + build_content = ["# Generated by python/pip.bzl"] + for alias_name in ["pkg", "whl", "data", "dist_info"]: + build_content.append(_whl_library_render_alias_target( + alias_name = alias_name, + default_repo_prefix = default_repo_prefix, + rules_python = rules_python, + version_map = version_map, + wheel_name = rctx.attr.wheel_name, + )) + rctx.file("BUILD.bazel", "\n".join(build_content)) + +def _whl_library_render_alias_target( + alias_name, + default_repo_prefix, + rules_python, + version_map, + wheel_name): + alias = ["""\ +alias( + name = "{alias_name}", + actual = select({{""".format(alias_name = alias_name)] + for [python_version, repo_prefix] in version_map: + alias.append("""\ + "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( + full_python_version = full_version(python_version), + actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( + repo_prefix = repo_prefix, + wheel_name = wheel_name, + alias_name = alias_name, + ), + rules_python = rules_python, + )) + if default_repo_prefix: + default_actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( + repo_prefix = default_repo_prefix, + wheel_name = wheel_name, + alias_name = alias_name, + ) + alias.append(' "//conditions:default": "{default_actual}",'.format( + default_actual = default_actual, + )) + + alias.append(" },") # Close select expression condition dict + if not default_repo_prefix: + supported_versions = sorted([python_version for python_version, _ in version_map]) + alias.append(' no_match_error="""{}""",'.format( + NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( + supported_versions = ", ".join(supported_versions), + rules_python = rules_python, + ), + )) + alias.append(" ),") # Close the select expression + alias.append(' visibility = ["//visibility:public"],') + alias.append(")") # Close the alias() expression + return "\n".join(alias) + +whl_library_alias = repository_rule( + _whl_library_alias_impl, + attrs = { + "default_version": attr.string( + mandatory = False, + doc = "Optional Python version in major.minor format, e.g. '3.10'." + + "The Python version of the wheel to use when the versions " + + "from `version_map` don't match. This allows the default " + + "(version unaware) rules to match and select a wheel. If " + + "not specified, then the default rules won't be able to " + + "resolve a wheel and an error will occur.", + ), + "version_map": attr.string_dict(mandatory = True), + "wheel_name": attr.string(mandatory = True), + "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), + }, +) diff --git a/python/private/pip_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl similarity index 94% rename from python/private/pip_repo_name.bzl rename to python/private/pypi/whl_repo_name.bzl index bef4304e15..295f5a45c4 100644 --- a/python/private/pip_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -15,10 +15,10 @@ """A function to convert a dist name to a valid bazel repo name. """ -load(":normalize_name.bzl", "normalize_name") +load("//python/private:normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -def pip_repo_name(prefix, filename, sha256): +def whl_repo_name(prefix, filename, sha256): """Return a valid whl_library repo name given a distribution filename. Args: diff --git a/python/private/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl similarity index 67% rename from python/private/whl_target_platforms.bzl rename to python/private/pypi/whl_target_platforms.bzl index 08177ca1b3..bee795732a 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -46,63 +46,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def _whl_priority(value): - """Return a value for sorting whl lists. - - TODO @aignas 2024-03-29: In the future we should create a repo for each - repo that matches the abi and then we could have config flags for the - preference of `any` wheels or `sdist` or `manylinux` vs `musllinux` or - `universal2`. Ideally we use `select` statements in the hub repo to do - the selection based on the config, but for now this is the best way to - get this working for the host platform. - - In the future the right thing would be to have `bool_flag` or something - similar to be able to have select statements that does the right thing: - * select whls vs sdists. - * select manylinux vs musllinux - * select universal2 vs arch-specific whls - - All of these can be expressed as configuration settings and included in the - select statements in the `whl` repo. This means that the user can configure - for a particular target what they need. - - Returns a 4-tuple where the items are: - * bool - is it an 'any' wheel? True if it is. - * bool - is it an 'universal' wheel? True if it is. (e.g. macos universal2 wheels) - * int - the minor plaform version (e.g. osx os version, libc version) - * int - the major plaform version (e.g. osx os version, libc version) - """ - if "." in value: - value, _, _ = value.partition(".") - - if "any" == value: - return (True, False, 0, 0) - - if "linux" in value: - os, _, tail = value.partition("_") - if os == "linux": - # If the platform tag starts with 'linux', then return something less than what 'any' returns - minor = 0 - major = 0 - else: - major, _, tail = tail.partition("_") # We don't need to use that because it's the same for all candidates now - minor, _, _ = tail.partition("_") - - return (False, os == "linux", int(minor), int(major)) - - if "mac" in value or "osx" in value: - _, _, tail = value.partition("_") - major, _, tail = tail.partition("_") - minor, _, _ = tail.partition("_") - - return (False, "universal2" in value, int(minor), int(major)) - - if not "win" in value: - fail("BUG: only windows, linux and mac platforms are supported, but got: {}".format(value)) - - # Windows does not have multiple wheels for the same target platform - return (False, False, 0, 0) - def select_whls(*, whls, want_python_version = "3.0", want_abis = [], want_platforms = [], logger = None): """Select a subset of wheels suitable for target platforms from a list. @@ -203,46 +146,6 @@ def select_whls(*, whls, want_python_version = "3.0", want_abis = [], want_platf for key, v in candidates.items() ] -def select_whl(*, whls, want_platform): - """Select a suitable wheel from a list. - - Args: - whls(list[struct]): A list of candidates. - want_platform(str): The target platform. - - Returns: - None or a struct with `url`, `sha256` and `filename` attributes for the - selected whl. If no match is found, None is returned. - """ - if not whls: - return None - - # TODO @aignas 2024-05-23: once we do the selection in the hub repo using - # an actual select, then this function will be the one that is used within - # the repository context instead of `select_whl`. - whls = select_whls( - whls = whls, - want_python_version = "", - want_platforms = [want_platform], - ) - - candidates = { - parse_whl_name(w.filename).platform_tag: w - for w in whls - # TODO @aignas 2024-06-01: to be addressed in #1837, where we add the necessary - # config settings. - if "musllinux_" not in w.filename - } - - target_whl_platform = sorted( - candidates.keys(), - key = _whl_priority, - ) - if not target_whl_platform: - return None - - return candidates[target_whl_platform[0]] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/python/private/bzlmod/python.bzl b/python/private/python.bzl similarity index 99% rename from python/private/bzlmod/python.bzl rename to python/private/python.bzl index f8be271fd6..56566790d3 100644 --- a/python/private/bzlmod/python.bzl +++ b/python/private/python.bzl @@ -16,9 +16,9 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:repositories.bzl", "python_register_toolchains") -load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases") -load("//python/private:util.bzl", "IS_BAZEL_6_4_OR_HIGHER") load(":pythons_hub.bzl", "hub_repo") +load(":toolchains_repo.bzl", "multi_toolchain_aliases") +load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") # This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all # targets using any of these toolchains due to the changed repository name. diff --git a/python/private/bzlmod/pythons_hub.bzl b/python/private/pythons_hub.bzl similarity index 100% rename from python/private/bzlmod/pythons_hub.bzl rename to python/private/pythons_hub.bzl diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index 52677584e4..54ad45c2f1 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -86,7 +86,7 @@ def _execute_internal( arguments, environment = {}, **kwargs): - """Execute a subprocess with debugging instrumention. + """Execute a subprocess with debugging instrumentation. Args: rctx: repository_ctx object diff --git a/python/private/runtime_env_toolchain.bzl b/python/private/runtime_env_toolchain.bzl new file mode 100644 index 0000000000..1601926178 --- /dev/null +++ b/python/private/runtime_env_toolchain.bzl @@ -0,0 +1,112 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Definitions related to the Python toolchain.""" + +load("@rules_cc//cc:defs.bzl", "cc_library") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") +load("//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") +load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") +load(":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", "PY_CC_TOOLCHAIN_TYPE", "TARGET_TOOLCHAIN_TYPE") + +_IS_EXEC_TOOLCHAIN_ENABLED = Label("//python/config_settings:is_exec_tools_toolchain_enabled") + +def define_runtime_env_toolchain(name): + """Defines the runtime_env Python toolchain. + + This is a minimal suite of toolchains that provided limited functionality. + They're mostly only useful to aid migration off the builtin + `@bazel_tools//tools/python:autodetecting_toolchain` toolchain. + + NOTE: This was previously called the "autodetecting" toolchain, but was + renamed to better reflect its behavior, since it doesn't autodetect + anything. + + Args: + name: The name of the toolchain to introduce. + """ + base_name = name.replace("_toolchain", "") + + py_runtime( + name = "_runtime_env_py3_runtime", + interpreter = "//python/private:runtime_env_toolchain_interpreter.sh", + python_version = "PY3", + stub_shebang = "#!/usr/bin/env python3", + visibility = ["//visibility:private"], + tags = ["manual"], + ) + + # This is a dummy runtime whose interpreter_path triggers the native rule + # logic to use the legacy behavior on Windows. + # TODO(#7844): Remove this target. + py_runtime( + name = "_magic_sentinel_runtime", + interpreter_path = "/_magic_pyruntime_sentinel_do_not_use", + python_version = "PY3", + visibility = ["//visibility:private"], + tags = ["manual"], + ) + + py_runtime_pair( + name = "_runtime_env_py_runtime_pair", + py3_runtime = select({ + # If we're on windows, inject the sentinel to tell native rule logic + # that we attempted to use the runtime_env toolchain and need to + # switch back to legacy behavior. + # TODO(#7844): Remove this hack. + "@platforms//os:windows": ":_magic_sentinel_runtime", + "//conditions:default": ":_runtime_env_py3_runtime", + }), + visibility = ["//visibility:public"], + tags = ["manual"], + ) + + native.toolchain( + name = name, + toolchain = ":_runtime_env_py_runtime_pair", + toolchain_type = TARGET_TOOLCHAIN_TYPE, + visibility = ["//visibility:public"], + ) + + py_exec_tools_toolchain( + name = "_runtime_env_py_exec_tools_toolchain_impl", + precompiler = Label("//tools/precompiler:precompiler"), + visibility = ["//visibility:private"], + tags = ["manual"], + ) + native.toolchain( + name = base_name + "_py_exec_tools_toolchain", + toolchain = "_runtime_env_py_exec_tools_toolchain_impl", + toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, + target_settings = [_IS_EXEC_TOOLCHAIN_ENABLED], + visibility = ["//visibility:public"], + ) + cc_library( + name = "_empty_cc_lib", + visibility = ["//visibility:private"], + tags = ["manual"], + ) + py_cc_toolchain( + name = "_runtime_env_py_cc_toolchain_impl", + headers = ":_empty_cc_lib", + libs = ":_empty_cc_lib", + python_version = "0.0", + tags = ["manual"], + ) + native.toolchain( + name = base_name + "_py_cc_toolchain", + toolchain = ":_runtime_env_py_cc_toolchain_impl", + toolchain_type = PY_CC_TOOLCHAIN_TYPE, + visibility = ["//visibility:public"], + ) diff --git a/python/private/autodetecting_toolchain_interpreter.sh b/python/private/runtime_env_toolchain_interpreter.sh similarity index 94% rename from python/private/autodetecting_toolchain_interpreter.sh rename to python/private/runtime_env_toolchain_interpreter.sh index 5c8a10d601..2cb7cc7151 100644 --- a/python/private/autodetecting_toolchain_interpreter.sh +++ b/python/private/runtime_env_toolchain_interpreter.sh @@ -8,8 +8,8 @@ set -u # We do lose the ability to set -o pipefail. FAILURE_HEADER="\ -Error occurred while attempting to use the default Python toolchain \ -(@rules_python//python:autodetecting_toolchain)." +Error occurred while attempting to use the deprecated Python toolchain \ +(@rules_python//python/runtime_env_toolchain:all)." die() { echo "$FAILURE_HEADER" 1>&2 diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl index dade9cba9b..702a08e281 100644 --- a/python/private/text_util.bzl +++ b/python/private/text_util.bzl @@ -36,6 +36,9 @@ def _render_alias(name, actual, *, visibility = None): ]) def _render_dict(d, *, key_repr = repr, value_repr = repr): + if not d: + return "{}" + return "\n".join([ "{", _indent("\n".join([ diff --git a/python/private/zip_main_template.py b/python/private/zip_main_template.py index 18eaed9630..2d3aea7b7b 100644 --- a/python/private/zip_main_template.py +++ b/python/private/zip_main_template.py @@ -2,7 +2,7 @@ # # NOTE: This file is a "stage 1" bootstrap, so it's responsible for locating the # desired runtime and having it run the stage 2 bootstrap. This means it can't -# assume much about the current runtime and environment. e.g, the current +# assume much about the current runtime and environment. e.g., the current # runtime may not be the correct one, the zip may not have been extract, the # runfiles env vars may not be set, etc. # diff --git a/python/repositories.bzl b/python/repositories.bzl index 4ffadd050a..d58feefd31 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -19,7 +19,6 @@ For historic reasons, pip_repositories() is defined in //python:pip.bzl. load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:auth.bzl", "get_auth") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") @@ -33,6 +32,7 @@ load( "toolchain_aliases", "toolchains_repo", ) +load("//python/private/pypi:deps.bzl", "pypi_deps") load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", @@ -68,7 +68,7 @@ def py_repositories(): sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf", strip_prefix = "rules_cc-0.0.9", ) - pip_install_dependencies() + pypi_deps() ######## # Remaining content of the file is only used to support toolchains. @@ -399,12 +399,6 @@ py_cc_toolchain( py_exec_tools_toolchain( name = "py_exec_tools_toolchain", - exec_interpreter = "{python_path}", - exec_interpreter_version_info = {{ - "major": "{interpreter_version_info_major}", - "minor": "{interpreter_version_info_minor}", - "micro": "{interpreter_version_info_micro}", - }}, precompiler = "@rules_python//tools/precompiler:precompiler", ) """.format( diff --git a/python/runtime_env_toolchains/BUILD.bazel b/python/runtime_env_toolchains/BUILD.bazel new file mode 100644 index 0000000000..21355ac939 --- /dev/null +++ b/python/runtime_env_toolchains/BUILD.bazel @@ -0,0 +1,19 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//python/private:runtime_env_toolchain.bzl", "define_runtime_env_toolchain") + +package(default_visibility = ["//:__subpackages__"]) + +define_runtime_env_toolchain(name = "runtime_env_toolchain") diff --git a/python/versions.bzl b/python/versions.bzl index 26b975d068..fd385cd1d5 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -532,6 +532,17 @@ PLATFORMS = { os_name = LINUX_NAME, arch = "armv7", ), + "i386-unknown-linux-gnu": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:i386", + ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, + os_name = LINUX_NAME, + arch = "i386", + ), "ppc64le-unknown-linux-gnu": struct( compatible_with = [ "@platforms//os:linux", diff --git a/renovate.json b/renovate.json deleted file mode 100644 index ee8c906b91..0000000000 --- a/renovate.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extends": [ - "config:base" - ] -} diff --git a/tests/base_rules/precompile/precompile_tests.bzl b/tests/base_rules/precompile/precompile_tests.bzl index 02ab4ab19c..5599f6101f 100644 --- a/tests/base_rules/precompile/precompile_tests.bzl +++ b/tests/base_rules/precompile/precompile_tests.bzl @@ -27,6 +27,7 @@ load("//tests/base_rules:py_info_subject.bzl", "py_info_subject") load( "//tests/support:support.bzl", "CC_TOOLCHAIN", + "EXEC_TOOLS_TOOLCHAIN", "PLATFORM_TOOLCHAIN", "PRECOMPILE", "PRECOMPILE_ADD_TO_RUNFILES", @@ -61,6 +62,7 @@ def _test_precompile_enabled_setup(name, py_rule, **kwargs): target = name + "_subject", config_settings = { "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, + EXEC_TOOLS_TOOLCHAIN: "enabled", }, ) @@ -119,6 +121,8 @@ def _test_pyc_only(name): config_settings = { "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, ##PRECOMPILE_SOURCE_RETENTION: "omit_source", + EXEC_TOOLS_TOOLCHAIN: "enabled", + PRECOMPILE: "enabled", }, target = name + "_subject", ) @@ -161,6 +165,7 @@ def _test_precompile_if_generated(name): target = name + "_subject", config_settings = { "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, + EXEC_TOOLS_TOOLCHAIN: "enabled", }, ) @@ -203,6 +208,7 @@ def _test_omit_source_if_generated_source(name): config_settings = { "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, PRECOMPILE_SOURCE_RETENTION: "omit_if_generated_source", + EXEC_TOOLS_TOOLCHAIN: "enabled", }, ) @@ -252,6 +258,7 @@ def _test_precompile_add_to_runfiles_decided_elsewhere(name): "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, PRECOMPILE_ADD_TO_RUNFILES: "decided_elsewhere", PRECOMPILE: "enabled", + EXEC_TOOLS_TOOLCHAIN: "enabled", }, ) @@ -288,6 +295,7 @@ def _test_precompiler_action(name): target = name + "_subject", config_settings = { "//command_line_option:extra_toolchains": _TEST_TOOLCHAINS, + EXEC_TOOLS_TOOLCHAIN: "enabled", }, ) diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index c96ec4e108..eb1a1b6c07 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -18,6 +18,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility load("//tests/base_rules:base_tests.bzl", "create_base_tests") load("//tests/base_rules:util.bzl", "WINDOWS_ATTR", pt_util = "util") load("//tests/support:support.bzl", "LINUX_X86_64", "WINDOWS_X86_64") @@ -297,6 +298,16 @@ def _test_files_to_build_impl(env, target): "{package}/{test_name}_subject.py", ]) + if IS_BAZEL_7_OR_HIGHER: + # As of Bazel 7, the first default output is the executable, so + # verify that is the case. rules_testing + # DepsetFileSubject.contains_exactly doesn't provide an in_order() + # call, nor access to the underlying depset, so we have to do things + # manually. + first_default_output = target[DefaultInfo].files.to_list()[0] + executable = target[DefaultInfo].files_to_run.executable + env.expect.that_file(first_default_output).equals(executable) + def _test_name_cannot_end_in_py(name, config): # Bazel 5 will crash with a Java stacktrace when the native Python # rules have an error. diff --git a/tests/private/envsubst/BUILD.bazel b/tests/envsubst/BUILD.bazel similarity index 100% rename from tests/private/envsubst/BUILD.bazel rename to tests/envsubst/BUILD.bazel diff --git a/tests/private/envsubst/envsubst_tests.bzl b/tests/envsubst/envsubst_tests.bzl similarity index 100% rename from tests/private/envsubst/envsubst_tests.bzl rename to tests/envsubst/envsubst_tests.bzl diff --git a/tests/exec_toolchain_matching/BUILD.bazel b/tests/exec_toolchain_matching/BUILD.bazel new file mode 100644 index 0000000000..ce04bf7897 --- /dev/null +++ b/tests/exec_toolchain_matching/BUILD.bazel @@ -0,0 +1,76 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") # buildifier: disable=bzl-visibility +load( + ":exec_toolchain_matching_tests.bzl", + "define_py_runtime", + "exec_toolchain_matching_test_suite", +) + +exec_toolchain_matching_test_suite( + name = "exec_toolchain_matching_tests", +) + +define_py_runtime( + name = "target_3.12_linux", + interpreter_path = "/linux/python3.12", + interpreter_version_info = { + "major": "3", + "minor": "12", + }, +) + +define_py_runtime( + name = "target_3.12_mac", + interpreter_path = "/mac/python3.12", + interpreter_version_info = { + "major": "3", + "minor": "12", + }, +) + +define_py_runtime( + name = "target_3.12_any", + interpreter_path = "/any/python3.11", + interpreter_version_info = { + "major": "3", + "minor": "11", + }, +) + +define_py_runtime( + name = "target_default", + interpreter_path = "/should_not_match_anything", + interpreter_version_info = { + "major": "-1", + "minor": "-1", + }, +) + +# While these have the same definition, we register duplicates with different +# names because it makes understanding toolchain resolution easier. Toolchain +# resolution debug output shows the implementation name, not the toolchain() +# call that was being evaluated. +py_exec_tools_toolchain( + name = "exec_3.12", +) + +py_exec_tools_toolchain( + name = "exec_3.11_any", +) + +py_exec_tools_toolchain( + name = "exec_default", +) diff --git a/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl b/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl new file mode 100644 index 0000000000..f6eae5ad5f --- /dev/null +++ b/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl @@ -0,0 +1,152 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starlark tests for PyRuntimeInfo provider.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") +load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", "TARGET_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility +load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility +load("//tests/support:support.bzl", "LINUX", "MAC", "PYTHON_VERSION") + +_LookupInfo = provider() # buildifier: disable=provider-params + +def _lookup_toolchains_impl(ctx): + return [_LookupInfo( + target = ctx.toolchains[TARGET_TOOLCHAIN_TYPE], + exec = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE], + )] + +_lookup_toolchains = rule( + implementation = _lookup_toolchains_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE, EXEC_TOOLS_TOOLCHAIN_TYPE], + attrs = {"_use_auto_exec_groups": attr.bool(default = True)}, +) + +def define_py_runtime(name, **kwargs): + py_runtime( + name = name + "_runtime", + **kwargs + ) + py_runtime_pair( + name = name, + py3_runtime = name + "_runtime", + ) + +_tests = [] + +def _test_exec_matches_target_python_version(name): + rt_util.helper_target( + _lookup_toolchains, + name = name + "_subject", + ) + + # ==== Target toolchains ===== + + # This is never matched. It comes first to ensure the python version + # constraint is being respected. + native.toolchain( + name = "00_target_3.11_any", + toolchain_type = TARGET_TOOLCHAIN_TYPE, + toolchain = ":target_3.12_linux", + target_settings = ["//python/config_settings:is_python_3.11"], + ) + + # This is matched by the top-level target being built in what --platforms + # specifies. + native.toolchain( + name = "10_target_3.12_linux", + toolchain_type = TARGET_TOOLCHAIN_TYPE, + toolchain = ":target_3.12_linux", + target_compatible_with = ["@platforms//os:linux"], + target_settings = ["//python/config_settings:is_python_3.12"], + ) + + # This is matched when the exec config switches to the mac platform and + # then looks for a Python runtime for itself. + native.toolchain( + name = "15_target_3.12_mac", + toolchain_type = TARGET_TOOLCHAIN_TYPE, + toolchain = ":target_3.12_mac", + target_compatible_with = ["@platforms//os:macos"], + target_settings = ["//python/config_settings:is_python_3.12"], + ) + + # This is never matched. It's just here so that toolchains from the + # environment don't match. + native.toolchain( + name = "99_target_default", + toolchain_type = TARGET_TOOLCHAIN_TYPE, + toolchain = ":target_default", + ) + + # ==== Exec tools toolchains ===== + + # Register a 3.11 before to ensure it the python version is respected + native.toolchain( + name = "00_exec_3.11_any", + toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, + toolchain = ":exec_3.11_any", + target_settings = ["//python/config_settings:is_python_3.11"], + ) + + # Note that mac comes first. This is so it matches instead of linux + # We only ever look for mac ones, so no need to register others. + native.toolchain( + name = "10_exec_3.12_mac", + toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, + toolchain = ":exec_3.12", + exec_compatible_with = ["@platforms//os:macos"], + target_settings = ["//python/config_settings:is_python_3.12"], + ) + + # This is never matched. It's just here so that toolchains from the + # environment don't match. + native.toolchain( + name = "99_exec_default", + toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, + toolchain = ":exec_default", + ) + + analysis_test( + name = name, + target = name + "_subject", + impl = _test_exec_matches_target_python_version_impl, + config_settings = { + "//command_line_option:extra_execution_platforms": [str(MAC)], + "//command_line_option:extra_toolchains": ["//tests/exec_toolchain_matching:all"], + "//command_line_option:platforms": [str(LINUX)], + PYTHON_VERSION: "3.12", + }, + ) + +_tests.append(_test_exec_matches_target_python_version) + +def _test_exec_matches_target_python_version_impl(env, target): + target_runtime = target[_LookupInfo].target.py3_runtime + exec_runtime = target[_LookupInfo].exec.exec_tools.exec_interpreter[platform_common.ToolchainInfo].py3_runtime + + env.expect.that_str(target_runtime.interpreter_path).equals("/linux/python3.12") + env.expect.that_str(exec_runtime.interpreter_path).equals("/mac/python3.12") + + if IS_BAZEL_7_OR_HIGHER: + target_version = target_runtime.interpreter_version_info + exec_version = exec_runtime.interpreter_version_info + + env.expect.that_bool(target_version == exec_version) + +def exec_toolchain_matching_test_suite(name): + test_suite(name = name, tests = _tests) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index f48bd50353..f1c427f463 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -63,15 +63,6 @@ default_test_runner( # TODO: add compile_pip_requirements_test_from_external_repo -rules_python_integration_test( - name = "pip_repository_entry_points_workspace_test", - timeout = "long", - bzlmod = False, - # The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260 - tags = ["fix-windows"], - workspace_path = "pip_repository_entry_points", -) - rules_python_integration_test( name = "compile_pip_requirements_test", ) diff --git a/tests/integration/compile_pip_requirements/WORKSPACE b/tests/integration/compile_pip_requirements/WORKSPACE index 5a2204bc37..0eeab2067c 100644 --- a/tests/integration/compile_pip_requirements/WORKSPACE +++ b/tests/integration/compile_pip_requirements/WORKSPACE @@ -7,10 +7,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - python_register_toolchains( name = "python39", python_version = "3.9", diff --git a/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE b/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE index 9a08c285e3..48caeb442f 100644 --- a/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE +++ b/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE @@ -7,10 +7,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - python_register_toolchains( name = "python39", python_version = "3.9", diff --git a/tests/integration/pip_parse/WORKSPACE b/tests/integration/pip_parse/WORKSPACE index 4b4f41ddab..db0cd0c7c8 100644 --- a/tests/integration/pip_parse/WORKSPACE +++ b/tests/integration/pip_parse/WORKSPACE @@ -7,6 +7,11 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() +# This call is included in `py_repositories` and we are calling +# `pip_install_dependencies` only to ensure that we are not breaking really old +# code. +# +# TODO @aignas 2024-06-23: remove this before 1.0.0 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") pip_install_dependencies() diff --git a/tests/integration/pip_repository_entry_points/.bazelrc b/tests/integration/pip_repository_entry_points/.bazelrc deleted file mode 100644 index 55ee08f1f0..0000000000 --- a/tests/integration/pip_repository_entry_points/.bazelrc +++ /dev/null @@ -1,10 +0,0 @@ -# Bazel configuration flags - -build --enable_runfiles - -# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file -try-import %workspace%/user.bazelrc - -# The requirements.bzl entry_point functions aren't supported under bzlmod. -# They are replaced by py_console_script_binary, which already has tests -build --noexperimental_enable_bzlmod diff --git a/tests/integration/pip_repository_entry_points/.gitignore b/tests/integration/pip_repository_entry_points/.gitignore deleted file mode 100644 index e5ae073b3c..0000000000 --- a/tests/integration/pip_repository_entry_points/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# git ignore patterns - -/bazel-* -user.bazelrc diff --git a/tests/integration/pip_repository_entry_points/BUILD.bazel b/tests/integration/pip_repository_entry_points/BUILD.bazel deleted file mode 100644 index c39b1f0a2d..0000000000 --- a/tests/integration/pip_repository_entry_points/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@pip//:requirements.bzl", "entry_point") -load("@rules_python//python:defs.bzl", "py_test") -load("@rules_python//python:pip.bzl", "compile_pip_requirements") - -# This rule adds a convenient way to update the requirements file. -compile_pip_requirements( - name = "requirements", - src = "requirements.in", - requirements_windows = ":requirements_windows.txt", -) - -pip_sphinx = entry_point( - pkg = "sphinx", - script = "sphinx-build", -) - -pip_yamllint = entry_point("yamllint") - -py_test( - name = "pip_parse_entry_points_test", - srcs = ["pip_repository_entry_points_test.py"], - data = [ - pip_sphinx, - pip_yamllint, - ], - env = { - "SPHINX_BUILD_ENTRY_POINT": "$(rootpath {})".format(pip_sphinx), - "YAMLLINT_ENTRY_POINT": "$(rootpath {})".format(pip_yamllint), - }, - main = "pip_repository_entry_points_test.py", - deps = ["@rules_python//python/runfiles"], -) diff --git a/tests/integration/pip_repository_entry_points/WORKSPACE b/tests/integration/pip_repository_entry_points/WORKSPACE deleted file mode 100644 index 0ae087b2e0..0000000000 --- a/tests/integration/pip_repository_entry_points/WORKSPACE +++ /dev/null @@ -1,32 +0,0 @@ -workspace(name = "pip_entry_points_example") - -local_repository( - name = "rules_python", - path = "../../..", -) - -load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") - -py_repositories() - -# This toolchain is explicitly 3.10 while `rules_python` is 3.9 to act as -# a regression test, ensuring 3.10 is functional -python_register_toolchains( - name = "python310", - python_version = "3.10", -) - -load("@python310//:defs.bzl", "interpreter") -load("@rules_python//python:pip.bzl", "pip_parse") - -# For a more thorough example of `pip_parse`. See `@rules_python//examples/pip_parse` -pip_parse( - name = "pip", - python_interpreter_target = interpreter, - requirements_lock = "//:requirements.txt", - requirements_windows = "//:requirements_windows.txt", -) - -load("@pip//:requirements.bzl", "install_deps") - -install_deps() diff --git a/tests/integration/pip_repository_entry_points/pip_repository_entry_points_test.py b/tests/integration/pip_repository_entry_points/pip_repository_entry_points_test.py deleted file mode 100644 index 0375153615..0000000000 --- a/tests/integration/pip_repository_entry_points/pip_repository_entry_points_test.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import subprocess -import unittest -from pathlib import Path - -from rules_python.python.runfiles import runfiles - - -class PipRepositoryEntryPointsTest(unittest.TestCase): - maxDiff = None - - def test_entry_point_void_return(self): - env = os.environ.get("YAMLLINT_ENTRY_POINT") - self.assertIsNotNone(env) - - r = runfiles.Create() - entry_point = Path(r.Rlocation(str(Path(*Path(env).parts[1:])))) - self.assertTrue(entry_point.exists()) - - proc = subprocess.run( - [str(entry_point), "--version"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.28.0") - - # yamllint entry_point is of the form `def run(argv=None):` - with self.assertRaises(subprocess.CalledProcessError) as context: - subprocess.run( - [str(entry_point), "--option-does-not-exist"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - self.assertIn("returned non-zero exit status 2", str(context.exception)) - - def test_entry_point_int_return(self): - env = os.environ.get("SPHINX_BUILD_ENTRY_POINT") - self.assertIsNotNone(env) - - r = runfiles.Create() - entry_point = Path(r.Rlocation(str(Path(*Path(env).parts[1:])))) - self.assertTrue(entry_point.exists()) - - proc = subprocess.run( - [str(entry_point), "--version"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - # sphinx-build uses args[0] for its name, only assert the version here - self.assertTrue(proc.stdout.decode("utf-8").strip().endswith("4.3.2")) - - # sphinx-build entry_point is of the form `def main(argv: List[str] = sys.argv[1:]) -> int:` - with self.assertRaises(subprocess.CalledProcessError) as context: - subprocess.run( - [entry_point, "--option-does-not-exist"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - self.assertIn("returned non-zero exit status 2", str(context.exception)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/integration/pip_repository_entry_points/requirements.in b/tests/integration/pip_repository_entry_points/requirements.in deleted file mode 100644 index f5391eb014..0000000000 --- a/tests/integration/pip_repository_entry_points/requirements.in +++ /dev/null @@ -1,8 +0,0 @@ -sphinx==4.3.2 -yamllint>=1.28.0 - -# Last available for Ubuntu python3.6 -setuptools==65.5.1 - -certifi>=2023.7.22 # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 - diff --git a/tests/integration/pip_repository_entry_points/requirements.txt b/tests/integration/pip_repository_entry_points/requirements.txt deleted file mode 100644 index 1ef1f9dd5c..0000000000 --- a/tests/integration/pip_repository_entry_points/requirements.txt +++ /dev/null @@ -1,219 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# bazel run //:requirements.update -# -alabaster==0.7.12 \ - --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \ - --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 - # via sphinx -babel==2.9.1 \ - --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ - --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 - # via sphinx -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 - # via - # -r requirements.in - # requests -charset-normalizer==2.0.10 \ - --hash=sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd \ - --hash=sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455 - # via requests -docutils==0.17.1 \ - --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ - --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 - # via sphinx -idna==3.3 \ - --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ - --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d - # via requests -imagesize==1.3.0 \ - --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \ - --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d - # via sphinx -jinja2==3.0.3 \ - --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \ - --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 - # via sphinx -markupsafe==2.0.1 \ - --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ - --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ - --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ - --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ - --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ - --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ - --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ - --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ - --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ - --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ - --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ - --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ - --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ - --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ - --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ - --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ - --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ - --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ - --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ - --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ - --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ - --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ - --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ - --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ - --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ - --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ - --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ - --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ - --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ - --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ - --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ - --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ - --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ - --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ - --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ - --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ - --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ - --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ - --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ - --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ - --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ - --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ - --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ - --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ - --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ - --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ - --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ - --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ - --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ - --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ - --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ - --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ - --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ - --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ - --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ - --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ - --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ - --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ - --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ - --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ - --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ - --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ - --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ - --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ - --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ - --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ - --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ - --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ - --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 - # via jinja2 -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via sphinx -pathspec==0.9.0 \ - --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ - --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 - # via yamllint -pygments==2.11.2 \ - --hash=sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65 \ - --hash=sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a - # via sphinx -pyparsing==3.0.6 \ - --hash=sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4 \ - --hash=sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81 - # via packaging -pytz==2021.3 \ - --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ - --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 - # via babel -pyyaml==6.0 \ - --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ - --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ - --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ - --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ - --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ - --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ - --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ - --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ - --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ - --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ - --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ - --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ - --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ - --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ - --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ - --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ - --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ - --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ - --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ - --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ - --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ - --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ - --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ - --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ - --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ - --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ - --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ - --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ - --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ - --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ - --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ - --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ - --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 - # via yamllint -requests==2.27.1 \ - --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ - --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d - # via sphinx -snowballstemmer==2.2.0 \ - --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ - --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a - # via sphinx -sphinx==4.3.2 \ - --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \ - --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 \ - --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ - --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 - # via sphinx -sphinxcontrib-devhelp==1.0.2 \ - --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ - --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 \ - --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ - --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 - # via sphinx -sphinxcontrib-jsmath==1.0.1 \ - --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ - --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 - # via sphinx -sphinxcontrib-qthelp==1.0.3 \ - --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ - --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 \ - --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ - --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 - # via sphinx -urllib3==1.26.7 \ - --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ - --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 - # via requests -yamllint==1.28.0 \ - --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ - --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r requirements.in - -# The following packages are considered to be unsafe in a requirements file: -setuptools==65.5.1 \ - --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ - --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f - # via - # -r requirements.in - # sphinx - # yamllint diff --git a/tests/integration/pip_repository_entry_points/requirements_windows.txt b/tests/integration/pip_repository_entry_points/requirements_windows.txt deleted file mode 100644 index 83f2daaf4d..0000000000 --- a/tests/integration/pip_repository_entry_points/requirements_windows.txt +++ /dev/null @@ -1,223 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# bazel run //:requirements.update -# -alabaster==0.7.12 \ - --hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \ - --hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 - # via sphinx -babel==2.9.1 \ - --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ - --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 - # via sphinx -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 - # via - # -r requirements.in - # requests -charset-normalizer==2.0.10 \ - --hash=sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd \ - --hash=sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455 - # via requests -colorama==0.4.6 \ - --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ - --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 - # via sphinx -docutils==0.17.1 \ - --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ - --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 - # via sphinx -idna==3.7 \ - --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ - --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 - # via requests -imagesize==1.3.0 \ - --hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \ - --hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d - # via sphinx -jinja2==3.1.4 \ - --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ - --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d - # via sphinx -markupsafe==2.0.1 \ - --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ - --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ - --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ - --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ - --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ - --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ - --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ - --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ - --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ - --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ - --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ - --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ - --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ - --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ - --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ - --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ - --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ - --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ - --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ - --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ - --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ - --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ - --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ - --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ - --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ - --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ - --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ - --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ - --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ - --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ - --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ - --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ - --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ - --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ - --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ - --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ - --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ - --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ - --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ - --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ - --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ - --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ - --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ - --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ - --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ - --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ - --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ - --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ - --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ - --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ - --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ - --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ - --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ - --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ - --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ - --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ - --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ - --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ - --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ - --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ - --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ - --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ - --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ - --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ - --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ - --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ - --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ - --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ - --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 - # via jinja2 -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via sphinx -pathspec==0.9.0 \ - --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ - --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 - # via yamllint -pygments==2.15.0 \ - --hash=sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094 \ - --hash=sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500 - # via sphinx -pyparsing==3.0.6 \ - --hash=sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4 \ - --hash=sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81 - # via packaging -pytz==2021.3 \ - --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ - --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 - # via babel -pyyaml==6.0 \ - --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ - --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ - --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ - --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ - --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ - --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ - --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ - --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ - --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ - --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ - --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ - --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ - --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ - --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ - --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ - --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ - --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ - --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ - --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ - --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ - --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ - --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ - --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ - --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ - --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ - --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ - --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ - --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ - --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ - --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ - --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ - --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ - --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 - # via yamllint -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 - # via sphinx -snowballstemmer==2.2.0 \ - --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ - --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a - # via sphinx -sphinx==4.3.2 \ - --hash=sha256:0a8836751a68306b3fe97ecbe44db786f8479c3bf4b80e3a7f5c838657b4698c \ - --hash=sha256:6a11ea5dd0bdb197f9c2abc2e0ce73e01340464feaece525e64036546d24c851 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 \ - --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ - --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 - # via sphinx -sphinxcontrib-devhelp==1.0.2 \ - --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ - --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 \ - --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ - --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 - # via sphinx -sphinxcontrib-jsmath==1.0.1 \ - --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ - --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 - # via sphinx -sphinxcontrib-qthelp==1.0.3 \ - --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ - --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 \ - --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ - --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 - # via sphinx -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 - # via requests -yamllint==1.28.0 \ - --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ - --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b - # via -r requirements.in - -# The following packages are considered to be unsafe in a requirements file: -setuptools==65.5.1 \ - --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ - --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f - # via - # -r requirements.in - # sphinx - # yamllint diff --git a/tests/pip_hub_repository/normalize_name/BUILD.bazel b/tests/normalize_name/BUILD.bazel similarity index 100% rename from tests/pip_hub_repository/normalize_name/BUILD.bazel rename to tests/normalize_name/BUILD.bazel diff --git a/tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl b/tests/normalize_name/normalize_name_tests.bzl similarity index 100% rename from tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl rename to tests/normalize_name/normalize_name_tests.bzl diff --git a/tests/pip_install/BUILD.bazel b/tests/pip_install/BUILD.bazel deleted file mode 100644 index 60d25de7df..0000000000 --- a/tests/pip_install/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@bazel_skylib//rules:diff_test.bzl", "diff_test") - -diff_test( - name = "srcs_diff_test", - failure_message = ( - "Please run 'bazel run //python/pip_install/private:srcs_module.update' " + - "to update the 'srcs.bzl' module found in the same package." - ), - file1 = "//python/pip_install/private:srcs_module", - file2 = "//python/pip_install/private:srcs.bzl", - # TODO: The diff_test here fails on Windows. As does the - # install script. This should be fixed. - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), -) diff --git a/tests/pip_install/group_library/BUILD.bazel b/tests/pip_install/group_library/BUILD.bazel deleted file mode 100644 index 5a27e112db..0000000000 --- a/tests/pip_install/group_library/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite") - -generate_build_bazel_test_suite(name = "generate_build_bazel_tests") diff --git a/tests/pip_install/requirements_parser/BUILD.bazel b/tests/pip_install/requirements_parser/BUILD.bazel deleted file mode 100644 index 2787f16f94..0000000000 --- a/tests/pip_install/requirements_parser/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":requirements_parser_tests.bzl", parse_requirements_tests = "parse_tests") - -parse_requirements_tests(name = "test_parse_requirements") diff --git a/tests/pip_install/whl_library/BUILD.bazel b/tests/pip_install/whl_library/BUILD.bazel deleted file mode 100644 index 5a27e112db..0000000000 --- a/tests/pip_install/whl_library/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite") - -generate_build_bazel_test_suite(name = "generate_build_bazel_tests") diff --git a/tests/private/pip_config_settings/BUILD.bazel b/tests/private/pip_config_settings/BUILD.bazel deleted file mode 100644 index c3752e0ac3..0000000000 --- a/tests/private/pip_config_settings/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -load(":pip_config_settings_tests.bzl", "pip_config_settings_test_suite") - -pip_config_settings_test_suite( - name = "pip_config_settings", -) diff --git a/tests/private/pip_repo_name/BUILD.bazel b/tests/private/pip_repo_name/BUILD.bazel deleted file mode 100644 index 7c6782daaf..0000000000 --- a/tests/private/pip_repo_name/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pip_repo_name_tests.bzl", "pip_repo_name_test_suite") - -pip_repo_name_test_suite(name = "pip_repo_name_tests") diff --git a/tests/private/pypi_index/BUILD.bazel b/tests/private/pypi_index/BUILD.bazel deleted file mode 100644 index d365896cd3..0000000000 --- a/tests/private/pypi_index/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pypi_index_tests.bzl", "pypi_index_test_suite") - -pypi_index_test_suite(name = "pypi_index_tests") diff --git a/tests/private/pypi_index_sources/BUILD.bazel b/tests/private/pypi_index_sources/BUILD.bazel deleted file mode 100644 index 212615f480..0000000000 --- a/tests/private/pypi_index_sources/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pypi_index_sources_tests.bzl", "pypi_index_sources_test_suite") - -pypi_index_sources_test_suite(name = "pypi_index_sources_tests") diff --git a/tests/py_runtime_info/BUILD.bazel b/tests/py_runtime_info/BUILD.bazel new file mode 100644 index 0000000000..c501d6d8b1 --- /dev/null +++ b/tests/py_runtime_info/BUILD.bazel @@ -0,0 +1,5 @@ +load(":py_runtime_info_tests.bzl", "py_runtime_info_test_suite") + +py_runtime_info_test_suite( + name = "py_runtime_info_tests", +) diff --git a/tests/py_runtime_info/py_runtime_info_tests.bzl b/tests/py_runtime_info/py_runtime_info_tests.bzl new file mode 100644 index 0000000000..9acf541683 --- /dev/null +++ b/tests/py_runtime_info/py_runtime_info_tests.bzl @@ -0,0 +1,65 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starlark tests for PyRuntimeInfo provider.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility + +def _create_py_runtime_info_without_interpreter_version_info_impl(ctx): + kwargs = {} + if IS_BAZEL_7_OR_HIGHER: + kwargs["bootstrap_template"] = ctx.attr.bootstrap_template + + return [PyRuntimeInfo( + interpreter = ctx.file.interpreter, + files = depset(ctx.files.files), + python_version = "PY3", + **kwargs + )] + +_create_py_runtime_info_without_interpreter_version_info = rule( + implementation = _create_py_runtime_info_without_interpreter_version_info_impl, + attrs = { + "bootstrap_template": attr.label(allow_single_file = True, default = "bootstrap.txt"), + "files": attr.label_list(allow_files = True, default = ["data.txt"]), + "interpreter": attr.label(allow_single_file = True, default = "interpreter.sh"), + "python_version": attr.string(default = "PY3"), + }, +) + +_tests = [] + +def _test_can_create_py_runtime_info_without_interpreter_version_info(name): + _create_py_runtime_info_without_interpreter_version_info( + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_can_create_py_runtime_info_without_interpreter_version_info_impl, + ) + +def _test_can_create_py_runtime_info_without_interpreter_version_info_impl(env, target): + # If we get this for, construction succeeded, so nothing to check + _ = env, target # @unused + +_tests.append(_test_can_create_py_runtime_info_without_interpreter_version_info) + +def py_runtime_info_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/pypi/config_settings/BUILD.bazel b/tests/pypi/config_settings/BUILD.bazel new file mode 100644 index 0000000000..15dbd7f70e --- /dev/null +++ b/tests/pypi/config_settings/BUILD.bazel @@ -0,0 +1,5 @@ +load(":config_settings_tests.bzl", "config_settings_test_suite") + +config_settings_test_suite( + name = "config_settings_tests", +) diff --git a/tests/private/pip_config_settings/pip_config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl similarity index 98% rename from tests/private/pip_config_settings/pip_config_settings_tests.bzl rename to tests/pypi/config_settings/config_settings_tests.bzl index a66e7f47d5..87e18b412f 100644 --- a/tests/private/pip_config_settings/pip_config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -17,7 +17,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") load("@rules_testing//lib:util.bzl", test_util = "util") -load("//python/private:pip_config_settings.bzl", "pip_config_settings") # buildifier: disable=bzl-visibility +load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility def _subject_impl(ctx): _ = ctx # @unused @@ -520,13 +520,13 @@ def _test_all(name): _tests.append(_test_all) -def pip_config_settings_test_suite(name): # buildifier: disable=function-docstring +def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, tests = _tests, ) - pip_config_settings( + config_settings( name = "dummy", python_versions = ["3.8", "3.9", "3.10"], glibc_versions = [(2, 14), (2, 17)], diff --git a/tests/pypi/generate_group_library_build_bazel/BUILD.bazel b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel new file mode 100644 index 0000000000..df5ab82320 --- /dev/null +++ b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel @@ -0,0 +1,3 @@ +load(":generate_group_library_build_bazel_tests.bzl", "generate_group_library_build_bazel_test_suite") + +generate_group_library_build_bazel_test_suite(name = "generate_group_library_build_bazel_tests") diff --git a/tests/pip_install/group_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl similarity index 91% rename from tests/pip_install/group_library/generate_build_bazel_tests.bzl rename to tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl index cf082c2990..a46aa413a3 100644 --- a/tests/pip_install/group_library/generate_build_bazel_tests.bzl +++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility _tests = [] @@ -95,7 +95,7 @@ py_library( _tests.append(_test_in_hub) -def generate_build_bazel_test_suite(name): +def generate_group_library_build_bazel_test_suite(name): """Create the test suite. Args: diff --git a/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel new file mode 100644 index 0000000000..bea8e82ce3 --- /dev/null +++ b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel @@ -0,0 +1,3 @@ +load(":generate_whl_library_build_bazel_tests.bzl", "generate_whl_library_build_bazel_test_suite") + +generate_whl_library_build_bazel_test_suite(name = "generate_whl_library_build_bazel_tests") diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl similarity index 98% rename from tests/pip_install/whl_library/generate_build_bazel_tests.bzl rename to tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 62858afc94..3d4df14b5b 100644 --- a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility _tests = [] @@ -569,7 +569,7 @@ config_setting( _tests.append(_test_group_member_deps_to_hub) -def generate_build_bazel_test_suite(name): +def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. Args: diff --git a/tests/pypi/index_sources/BUILD.bazel b/tests/pypi/index_sources/BUILD.bazel new file mode 100644 index 0000000000..7cd327abef --- /dev/null +++ b/tests/pypi/index_sources/BUILD.bazel @@ -0,0 +1,3 @@ +load(":index_sources_tests.bzl", "index_sources_test_suite") + +index_sources_test_suite(name = "index_sources_tests") diff --git a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl similarity index 88% rename from tests/private/pypi_index_sources/pypi_index_sources_tests.bzl rename to tests/pypi/index_sources/index_sources_tests.bzl index 48d790fc68..0a767078ba 100644 --- a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl +++ b/tests/pypi/index_sources/index_sources_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:pypi_index_sources.bzl", "get_simpleapi_sources") # buildifier: disable=bzl-visibility +load("//python/private/pypi:index_sources.bzl", "index_sources") # buildifier: disable=bzl-visibility _tests = [] @@ -27,7 +27,7 @@ def _test_no_simple_api_sources(env): "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", ] for input in inputs: - got = get_simpleapi_sources(input) + got = index_sources(input) env.expect.that_collection(got.shas).contains_exactly([]) env.expect.that_str(got.version).equals("0.0.1") @@ -45,13 +45,13 @@ def _test_simple_api_sources(env): ], } for input, want_shas in tests.items(): - got = get_simpleapi_sources(input) + got = index_sources(input) env.expect.that_collection(got.shas).contains_exactly(want_shas) env.expect.that_str(got.version).equals("0.0.2") _tests.append(_test_simple_api_sources) -def pypi_index_sources_test_suite(name): +def index_sources_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/parse_requirements/BUILD.bazel b/tests/pypi/parse_requirements/BUILD.bazel similarity index 100% rename from tests/private/parse_requirements/BUILD.bazel rename to tests/pypi/parse_requirements/BUILD.bazel diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl similarity index 96% rename from tests/private/parse_requirements/parse_requirements_tests.bzl rename to tests/pypi/parse_requirements/parse_requirements_tests.bzl index 81cf523460..5c33dd83b2 100644 --- a/tests/private/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -98,6 +98,7 @@ def _test_simple(env): ], whls = [], sdist = None, + is_exposed = True, ), ], }) @@ -145,6 +146,7 @@ def _test_platform_markers_with_python_version(env): ], whls = [], sdist = None, + is_exposed = True, ), ], }) @@ -181,6 +183,7 @@ def _test_dupe_requirements(env): ], whls = [], sdist = None, + is_exposed = True, ), ], }) @@ -220,6 +223,7 @@ def _test_multi_os(env): target_platforms = ["windows_x86_64"], whls = [], sdist = None, + is_exposed = False, ), ], "foo": [ @@ -244,6 +248,7 @@ def _test_multi_os(env): ], whls = [], sdist = None, + is_exposed = True, ), struct( distribution = "foo", @@ -258,6 +263,7 @@ def _test_multi_os(env): target_platforms = ["windows_x86_64"], whls = [], sdist = None, + is_exposed = True, ), ], }) @@ -317,6 +323,7 @@ def _test_multi_os_download_only_platform(env): target_platforms = ["linux_x86_64"], whls = [], sdist = None, + is_exposed = True, ), ], }) @@ -371,6 +378,7 @@ def _test_os_arch_requirements_with_default(env): target_platforms = ["linux_aarch64", "linux_x86_64"], whls = [], sdist = None, + is_exposed = True, ), struct( distribution = "foo", @@ -385,6 +393,7 @@ def _test_os_arch_requirements_with_default(env): target_platforms = ["linux_super_exotic"], whls = [], sdist = None, + is_exposed = True, ), struct( distribution = "foo", @@ -406,6 +415,7 @@ def _test_os_arch_requirements_with_default(env): ], whls = [], sdist = None, + is_exposed = True, ), ], }) diff --git a/tests/pypi/parse_requirements_txt/BUILD.bazel b/tests/pypi/parse_requirements_txt/BUILD.bazel new file mode 100644 index 0000000000..526fa73d4b --- /dev/null +++ b/tests/pypi/parse_requirements_txt/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_requirements_txt_tests.bzl", "parse_requirements_txt_test_suite") + +parse_requirements_txt_test_suite(name = "parse_requirements_txt_tests") diff --git a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl similarity index 88% rename from tests/pip_install/requirements_parser/requirements_parser_tests.bzl rename to tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl index 5ea742e70d..f4e899054a 100644 --- a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl +++ b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl @@ -15,83 +15,83 @@ "Unit tests for yaml.bzl" load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load("//python/pip_install:requirements_parser.bzl", "parse") +load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt") # buildifier: disable=bzl-visibility def _parse_basic_test_impl(ctx): env = unittest.begin(ctx) # Base cases - asserts.equals(env, [], parse("").requirements) - asserts.equals(env, [], parse("\n").requirements) + asserts.equals(env, [], parse_requirements_txt("").requirements) + asserts.equals(env, [], parse_requirements_txt("\n").requirements) # Various requirement specifiers (https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers) - asserts.equals(env, [("SomeProject", "SomeProject")], parse("SomeProject\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("SomeProject == 1.3\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse("SomeProject >= 1.2, < 2.0\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse("SomeProject[foo, bar]\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse("SomeProject ~= 1.4.2\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse("SomeProject == 5.4 ; python_version < '3.8'\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse("SomeProject ; sys_platform == 'win32'\n").requirements) - asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("SomeProject\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("SomeProject == 1.3\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse_requirements_txt("SomeProject >= 1.2, < 2.0\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse_requirements_txt("SomeProject[foo, bar]\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse_requirements_txt("SomeProject ~= 1.4.2\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse_requirements_txt("SomeProject == 5.4 ; python_version < '3.8'\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse_requirements_txt("SomeProject ; sys_platform == 'win32'\n").requirements) + asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse_requirements_txt("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements) # Multiple requirements - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 BarProject==2.0.0 """).requirements) - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 BarProject==2.0.0 """).requirements) # Comments - asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\ # This is a comment SomeProject """).requirements) - asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\ SomeProject # This is a comment """).requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("""\ SomeProject == 1.3 # This is a comment """).requirements) - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 # Comment BarProject==2.0.0 #Comment """).requirements) - asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse("""\ + asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse_requirements_txt("""\ requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 """).requirements) # Multiline - asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse("""\ + asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse_requirements_txt("""\ certifi==2021.10.8 \ --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 # via requests """).requirements) - asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse("""\ + asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse_requirements_txt("""\ requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 \ --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd # via requirements.txt """).requirements) # Options - asserts.equals(env, ["--pre"], parse("--pre\n").options) - asserts.equals(env, ["--find-links", "/my/local/archives"], parse("--find-links /my/local/archives\n").options) - asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\ + asserts.equals(env, ["--pre"], parse_requirements_txt("--pre\n").options) + asserts.equals(env, ["--find-links", "/my/local/archives"], parse_requirements_txt("--find-links /my/local/archives\n").options) + asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\ --pre --find-links /my/local/archives """).options) - asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\ + asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\ --pre # Comment --find-links /my/local/archives """).options) - asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse("""\ + asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse_requirements_txt("""\ --pre # Comment FooProject==1.0.0 --find-links /my/local/archives @@ -116,7 +116,7 @@ def _parse_requirements_lockfile_test_impl(ctx): ("urllib3", "urllib3==1.26.7 --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"), ("yamllint", "yamllint==1.26.3 --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e"), ("setuptools", "setuptools==59.6.0 --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"), - ], parse("""\ + ], parse_requirements_txt("""\ # # This file is autogenerated by pip-compile with python 3.9 # To update, run: @@ -220,5 +220,5 @@ parse_requirements_lockfile_test = unittest.make( attrs = {}, ) -def parse_tests(name): +def parse_requirements_txt_test_suite(name): unittest.suite(name, parse_basic_test, parse_requirements_lockfile_test) diff --git a/tests/pypi/parse_simpleapi_html/BUILD.bazel b/tests/pypi/parse_simpleapi_html/BUILD.bazel new file mode 100644 index 0000000000..e63ef0d5fa --- /dev/null +++ b/tests/pypi/parse_simpleapi_html/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_simpleapi_html_tests.bzl", "parse_simpleapi_html_test_suite") + +parse_simpleapi_html_test_suite(name = "parse_simpleapi_html_tests") diff --git a/tests/private/pypi_index/pypi_index_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl similarity index 95% rename from tests/private/pypi_index/pypi_index_tests.bzl rename to tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl index fa381065b1..a60bb1f330 100644 --- a/tests/private/pypi_index/pypi_index_tests.bzl +++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl @@ -16,7 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private:pypi_index.bzl", "parse_simple_api_html") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility _tests = [] @@ -42,7 +42,7 @@ def _generate_html(*items): ]), ) -def _test_parse_simple_api_html(env): +def _test_sdist(env): # buildifier: disable=unsorted-dict-items tests = [ ( @@ -65,7 +65,7 @@ def _test_parse_simple_api_html(env): for (input, want) in tests: html = _generate_html(input) - got = parse_simple_api_html(url = input.url, content = html) + got = parse_simpleapi_html(url = input.url, content = html) env.expect.that_collection(got.sdists).has_size(1) env.expect.that_collection(got.whls).has_size(0) if not got: @@ -85,9 +85,9 @@ def _test_parse_simple_api_html(env): actual.url().equals(want.url) actual.yanked().equals(want.yanked) -_tests.append(_test_parse_simple_api_html) +_tests.append(_test_sdist) -def _test_parse_simple_api_html_whls(env): +def _test_whls(env): # buildifier: disable=unsorted-dict-items tests = [ ( @@ -189,7 +189,7 @@ def _test_parse_simple_api_html_whls(env): for (input, want) in tests: html = _generate_html(input) - got = parse_simple_api_html(url = input.url, content = html) + got = parse_simpleapi_html(url = input.url, content = html) env.expect.that_collection(got.sdists).has_size(0) env.expect.that_collection(got.whls).has_size(1) if not got: @@ -213,9 +213,9 @@ def _test_parse_simple_api_html_whls(env): actual.url().equals(want.url) actual.yanked().equals(want.yanked) -_tests.append(_test_parse_simple_api_html_whls) +_tests.append(_test_whls) -def pypi_index_test_suite(name): +def parse_simpleapi_html_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/parse_whl_name/BUILD.bazel b/tests/pypi/parse_whl_name/BUILD.bazel similarity index 100% rename from tests/private/parse_whl_name/BUILD.bazel rename to tests/pypi/parse_whl_name/BUILD.bazel diff --git a/tests/private/parse_whl_name/parse_whl_name_tests.bzl b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl similarity index 96% rename from tests/private/parse_whl_name/parse_whl_name_tests.bzl rename to tests/pypi/parse_whl_name/parse_whl_name_tests.bzl index c249f9fb1a..4a88a6e7c5 100644 --- a/tests/private/parse_whl_name/parse_whl_name_tests.bzl +++ b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility _tests = [] diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pypi/render_pkg_aliases/BUILD.bazel similarity index 100% rename from tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel rename to tests/pypi/render_pkg_aliases/BUILD.bazel diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl similarity index 98% rename from tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl rename to tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index a0689f70b9..0d4c75e3c2 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -16,12 +16,9 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility load( - "//python/private:pip_config_settings.bzl", - "pip_config_settings", -) # buildifier: disable=bzl-visibility -load( - "//python/private:render_pkg_aliases.bzl", + "//python/private/pypi:render_pkg_aliases.bzl", "get_filename_config_settings", "get_whl_flag_versions", "multiplatform_whl_aliases", @@ -182,10 +179,10 @@ alias( env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals( """\ -load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") +load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") -pip_config_settings( - name = "pip_config_settings", +config_settings( + name = "config_settings", glibc_versions = [], muslc_versions = [], osx_versions = [], @@ -722,7 +719,6 @@ def _test_cp37_abi3_linux_x86_64(env): python_default = True, want = [ ":is_cp3x_abi3_linux_x86_64", - # TODO @aignas 2024-05-29: update the pip_config_settings to generate this ":is_cp3.2_cp3x_abi3_linux_x86_64", ], ) @@ -924,10 +920,12 @@ def _test_config_settings_exist(env): ] available_config_settings = [] mock_rule = lambda name, **kwargs: available_config_settings.append(name) - pip_config_settings( + config_settings( python_versions = ["3.11"], - alias_rule = mock_rule, - config_setting_rule = mock_rule, + native = struct( + alias = mock_rule, + config_setting = mock_rule, + ), **kwargs ) diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel new file mode 100644 index 0000000000..048a877b65 --- /dev/null +++ b/tests/pypi/whl_installer/BUILD.bazel @@ -0,0 +1,52 @@ +load("//python:defs.bzl", "py_test") + +alias( + name = "lib", + actual = "//python/private/pypi/whl_installer:lib", +) + +py_test( + name = "arguments_test", + size = "small", + srcs = [ + "arguments_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( + name = "namespace_pkgs_test", + size = "small", + srcs = [ + "namespace_pkgs_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( + name = "wheel_installer_test", + size = "small", + srcs = [ + "wheel_installer_test.py", + ], + data = ["//examples/wheel:minimal_with_py_package"], + deps = [ + ":lib", + ], +) + +py_test( + name = "wheel_test", + size = "small", + srcs = [ + "wheel_test.py", + ], + data = ["//examples/wheel:minimal_with_py_package"], + deps = [ + ":lib", + ], +) diff --git a/python/pip_install/tools/wheel_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py similarity index 94% rename from python/pip_install/tools/wheel_installer/arguments_test.py rename to tests/pypi/whl_installer/arguments_test.py index fa018da40f..5538054a59 100644 --- a/python/pip_install/tools/wheel_installer/arguments_test.py +++ b/tests/pypi/whl_installer/arguments_test.py @@ -12,18 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import json import unittest -from python.pip_install.tools.wheel_installer import arguments, wheel +from python.private.pypi.whl_installer import arguments, wheel class ArgumentsTestCase(unittest.TestCase): def test_arguments(self) -> None: parser = arguments.parser() - repo_name = "foo" - repo_prefix = "pypi_" index_url = "--index_url=pypi.org/simple" extra_pip_args = [index_url] requirement = "foo==1.0.0 --hash=sha256:deadbeef" diff --git a/python/pip_install/tools/wheel_installer/namespace_pkgs_test.py b/tests/pypi/whl_installer/namespace_pkgs_test.py similarity index 98% rename from python/pip_install/tools/wheel_installer/namespace_pkgs_test.py rename to tests/pypi/whl_installer/namespace_pkgs_test.py index 4aa0fea978..fbbd50926a 100644 --- a/python/pip_install/tools/wheel_installer/namespace_pkgs_test.py +++ b/tests/pypi/whl_installer/namespace_pkgs_test.py @@ -19,7 +19,7 @@ import unittest from typing import Optional, Set -from python.pip_install.tools.wheel_installer import namespace_pkgs +from python.private.pypi.whl_installer import namespace_pkgs class TempDir: diff --git a/python/pip_install/tools/wheel_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py similarity index 97% rename from python/pip_install/tools/wheel_installer/wheel_installer_test.py rename to tests/pypi/whl_installer/wheel_installer_test.py index 74b9c305f5..7139779c3e 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer_test.py +++ b/tests/pypi/whl_installer/wheel_installer_test.py @@ -19,7 +19,7 @@ import unittest from pathlib import Path -from python.pip_install.tools.wheel_installer import wheel, wheel_installer +from python.private.pypi.whl_installer import wheel_installer class TestRequirementExtrasParsing(unittest.TestCase): diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/tests/pypi/whl_installer/wheel_test.py similarity index 89% rename from python/pip_install/tools/wheel_installer/wheel_test.py rename to tests/pypi/whl_installer/wheel_test.py index 3ddfaf7f2e..76bfe720bc 100644 --- a/python/pip_install/tools/wheel_installer/wheel_test.py +++ b/tests/pypi/whl_installer/wheel_test.py @@ -2,7 +2,7 @@ from random import shuffle from unittest import mock -from python.pip_install.tools.wheel_installer import wheel +from python.private.pypi.whl_installer import wheel class DepsTest(unittest.TestCase): @@ -218,7 +218,43 @@ def test_can_get_deps_based_on_specific_python_version(self): self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" + ) + def test_no_version_select_when_single_version(self, mock_host_interpreter_version): + requires_dist = [ + "bar", + "baz; python_version >= '3.8'", + "posix_dep; os_name=='posix'", + "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", + "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'", + ] + mock_host_interpreter_version.return_value = 7 + + self.maxDiff = None + + deps = wheel.Deps( + "foo", + requires_dist=requires_dist, + platforms=[ + wheel.Platform(os=os, arch=wheel.Arch.x86_64, minor_version=minor) + for minor in [8] + for os in [wheel.OS.linux, wheel.OS.windows] + ], + ) + got = deps.build() + + self.assertEqual(["bar", "baz"], got.deps) + self.assertEqual( + { + "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"], + "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"], + "windows_x86_64": ["arch_dep"], + }, + got.deps_select, + ) + + @mock.patch( + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_can_get_version_select(self, mock_host_interpreter_version): requires_dist = [ @@ -227,6 +263,7 @@ def test_can_get_version_select(self, mock_host_interpreter_version): "baz_new; python_version >= '3.8'", "posix_dep; os_name=='posix'", "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", + "arch_dep; platform_machine=='x86_64' and python_version < '3.8'", ] mock_host_interpreter_version.return_value = 7 @@ -251,6 +288,8 @@ def test_can_get_version_select(self, mock_host_interpreter_version): "@//python/config_settings:is_python_3.8": ["baz_new"], "@//python/config_settings:is_python_3.9": ["baz_new"], "@platforms//os:linux": ["baz", "posix_dep"], + "cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"], + "cp37_windows_x86_64": ["arch_dep", "baz"], "cp37_linux_anyarch": ["baz", "posix_dep"], "cp38_linux_anyarch": [ "baz_new", @@ -262,12 +301,14 @@ def test_can_get_version_select(self, mock_host_interpreter_version): "posix_dep", "posix_dep_with_version", ], + "linux_x86_64": ["arch_dep", "baz", "posix_dep"], + "windows_x86_64": ["arch_dep", "baz"], }, got.deps_select, ) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_spanning_all_target_py_versions_are_added_to_common( self, mock_host_version @@ -290,7 +331,7 @@ def test_deps_spanning_all_target_py_versions_are_added_to_common( self.assertEqual({}, got.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_are_not_duplicated(self, mock_host_version): mock_host_version.return_value = 7 @@ -319,7 +360,7 @@ def test_deps_are_not_duplicated(self, mock_host_version): self.assertEqual({}, got.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_are_not_duplicated_when_encountering_platform_dep_first( self, mock_host_version diff --git a/tests/pypi/whl_repo_name/BUILD.bazel b/tests/pypi/whl_repo_name/BUILD.bazel new file mode 100644 index 0000000000..8671dd7754 --- /dev/null +++ b/tests/pypi/whl_repo_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":whl_repo_name_tests.bzl", "whl_repo_name_test_suite") + +whl_repo_name_test_suite(name = "whl_repo_name_tests") diff --git a/tests/private/pip_repo_name/pip_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl similarity index 82% rename from tests/private/pip_repo_name/pip_repo_name_tests.bzl rename to tests/pypi/whl_repo_name/whl_repo_name_tests.bzl index 574d558277..8b7df83530 100644 --- a/tests/private/pip_repo_name/pip_repo_name_tests.bzl +++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl @@ -15,24 +15,24 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:pip_repo_name.bzl", "pip_repo_name") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name") # buildifier: disable=bzl-visibility _tests = [] def _test_simple(env): - got = pip_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") + got = whl_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") _tests.append(_test_simple) def _test_sdist(env): - got = pip_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") + got = whl_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") env.expect.that_str(got).equals("prefix_foo_sdist_deadbeef") _tests.append(_test_sdist) def _test_platform_whl(env): - got = pip_repo_name( + got = whl_repo_name( "prefix", "foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl", "deadbeef000deadbeef", @@ -43,7 +43,7 @@ def _test_platform_whl(env): _tests.append(_test_platform_whl) -def pip_repo_name_test_suite(name): +def whl_repo_name_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel similarity index 100% rename from tests/private/whl_target_platforms/BUILD.bazel rename to tests/pypi/whl_target_platforms/BUILD.bazel diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl similarity index 89% rename from tests/private/whl_target_platforms/select_whl_tests.bzl rename to tests/pypi/whl_target_platforms/select_whl_tests.bzl index ebd2b263f1..3fd80e3ee3 100644 --- a/tests/private/whl_target_platforms/select_whl_tests.bzl +++ b/tests/pypi/whl_target_platforms/select_whl_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:whl_target_platforms.bzl", "select_whl", "select_whls") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility WHL_LIST = [ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", @@ -79,11 +79,6 @@ def _match(env, got, *want_filenames): # Check that we pass the original structs env.expect.that_str(got[0].other).equals("dummy") -def _select_whl(**kwargs): - """A small wrapper to make the tests more DRY.""" - got_single = select_whl(**kwargs) - return [got_single] if got_single else [] - def _select_whls(whls, **kwargs): return select_whls( whls = [ @@ -214,8 +209,6 @@ def _test_match_abi_and_not_py_version(env): "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", ) - got = _select_whl(whls = got, want_platform = "linux_x86_64") - _match(env, got, "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") _tests.append(_test_match_abi_and_not_py_version) @@ -228,8 +221,6 @@ def _test_select_filename_with_many_tags(env): "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", ) - got = _select_whl(whls = got, want_platform = "linux_x86_32") - _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl") _tests.append(_test_select_filename_with_many_tags) @@ -247,8 +238,6 @@ def _test_osx_prefer_arch_specific(env): "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", ) - got = _select_whl(whls = got, want_platform = "osx_x86_64") - _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl") got = _select_whls(whls = WHL_LIST, want_abis = ["cp311"], want_platforms = ["osx_aarch64"], want_python_version = "3.11") _match( @@ -257,8 +246,6 @@ def _test_osx_prefer_arch_specific(env): "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", ) - got = _select_whl(whls = got, want_platform = "osx_aarch64") - _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl") _tests.append(_test_osx_prefer_arch_specific) @@ -270,8 +257,6 @@ def _test_osx_fallback_to_universal2(env): got, "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", ) - got = _select_whl(whls = got, want_platform = "osx_aarch64") - _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl") _tests.append(_test_osx_fallback_to_universal2) @@ -286,8 +271,6 @@ def _test_prefer_manylinux_wheels(env): "pkg-0.0.1-cp39-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", ) - got = _select_whl(whls = got, want_platform = "linux_x86_64") - _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") _tests.append(_test_prefer_manylinux_wheels) diff --git a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl similarity index 97% rename from tests/private/whl_target_platforms/whl_target_platforms_tests.bzl rename to tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl index 07f3158b31..a72bdc275f 100644 --- a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl +++ b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility _tests = [] diff --git a/python/private/normalize_platform.bzl b/tests/runtime_env_toolchain/BUILD.bazel similarity index 73% rename from python/private/normalize_platform.bzl rename to tests/runtime_env_toolchain/BUILD.bazel index 633062f399..ebcdbaf017 100644 --- a/python/private/normalize_platform.bzl +++ b/tests/runtime_env_toolchain/BUILD.bazel @@ -4,10 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +load(":runtime_env_toolchain_tests.bzl", "runtime_env_toolchain_test_suite") + +runtime_env_toolchain_test_suite(name = "runtime_env_toolchain_tests") diff --git a/tests/runtime_env_toolchain/runtime_env_toolchain_tests.bzl b/tests/runtime_env_toolchain/runtime_env_toolchain_tests.bzl new file mode 100644 index 0000000000..9885a1ef9b --- /dev/null +++ b/tests/runtime_env_toolchain/runtime_env_toolchain_tests.bzl @@ -0,0 +1,101 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starlark tests for py_runtime rule.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load( + "//python/private:toolchain_types.bzl", + "EXEC_TOOLS_TOOLCHAIN_TYPE", + "PY_CC_TOOLCHAIN_TYPE", + "TARGET_TOOLCHAIN_TYPE", +) # buildifier: disable=bzl-visibility +load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility +load("//tests/support:support.bzl", "CC_TOOLCHAIN", "EXEC_TOOLS_TOOLCHAIN", "VISIBLE_FOR_TESTING") + +_LookupInfo = provider() # buildifier: disable=provider-params + +def _use_toolchains_impl(ctx): + return [ + _LookupInfo( + target = ctx.toolchains[TARGET_TOOLCHAIN_TYPE], + exec = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE], + cc = ctx.toolchains[PY_CC_TOOLCHAIN_TYPE], + ), + ] + +_use_toolchains = rule( + implementation = _use_toolchains_impl, + toolchains = [ + TARGET_TOOLCHAIN_TYPE, + EXEC_TOOLS_TOOLCHAIN_TYPE, + PY_CC_TOOLCHAIN_TYPE, + ], +) + +_tests = [] + +def _test_runtime_env_toolchain_matches(name): + rt_util.helper_target( + _use_toolchains, + name = name + "_subject", + ) + extra_toolchains = [ + str(Label("//python/runtime_env_toolchains:all")), + ] + + # We have to add a cc toolchain because py_cc toolchain depends on it. + # However, that package also defines a different fake py_cc toolchain we + # don't want to use, so we need to ensure the runtime_env toolchain has + # higher precendence. + # However, Bazel 6 and Bazel 7 process --extra_toolchains in different + # orders: + # * Bazel 6 goes left to right + # * Bazel 7 goes right to left + # We could just put our preferred toolchain before *and* after + # the undesired toolchain... + # However, Bazel 7 has a bug where *duplicate* entries are ignored, + # and only the *first* entry is respected. + if IS_BAZEL_7_OR_HIGHER: + extra_toolchains.insert(0, CC_TOOLCHAIN) + else: + extra_toolchains.append(CC_TOOLCHAIN) + analysis_test( + name = name, + impl = _test_runtime_env_toolchain_matches_impl, + target = name + "_subject", + config_settings = { + "//command_line_option:extra_toolchains": extra_toolchains, + EXEC_TOOLS_TOOLCHAIN: "enabled", + VISIBLE_FOR_TESTING: True, + }, + ) + +def _test_runtime_env_toolchain_matches_impl(env, target): + env.expect.that_str( + str(target[_LookupInfo].target.toolchain_label), + ).contains("runtime_env_py_runtime_pair") + env.expect.that_str( + str(target[_LookupInfo].exec.toolchain_label), + ).contains("runtime_env_py_exec_tools") + env.expect.that_str( + str(target[_LookupInfo].cc.toolchain_label), + ).contains("runtime_env_py_cc") + +_tests.append(_test_runtime_env_toolchain_matches) + +def runtime_env_toolchain_test_suite(name): + test_suite(name = name, tests = _tests) diff --git a/tests/support/support.bzl b/tests/support/support.bzl index 4bcc554854..a74346d7b3 100644 --- a/tests/support/support.bzl +++ b/tests/support/support.bzl @@ -31,7 +31,10 @@ CC_TOOLCHAIN = str(Label("//tests/cc:all")) # str() around Label() is necessary because rules_testing's config_settings # doesn't accept yet Label objects. +EXEC_TOOLS_TOOLCHAIN = str(Label("//python/config_settings:exec_tools_toolchain")) PRECOMPILE = str(Label("//python/config_settings:precompile")) -PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection")) -PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention")) PRECOMPILE_ADD_TO_RUNFILES = str(Label("//python/config_settings:precompile_add_to_runfiles")) +PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention")) +PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection")) +PYTHON_VERSION = str(Label("//python/config_settings:python_version")) +VISIBLE_FOR_TESTING = str(Label("//python/private:visible_for_testing")) diff --git a/tests/private/text_util/BUILD.bazel b/tests/text_util/BUILD.bazel similarity index 100% rename from tests/private/text_util/BUILD.bazel rename to tests/text_util/BUILD.bazel diff --git a/tests/private/text_util/render_tests.bzl b/tests/text_util/render_tests.bzl similarity index 100% rename from tests/private/text_util/render_tests.bzl rename to tests/text_util/render_tests.bzl diff --git a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel index a87e6aa67e..41485c18a3 100644 --- a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel +++ b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel @@ -20,7 +20,7 @@ py_binary( srcs = ["wheel_installer.py"], visibility = ["//visibility:public"], deps = [ - "//python/pip_install/tools/wheel_installer:lib", + "//python/private/pypi/whl_installer:lib", "@pypi__installer//:lib", ], ) diff --git a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py index 0c352cf129..c03c4c2523 100644 --- a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py +++ b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py @@ -30,7 +30,7 @@ from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile -from python.pip_install.tools.wheel_installer import namespace_pkgs +from python.private.pypi.whl_installer import namespace_pkgs def setup_namespace_pkg_compatibility(wheel_dir: Path) -> None: diff --git a/tools/private/update_deps/BUILD.bazel b/tools/private/update_deps/BUILD.bazel index 2ab7cc73a6..c83deb03db 100644 --- a/tools/private/update_deps/BUILD.bazel +++ b/tools/private/update_deps/BUILD.bazel @@ -50,14 +50,12 @@ py_binary( name = "update_pip_deps", srcs = ["update_pip_deps.py"], data = [ - "//:MODULE.bazel", - "//python/pip_install:repositories", - "//python/pip_install:requirements_txt", + "//python/private/pypi:deps.bzl", + "//python/private/pypi:requirements_txt", ], env = { - "MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)", - "REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)", - "REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)", + "DEPS_BZL": "$(rlocationpath //python/private/pypi:deps.bzl)", + "REQUIREMENTS_TXT": "$(rlocationpath //python/private/pypi:requirements_txt)", }, imports = ["../../.."], deps = [ diff --git a/tools/private/update_deps/update_pip_deps.py b/tools/private/update_deps/update_pip_deps.py index 3c4b46ecfd..1034382f0d 100755 --- a/tools/private/update_deps/update_pip_deps.py +++ b/tools/private/update_deps/update_pip_deps.py @@ -129,19 +129,13 @@ def main(): "--requirements-txt", type=path_from_runfiles, default=os.environ.get("REQUIREMENTS_TXT"), - help="The requirements.txt path for the pip_install tools, defaults to the value taken from REQUIREMENTS_TXT", + help="The requirements.txt path for the pypi tools, defaults to the value taken from REQUIREMENTS_TXT", ) parser.add_argument( - "--module-bazel", + "--deps-bzl", type=path_from_runfiles, - default=os.environ.get("MODULE_BAZEL"), - help="The path for the file to be updated, defaults to the value taken from MODULE_BAZEL", - ) - parser.add_argument( - "--repositories-bzl", - type=path_from_runfiles, - default=os.environ.get("REPOSITORIES_BZL"), - help="The path for the file to be updated, defaults to the value taken from REPOSITORIES_BZL", + default=os.environ.get("DEPS_BZL"), + help="The path for the file to be updated, defaults to the value taken from DEPS_BZL", ) args = parser.parse_args() @@ -149,21 +143,13 @@ def main(): deps = _get_deps(report) update_file( - path=args.repositories_bzl, + path=args.deps_bzl, snippet=_dep_snippet(deps), start_marker=args.start, end_marker=args.end, dry_run=args.dry_run, ) - update_file( - path=args.module_bazel, - snippet=_module_snippet(deps), - start_marker=args.start, - end_marker=args.end, - dry_run=args.dry_run, - ) - if __name__ == "__main__": main() diff --git a/tools/publish/requirements.txt b/tools/publish/requirements.txt index 0fda1bbb4e..2a9721df34 100644 --- a/tools/publish/requirements.txt +++ b/tools/publish/requirements.txt @@ -168,30 +168,39 @@ charset-normalizer==3.0.1 \ --hash=sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7 \ --hash=sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8 # via requests -cryptography==41.0.6 \ - --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \ - --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \ - --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \ - --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \ - --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \ - --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \ - --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \ - --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \ - --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \ - --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \ - --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \ - --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \ - --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \ - --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \ - --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \ - --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \ - --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \ - --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \ - --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \ - --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \ - --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \ - --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \ - --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae +cryptography==42.0.4 \ + --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ + --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ + --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ + --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ + --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ + --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ + --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ + --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ + --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ + --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ + --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ + --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ + --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ + --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ + --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ + --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ + --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ + --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ + --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ + --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ + --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ + --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ + --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ + --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ + --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ + --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ + --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ + --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ + --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ + --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ + --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ + --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 # via secretstorage docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ diff --git a/tools/publish/requirements_darwin.txt b/tools/publish/requirements_darwin.txt index 5bf8544c0a..290411a74c 100644 --- a/tools/publish/requirements_darwin.txt +++ b/tools/publish/requirements_darwin.txt @@ -106,9 +106,9 @@ docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc # via readme-renderer -idna==3.4 \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests importlib-metadata==6.0.0 \ --hash=sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad \ @@ -176,9 +176,9 @@ twine==4.0.2 \ --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 # via -r tools/publish/requirements.in -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via # requests # twine diff --git a/tools/publish/requirements_windows.txt b/tools/publish/requirements_windows.txt index 128c0b56b6..95b88339c7 100644 --- a/tools/publish/requirements_windows.txt +++ b/tools/publish/requirements_windows.txt @@ -106,9 +106,9 @@ docutils==0.19 \ --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc # via readme-renderer -idna==3.4 \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests importlib-metadata==6.0.0 \ --hash=sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad \ @@ -180,9 +180,9 @@ twine==4.0.2 \ --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 # via -r tools/publish/requirements.in -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via # requests # twine