Skip to content

Commit e8039db

Browse files
aignasaignas
authored
feat(twine): support 'bzlmod' users out of the box (bazel-contrib#1572)
Implements a test that starts a [`pypiserver`] and checks that the publishing with the new machinery still works. Fixes bazel-contrib#1369 [pypiserver]: https://github.com/pypiserver/pypiserver
1 parent c5c03b2 commit e8039db

16 files changed

Lines changed: 305 additions & 37 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ A brief description of the categories of changes:
4343
* (wheel) Add support for `data_files` attributes in py_wheel rule
4444
([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
4545

46+
* (py_wheel) `bzlmod` installations now provide a `twine` setup for the default
47+
Python toolchain in `rules_python` for version 3.11.
48+
4649
[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
4750
[python_default_visibility]: gazelle/README.md#directive-python_default_visibility
4851

@@ -269,7 +272,6 @@ A brief description of the categories of changes:
269272
attribute for every target in the package. This is enabled through a separate
270273
directive `python_generation_mode_per_file_include_init`.
271274

272-
273275
## [0.27.0] - 2023-11-16
274276

275277
[0.27.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.27.0

‎MODULE.bazel‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,24 @@ python.toolchain(
4848
is_default = True,
4949
python_version = "3.11",
5050
)
51-
use_repo(python, "pythons_hub")
51+
use_repo(python, "python_versions", "pythons_hub")
5252

5353
# This call registers the Python toolchains.
5454
register_toolchains("@pythons_hub//:all")
5555

56+
#####################
57+
# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
58+
59+
pip = use_extension("//python/extensions:pip.bzl", "pip")
60+
pip.parse(
61+
hub_name = "rules_python_publish_deps",
62+
python_version = "3.11",
63+
requirements_darwin = "//tools/publish:requirements_darwin.txt",
64+
requirements_lock = "//tools/publish:requirements.txt",
65+
requirements_windows = "//tools/publish:requirements_windows.txt",
66+
)
67+
use_repo(pip, "rules_python_publish_deps")
68+
5669
# ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
5770
bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
5871
bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
@@ -84,24 +97,12 @@ dev_pip.parse(
8497
python_version = "3.11",
8598
requirements_lock = "//docs/sphinx:requirements.txt",
8699
)
87-
88-
#####################
89-
# Install twine for our own runfiles wheel publishing.
90-
# Eventually we might want to install twine automatically for users too, see:
91-
# https://github.com/bazelbuild/rules_python/issues/1016.
92-
93100
dev_pip.parse(
94-
hub_name = "publish_deps",
101+
hub_name = "pypiserver",
95102
python_version = "3.11",
96-
requirements_darwin = "//tools/publish:requirements_darwin.txt",
97-
requirements_lock = "//tools/publish:requirements.txt",
98-
requirements_windows = "//tools/publish:requirements_windows.txt",
99-
)
100-
use_repo(
101-
dev_pip,
102-
"dev_pip",
103-
publish_deps_twine = "publish_deps_311_twine",
103+
requirements_lock = "//examples/wheel:requirements_server.txt",
104104
)
105+
use_repo(dev_pip, "dev_pip", "pypiserver")
105106

106107
# Bazel integration test setup below
107108

‎WORKSPACE‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,27 @@ load("@python//3.11.8:defs.bzl", "interpreter")
9494
load("@rules_python//python:pip.bzl", "pip_parse")
9595

9696
pip_parse(
97-
name = "publish_deps",
97+
name = "rules_python_publish_deps",
9898
python_interpreter_target = interpreter,
9999
requirements_darwin = "//tools/publish:requirements_darwin.txt",
100100
requirements_lock = "//tools/publish:requirements.txt",
101101
requirements_windows = "//tools/publish:requirements_windows.txt",
102102
)
103103

104-
load("@publish_deps//:requirements.bzl", "install_deps")
104+
load("@rules_python_publish_deps//:requirements.bzl", "install_deps")
105105

106106
install_deps()
107107

108+
pip_parse(
109+
name = "pypiserver",
110+
python_interpreter_target = interpreter,
111+
requirements_lock = "//examples/wheel:requirements_server.txt",
112+
)
113+
114+
load("@pypiserver//:requirements.bzl", install_pypiserver = "install_deps")
115+
116+
install_pypiserver()
117+
108118
#####################
109119
# Install sphinx for doc generation.
110120

‎examples/wheel/BUILD.bazel‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
1717
load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags")
1818
load("//python:defs.bzl", "py_library", "py_test")
1919
load("//python:packaging.bzl", "py_package", "py_wheel")
20+
load("//python:pip.bzl", "compile_pip_requirements")
2021
load("//python:versions.bzl", "gen_python_config_settings")
22+
load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
23+
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
2124

2225
package(default_visibility = ["//visibility:public"])
2326

@@ -56,6 +59,10 @@ py_wheel(
5659
# Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
5760
distribution = "example_minimal_library",
5861
python_tag = "py3",
62+
# NOTE: twine_binary = "//tools/publish:twine" does not work on non-bzlmod
63+
# setups because the `//tools/publish:twine` produces multiple files and is
64+
# unsuitable as the `src` to the underlying native_binary rule.
65+
twine = None if BZLMOD_ENABLED else "@rules_python_publish_deps_twine//:pkg",
5966
version = "0.0.1",
6067
deps = [
6168
"//examples/wheel/lib:module_with_data",
@@ -348,3 +355,39 @@ py_test(
348355
"//python/runfiles",
349356
],
350357
)
358+
359+
# Test wheel publishing
360+
361+
compile_pip_requirements(
362+
name = "requirements_server",
363+
src = "requirements_server.in",
364+
)
365+
366+
py_test(
367+
name = "test_publish",
368+
srcs = ["test_publish.py"],
369+
data = [
370+
":minimal_with_py_library",
371+
":minimal_with_py_library.publish",
372+
":pypiserver",
373+
],
374+
env = {
375+
"PUBLISH_PATH": "$(location :minimal_with_py_library.publish)",
376+
"SERVER_PATH": "$(location :pypiserver)",
377+
"WHEEL_PATH": "$(rootpath :minimal_with_py_library)",
378+
},
379+
target_compatible_with = select({
380+
"@platforms//os:linux": [],
381+
"@platforms//os:macos": [],
382+
"//conditions:default": ["@platforms//:incompatible"],
383+
}),
384+
deps = [
385+
"@pypiserver//pypiserver",
386+
],
387+
)
388+
389+
py_console_script_binary(
390+
name = "pypiserver",
391+
pkg = "@pypiserver//pypiserver",
392+
script = "pypi-server",
393+
)

‎examples/wheel/requirements_server.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# This is for running publishing tests
2+
pypiserver

‎examples/wheel/requirements_server.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
4+
#
5+
# bazel run //examples/wheel:requirements_server.update
6+
#
7+
pypiserver==2.0.1 \
8+
--hash=sha256:1dd98fb99d2da4199fb44c7284e57d69a9f7fda2c6c8dc01975c151c592677bf \
9+
--hash=sha256:7b58fbd54468235f79e4de07c4f7a9ff829e7ac6869bef47ec11e0710138e162
10+
# via -r examples/wheel/requirements_server.in
11+
12+
# The following packages are considered to be unsafe in a requirements file:
13+
pip==24.0 \
14+
--hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
15+
--hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
16+
# via pypiserver

‎examples/wheel/test_publish.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import os
2+
import socket
3+
import subprocess
4+
import textwrap
5+
import time
6+
import unittest
7+
from contextlib import closing
8+
from pathlib import Path
9+
from urllib.request import urlopen
10+
11+
12+
def find_free_port():
13+
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
14+
s.bind(("", 0))
15+
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
16+
return s.getsockname()[1]
17+
18+
19+
class TestTwineUpload(unittest.TestCase):
20+
def setUp(self):
21+
self.maxDiff = 1000
22+
self.port = find_free_port()
23+
self.url = f"http://localhost:{self.port}"
24+
self.dir = Path(os.environ["TEST_TMPDIR"])
25+
26+
self.log_file = self.dir / "pypiserver-log.txt"
27+
self.log_file.touch()
28+
_storage_dir = self.dir / "data"
29+
for d in [_storage_dir]:
30+
d.mkdir(exist_ok=True)
31+
32+
print("Starting PyPI server...")
33+
self._server = subprocess.Popen(
34+
[
35+
str(Path(os.environ["SERVER_PATH"])),
36+
"run",
37+
"--verbose",
38+
"--log-file",
39+
str(self.log_file),
40+
"--host",
41+
"localhost",
42+
"--port",
43+
str(self.port),
44+
# Allow unauthenticated access
45+
"--authenticate",
46+
".",
47+
"--passwords",
48+
".",
49+
str(_storage_dir),
50+
],
51+
)
52+
53+
line = "Hit Ctrl-C to quit"
54+
interval = 0.1
55+
wait_seconds = 40
56+
for _ in range(int(wait_seconds / interval)): # 40 second timeout
57+
current_logs = self.log_file.read_text()
58+
if line in current_logs:
59+
print(current_logs.strip())
60+
print("...")
61+
break
62+
63+
time.sleep(0.1)
64+
else:
65+
raise RuntimeError(
66+
f"Could not get the server running fast enough, waited for {wait_seconds}s"
67+
)
68+
69+
def tearDown(self):
70+
self._server.terminate()
71+
print(f"Stopped PyPI server, all logs:\n{self.log_file.read_text()}")
72+
73+
def test_upload_and_query_simple_api(self):
74+
# Given
75+
script_path = Path(os.environ["PUBLISH_PATH"])
76+
whl = Path(os.environ["WHEEL_PATH"])
77+
78+
# When I publish a whl to a package registry
79+
subprocess.check_output(
80+
[
81+
str(script_path),
82+
"--no-color",
83+
"upload",
84+
str(whl),
85+
"--verbose",
86+
"--non-interactive",
87+
"--disable-progress-bar",
88+
],
89+
env={
90+
"TWINE_REPOSITORY_URL": self.url,
91+
"TWINE_USERNAME": "dummy",
92+
"TWINE_PASSWORD": "dummy",
93+
},
94+
)
95+
96+
# Then I should be able to get its contents
97+
with urlopen(self.url + "/example-minimal-library/") as response:
98+
got_content = response.read().decode("utf-8")
99+
want_content = """
100+
<!DOCTYPE html>
101+
<html>
102+
<head>
103+
<title>Links for example-minimal-library</title>
104+
</head>
105+
<body>
106+
<h1>Links for example-minimal-library</h1>
107+
<a href="/packages/example_minimal_library-0.0.1-py3-none-any.whl#sha256=79a4e9c1838c0631d5d8fa49a26efd6e9a364f6b38d9597c0f6df112271a0e28">example_minimal_library-0.0.1-py3-none-any.whl</a><br>
108+
</body>
109+
</html>"""
110+
self.assertEqual(
111+
textwrap.dedent(want_content).strip(),
112+
textwrap.dedent(got_content).strip(),
113+
)
114+
115+
116+
if __name__ == "__main__":
117+
unittest.main()

‎python/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,13 @@ bzl_library(
7676
srcs = ["packaging.bzl"],
7777
deps = [
7878
":py_binary_bzl",
79+
"//python/private:bzlmod_enabled_bzl",
7980
"//python/private:py_package.bzl",
8081
"//python/private:py_wheel_bzl",
8182
"//python/private:py_wheel_normalize_pep440.bzl",
8283
"//python/private:stamp_bzl",
8384
"//python/private:util_bzl",
85+
"@bazel_skylib//rules:native_binary",
8486
],
8587
)
8688

0 commit comments

Comments
 (0)