From 058f3aeefe03d52e4e73ef1f5ee785f91bfbef6a Mon Sep 17 00:00:00 2001 From: Tony Liang Date: Fri, 8 Jun 2018 10:21:11 -0700 Subject: [PATCH 1/7] Print output of pip steps --- python/pip.bzl | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/python/pip.bzl b/python/pip.bzl index 9d27794e4d..45629952d2 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -23,18 +23,21 @@ def _pip_import_impl(repository_ctx): repository_ctx.file("BUILD", "") # To see the output, pass: quiet=False - result = repository_ctx.execute([ - "python", - repository_ctx.path(repository_ctx.attr._script), - "--name", - repository_ctx.attr.name, - "--input", - repository_ctx.path(repository_ctx.attr.requirements), - "--output", - repository_ctx.path("requirements.bzl"), - "--directory", - repository_ctx.path(""), - ]) + result = repository_ctx.execute( + [ + "python", + repository_ctx.path(repository_ctx.attr._script), + "--name", + repository_ctx.attr.name, + "--input", + repository_ctx.path(repository_ctx.attr.requirements), + "--output", + repository_ctx.path("requirements.bzl"), + "--directory", + repository_ctx.path(""), + ], + quiet=False + ) if result.return_code: fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) From 867459238df396a765803877772097a5fc66b558 Mon Sep 17 00:00:00 2001 From: Jon Brandvein Date: Wed, 3 Apr 2019 17:13:57 -0400 Subject: [PATCH 2/7] Mark Python toolchain proposal as formally accepted (#166) --- proposals/2019-02-12-design-for-a-python-toolchain.md | 5 +++-- proposals/README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/proposals/2019-02-12-design-for-a-python-toolchain.md b/proposals/2019-02-12-design-for-a-python-toolchain.md index 2002384afd..0d45866107 100644 --- a/proposals/2019-02-12-design-for-a-python-toolchain.md +++ b/proposals/2019-02-12-design-for-a-python-toolchain.md @@ -1,8 +1,8 @@ --- title: Design for a Python Toolchain -status: In review +status: Accepted created: 2019-02-12 -updated: 2019-02-15 +updated: 2019-02-21 authors: - [brandjon@](https://github.com/brandjon) reviewers: @@ -244,3 +244,4 @@ Date | Change 2019-02-12 | Initial version 2019-02-14 | Make `PyRuntimeInfo` natively defined 2019-02-15 | Clarify platform runtime vs in-build runtime +2019-02-21 | Formal approval diff --git a/proposals/README.md b/proposals/README.md index 77b87ecb6e..36a8a0b7d3 100644 --- a/proposals/README.md +++ b/proposals/README.md @@ -6,6 +6,6 @@ Proposals that impact native code are also indexed by [bazelbuild/proposals](htt Last updated | Status | Title | Author(s) ------------ | ------------- | ------| --------- -2019-02-15 | Draft | [Design for a Python Toolchain](https://github.com/bazelbuild/rules_python/blob/master/proposals/2019-02-12-design-for-a-python-toolchain.md) | [brandjon@](https://github.com/brandjon) +2019-02-21 | Accepted | [Design for a Python Toolchain](https://github.com/bazelbuild/rules_python/blob/master/proposals/2019-02-12-design-for-a-python-toolchain.md) | [brandjon@](https://github.com/brandjon) 2018-11-09 | Draft | [Customizing the Python Stub Template](https://github.com/bazelbuild/rules_python/blob/master/proposals/2018-11-08-customizing-the-python-stub-template.md) | [brandjon@](https://github.com/brandjon) 2019-01-11 | Accepted | [Selecting Between Python 2 and 3](https://github.com/bazelbuild/rules_python/blob/master/proposals/2018-10-25-selecting-between-python-2-and-3.md) | [brandjon@](https://github.com/brandjon) From f288195e317c8c6d17a16fedf8d859f64b116c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Stradomski?= <44680433+pstradomski@users.noreply.github.com> Date: Tue, 9 Apr 2019 14:27:13 +0200 Subject: [PATCH 3/7] Add experimental support for building wheels. (#159) Add experimental support for building wheels. Currently only pure python wheels are supported. This extension builds wheels directly, invoking a simple python script that creates the zip archive in the desired format instead of using distutils/setuptools. This will make building platform-dependent wheels easier in the future, as bazel will have full control on how extension code is built. --- experimental/BUILD | 16 + experimental/examples/wheel/BUILD | 96 ++++++ experimental/examples/wheel/README.md | 1 + experimental/examples/wheel/lib/BUILD | 35 +++ .../examples/wheel/lib/module_with_data.py | 16 + .../examples/wheel/lib/simple_module.py | 16 + experimental/examples/wheel/main.py | 30 ++ experimental/examples/wheel/wheel_test.py | 107 +++++++ experimental/python/BUILD | 18 ++ experimental/python/wheel.bzl | 273 ++++++++++++++++ experimental/rules_python/BUILD | 21 ++ experimental/rules_python/wheelmaker.py | 293 ++++++++++++++++++ 12 files changed, 922 insertions(+) create mode 100644 experimental/BUILD create mode 100644 experimental/examples/wheel/BUILD create mode 100644 experimental/examples/wheel/README.md create mode 100644 experimental/examples/wheel/lib/BUILD create mode 100644 experimental/examples/wheel/lib/module_with_data.py create mode 100644 experimental/examples/wheel/lib/simple_module.py create mode 100644 experimental/examples/wheel/main.py create mode 100644 experimental/examples/wheel/wheel_test.py create mode 100644 experimental/python/BUILD create mode 100644 experimental/python/wheel.bzl create mode 100644 experimental/rules_python/BUILD create mode 100644 experimental/rules_python/wheelmaker.py diff --git a/experimental/BUILD b/experimental/BUILD new file mode 100644 index 0000000000..a892ac193a --- /dev/null +++ b/experimental/BUILD @@ -0,0 +1,16 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 diff --git a/experimental/examples/wheel/BUILD b/experimental/examples/wheel/BUILD new file mode 100644 index 0000000000..d7ed5da49c --- /dev/null +++ b/experimental/examples/wheel/BUILD @@ -0,0 +1,96 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +load("//python:python.bzl", "py_library", "py_test") +load("//experimental/python:wheel.bzl", "py_package", "py_wheel") + +py_library( + name = "main", + srcs = ["main.py"], + deps = [ + "//experimental/examples/wheel/lib:simple_module", + "//experimental/examples/wheel/lib:module_with_data", + # Example dependency which is not packaged in the wheel + # due to "packages" filter on py_package rule. + "//examples/helloworld", + ], +) + +# Package just a specific py_libraries, without their dependencies +py_wheel( + name = "minimal_with_py_library", + # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" + distribution = "example_minimal_library", + python_tag = "py3", + version = "0.0.1", + deps = [ + "//experimental/examples/wheel/lib:module_with_data", + "//experimental/examples/wheel/lib:simple_module", + ], +) + +# Use py_package to collect all transitive dependencies of a target, +# selecting just the files within a specific python package. +py_package( + name = "example_pkg", + # Only include these Python packages. + packages = ["experimental.examples.wheel"], + deps = [":main"], +) + +py_wheel( + name = "minimal_with_py_package", + # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" + distribution = "example_minimal_package", + python_tag = "py3", + version = "0.0.1", + deps = [":example_pkg"], +) + +# An example that uses all features provided by py_wheel. +py_wheel( + name = "customized", + author = "Example Author with non-ascii characters: żółw", + author_email = "example@example.com", + classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + ], + console_scripts = { + "customized_wheel": "experimental.examples.wheel.main:main", + }, + description_file = "README.md", + # Package data. We're building "example_customized-0.0.1-py3-none-any.whl" + distribution = "example_customized", + homepage = "www.example.com", + license = "Apache 2.0", + python_tag = "py3", + # Requirements embedded into the wheel metadata. + requires = ["pytest"], + version = "0.0.1", + deps = [":example_pkg"], +) + +py_test( + name = "wheel_test", + srcs = ["wheel_test.py"], + data = [ + ":customized", + ":minimal_with_py_library", + ":minimal_with_py_package", + ], +) diff --git a/experimental/examples/wheel/README.md b/experimental/examples/wheel/README.md new file mode 100644 index 0000000000..1426ff4792 --- /dev/null +++ b/experimental/examples/wheel/README.md @@ -0,0 +1 @@ +This is a sample description of a wheel. \ No newline at end of file diff --git a/experimental/examples/wheel/lib/BUILD b/experimental/examples/wheel/lib/BUILD new file mode 100644 index 0000000000..6b01b1bf55 --- /dev/null +++ b/experimental/examples/wheel/lib/BUILD @@ -0,0 +1,35 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +load("//python:python.bzl", "py_library") + +py_library( + name = "simple_module", + srcs = ["simple_module.py"], +) + +py_library( + name = "module_with_data", + srcs = ["module_with_data.py"], + data = [":data.txt"], +) + +genrule( + name = "make_data", + outs = ["data.txt"], + cmd = "echo foo bar baz > $@", +) diff --git a/experimental/examples/wheel/lib/module_with_data.py b/experimental/examples/wheel/lib/module_with_data.py new file mode 100644 index 0000000000..7b28643c6e --- /dev/null +++ b/experimental/examples/wheel/lib/module_with_data.py @@ -0,0 +1,16 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def function(): + return "foo" diff --git a/experimental/examples/wheel/lib/simple_module.py b/experimental/examples/wheel/lib/simple_module.py new file mode 100644 index 0000000000..fb26a5180e --- /dev/null +++ b/experimental/examples/wheel/lib/simple_module.py @@ -0,0 +1,16 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def function(): + return "bar" diff --git a/experimental/examples/wheel/main.py b/experimental/examples/wheel/main.py new file mode 100644 index 0000000000..db16826c2f --- /dev/null +++ b/experimental/examples/wheel/main.py @@ -0,0 +1,30 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import experimental.examples.wheel.lib.module_with_data as module_with_data +import experimental.examples.wheel.lib.simple_module as simple_module + + +def function(): + return "baz" + + +def main(): + print(function()) + print(module_with_data.function()) + print(simple_module.function()) + + +if __name__ == '__main__': + main() diff --git a/experimental/examples/wheel/wheel_test.py b/experimental/examples/wheel/wheel_test.py new file mode 100644 index 0000000000..278f00726a --- /dev/null +++ b/experimental/examples/wheel/wheel_test.py @@ -0,0 +1,107 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +import zipfile + + +class WheelTest(unittest.TestCase): + def test_py_library_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_minimal_library-0.0.1-py3-none-any.whl') + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['experimental/examples/wheel/lib/module_with_data.py', + 'experimental/examples/wheel/lib/simple_module.py', + 'example_minimal_library-0.0.1.dist-info/WHEEL', + 'example_minimal_library-0.0.1.dist-info/METADATA', + 'example_minimal_library-0.0.1.dist-info/RECORD']) + + def test_py_package_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_minimal_package-0.0.1-py3-none-any.whl') + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['experimental/examples/wheel/lib/data.txt', + 'experimental/examples/wheel/lib/module_with_data.py', + 'experimental/examples/wheel/lib/simple_module.py', + 'experimental/examples/wheel/main.py', + 'example_minimal_package-0.0.1.dist-info/WHEEL', + 'example_minimal_package-0.0.1.dist-info/METADATA', + 'example_minimal_package-0.0.1.dist-info/RECORD']) + + def test_customized_wheel(self): + filename = os.path.join(os.environ['TEST_SRCDIR'], + 'io_bazel_rules_python', 'experimental', + 'examples', 'wheel', + 'example_customized-0.0.1-py3-none-any.whl') + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + ['experimental/examples/wheel/lib/data.txt', + 'experimental/examples/wheel/lib/module_with_data.py', + 'experimental/examples/wheel/lib/simple_module.py', + 'experimental/examples/wheel/main.py', + 'example_customized-0.0.1.dist-info/WHEEL', + 'example_customized-0.0.1.dist-info/METADATA', + 'example_customized-0.0.1.dist-info/entry_points.txt', + 'example_customized-0.0.1.dist-info/RECORD']) + record_contents = zf.read( + 'example_customized-0.0.1.dist-info/RECORD') + wheel_contents = zf.read( + 'example_customized-0.0.1.dist-info/WHEEL') + metadata_contents = zf.read( + 'example_customized-0.0.1.dist-info/METADATA') + # The entries are guaranteed to be sorted. + self.assertEquals(record_contents, b"""\ +example_customized-0.0.1.dist-info/METADATA,sha256=TeeEmokHE2NWjkaMcVJuSAq4_AXUoIad2-SLuquRmbg,372 +example_customized-0.0.1.dist-info/RECORD,, +example_customized-0.0.1.dist-info/WHEEL,sha256=F01lGfVCzcXUzzQHzUkBmXAcu_TXd5zqMLrvrspncJo,85 +example_customized-0.0.1.dist-info/entry_points.txt,sha256=olLJ8FK88aft2pcdj4BD05F8Xyz83Mo51I93tRGT2Yk,74 +experimental/examples/wheel/lib/data.txt,sha256=9vJKEdfLu8bZRArKLroPZJh1XKkK3qFMXiM79MBL2Sg,12 +experimental/examples/wheel/lib/module_with_data.py,sha256=K_IGAq_CHcZX0HUyINpD1hqSKIEdCn58d9E9nhWF2EA,636 +experimental/examples/wheel/lib/simple_module.py,sha256=72-91Dm6NB_jw-7wYQt7shzdwvk5RB0LujIah8g7kr8,636 +experimental/examples/wheel/main.py,sha256=E0xCyiPg6fCo4IrFmqo_tqpNGtk1iCewobqD0_KlFd0,935 +""") + self.assertEquals(wheel_contents, b"""\ +Wheel-Version: 1.0 +Generator: wheelmaker 1.0 +Root-Is-Purelib: true +Tag: py3-none-any +""") + self.assertEquals(metadata_contents, b"""\ +Metadata-Version: 2.1 +Name: example_customized +Version: 0.0.1 +Author: Example Author with non-ascii characters: \xc5\xbc\xc3\xb3\xc5\x82w +Author-email: example@example.com +Home-page: www.example.com +License: Apache 2.0 +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Requires-Dist: pytest + +This is a sample description of a wheel. +""") + + +if __name__ == '__main__': + unittest.main() diff --git a/experimental/python/BUILD b/experimental/python/BUILD new file mode 100644 index 0000000000..8e8a059006 --- /dev/null +++ b/experimental/python/BUILD @@ -0,0 +1,18 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["wheel.bzl"]) diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl new file mode 100644 index 0000000000..7603d93d51 --- /dev/null +++ b/experimental/python/wheel.bzl @@ -0,0 +1,273 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules for building wheels.""" + +def _path_inside_wheel(input_file): + # input_file.short_path is relative ("../${repository_root}/foobar") + # so it can't be a valid path within a zip file. Thus strip out the root + # manually instead of using short_path here. + root = input_file.root.path + if root != "": + # TODO: '/' is wrong on windows, but the path separator is not available in skylark. + # Fix this once ctx.configuration has directory separator information. + root += "/" + if not input_file.path.startswith(root): + fail("input_file.path '%s' does not start with expected root '%s'" % (input_file.path, root)) + return input_file.path[len(root):] + +def _py_package_impl(ctx): + inputs = depset( + transitive = [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.deps] + + [dep[DefaultInfo].default_runfiles.files for dep in ctx.attr.deps], + ) + + # TODO: '/' is wrong on windows, but the path separator is not available in skylark. + # Fix this once ctx.configuration has directory separator information. + packages = [p.replace(".", "/") for p in ctx.attr.packages] + if not packages: + filtered_inputs = inputs + else: + filtered_files = [] + + # TODO: flattening depset to list gives poor performance, + for input_file in inputs.to_list(): + wheel_path = _path_inside_wheel(input_file) + for package in packages: + if wheel_path.startswith(package): + filtered_files.append(input_file) + filtered_inputs = depset(direct = filtered_files) + + return [DefaultInfo( + files = filtered_inputs, + )] + +py_package = rule( + implementation = _py_package_impl, + doc = """ +A rule to select all files in transitive dependencies of deps which +belong to given set of Python packages. + +This rule is intended to be used as data dependency to py_wheel rule +""", + attrs = { + "deps": attr.label_list(), + "packages": attr.string_list( + mandatory = False, + allow_empty = True, + doc = """\ +List of Python packages to include in the distribution. +Sub-packages are automatically included. +""", + ), + }, +) + +def _py_wheel_impl(ctx): + outfile = ctx.actions.declare_file("-".join([ + ctx.attr.distribution, + ctx.attr.version, + ctx.attr.python_tag, + ctx.attr.abi, + ctx.attr.platform, + ]) + ".whl") + + inputs = depset( + direct = ctx.files.deps, + ) + + arguments = [ + "--name", + ctx.attr.distribution, + "--version", + ctx.attr.version, + "--python_tag", + ctx.attr.python_tag, + "--abi", + ctx.attr.abi, + "--platform", + ctx.attr.platform, + "--out", + outfile.path, + ] + + # TODO: Use args api instead of flattening the depset. + for input_file in inputs.to_list(): + arguments.append("--input_file") + arguments.append("%s;%s" % (_path_inside_wheel(input_file), input_file.path)) + + extra_headers = [] + if ctx.attr.author: + extra_headers.append("Author: %s" % ctx.attr.author) + if ctx.attr.author_email: + extra_headers.append("Author-email: %s" % ctx.attr.author_email) + if ctx.attr.homepage: + extra_headers.append("Home-page: %s" % ctx.attr.homepage) + if ctx.attr.license: + extra_headers.append("License: %s" % ctx.attr.license) + + for h in extra_headers: + arguments.append("--header") + arguments.append(h) + + for c in ctx.attr.classifiers: + arguments.append("--classifier") + arguments.append(c) + + for r in ctx.attr.requires: + arguments.append("--requires") + arguments.append(r) + + for option, requirements in ctx.attr.extra_requires.items(): + for r in requirements: + arguments.append("--extra_requires") + arguments.append(r + ";" + option) + + for name, ref in ctx.attr.console_scripts.items(): + arguments.append("--console_script") + arguments.append(name + " = " + ref) + + if ctx.attr.description_file: + description_file = ctx.file.description_file + arguments.append("--description_file") + arguments.append(description_file.path) + inputs = inputs.union([description_file]) + + ctx.actions.run( + inputs = inputs, + outputs = [outfile], + arguments = arguments, + executable = ctx.executable._wheelmaker, + progress_message = "Building wheel", + ) + return [DefaultInfo( + files = depset([outfile]), + data_runfiles = ctx.runfiles(files = [outfile]), + )] + +py_wheel = rule( + implementation = _py_wheel_impl, + doc = """ +A rule for building Python Wheels. + +Wheels are Python distribution format defined in https://www.python.org/dev/peps/pep-0427/. + +This rule packages a set of targets into a single wheel. + +Currently only pure-python wheels are supported. + +Examples: + + +# Package just a specific py_libraries, without their dependencies +py_wheel( + name = "minimal_with_py_library", + # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" + distribution = "example_minimal_library", + python_tag = "py3", + version = "0.0.1", + deps = [ + "//experimental/examples/wheel/lib:module_with_data", + "//experimental/examples/wheel/lib:simple_module", + ], +) + +# Use py_package to collect all transitive dependencies of a target, +# selecting just the files within a specific python package. +py_package( + name = "example_pkg", + # Only include these Python packages. + packages = ["experimental.examples.wheel"], + deps = [":main"], +) + +py_wheel( + name = "minimal_with_py_package", + # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" + distribution = "example_minimal_package", + python_tag = "py3", + version = "0.0.1", + deps = [":example_pkg"], +) + +""", + attrs = { + "deps": attr.label_list( + doc = """\ +Targets to be included in the distribution. + +The targets to package are usually `py_library` rules or filesets (for packaging data files). + +Note it's usually better to package `py_library` targets and use +`console_scripts` attribute to specify entry points than to package +`py_binary` rules. `py_binary` targets would wrap a executable script that +tries to locate `.runfiles` directory which is not packaged in the wheel. +""", + ), + # Attributes defining the distribution + "distribution": attr.string( + mandatory = True, + doc = """ +Name of the distribution. + +This should match the project name onm PyPI. It's also the name that is used +to refer to the package in other packages' dependencies. +""", + ), + "version": attr.string( + mandatory = True, + doc = "Version number of the package", + ), + "python_tag": attr.string( + default = "py3", + doc = "Supported Python major version. 'py2' or 'py3'", + values = ["py2", "py3"], + ), + "abi": attr.string( + default = "none", + doc = "Python ABI tag. 'none' for pure-Python wheels.", + ), + # TODO(pstradomski): Support non-pure wheels + "platform": attr.string( + default = "any", + doc = "Supported platforms. 'any' for pure-Python wheel.", + ), + # Other attributes + "author": attr.string(default = ""), + "author_email": attr.string(default = ""), + "homepage": attr.string(default = ""), + "license": attr.string(default = ""), + "classifiers": attr.string_list(), + "description_file": attr.label(allow_single_file = True), + # Requirements + "requires": attr.string_list( + doc = "List of requirements for this package", + ), + "extra_requires": attr.string_list_dict( + doc = "List of optional requirements for this package", + ), + # Entry points + "console_scripts": attr.string_dict( + doc = """\ +console_script entry points, e.g. 'experimental.examples.wheel.main:main'. +""", + ), + # Implementation details. + "_wheelmaker": attr.label( + executable = True, + cfg = "host", + default = "//experimental/rules_python:wheelmaker", + ), + }, +) diff --git a/experimental/rules_python/BUILD b/experimental/rules_python/BUILD new file mode 100644 index 0000000000..5f770a146e --- /dev/null +++ b/experimental/rules_python/BUILD @@ -0,0 +1,21 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//python:python.bzl", "py_binary") + +py_binary( + name = "wheelmaker", + srcs = ["wheelmaker.py"], + visibility = ["//visibility:public"], +) diff --git a/experimental/rules_python/wheelmaker.py b/experimental/rules_python/wheelmaker.py new file mode 100644 index 0000000000..7a50ae4a20 --- /dev/null +++ b/experimental/rules_python/wheelmaker.py @@ -0,0 +1,293 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import collections +import csv +import hashlib +import io +import os +import os.path +import sys +import zipfile + + +def commonpath(path1, path2): + ret = [] + for a, b in zip(path1.split(os.path.sep), path2.split(os.path.sep)): + if a != b: + break + ret.append(a) + return os.path.sep.join(ret) + + +class WheelMaker(object): + def __init__(self, name, version, build_tag, python_tag, abi, platform, + outfile=None): + self._name = name + self._version = version + self._build_tag = build_tag + self._python_tag = python_tag + self._abi = abi + self._platform = platform + self._outfile = outfile + + self._zipfile = None + self._record = [] + + def __enter__(self): + self._zipfile = zipfile.ZipFile(self.filename(), mode="w", + compression=zipfile.ZIP_DEFLATED) + return self + + def __exit__(self, type, value, traceback): + self._zipfile.close() + self._zipfile = None + + def filename(self): + if self._outfile: + return self._outfile + components = [self._name, self._version] + if self._build_tag: + components.append(self._build_tag) + components += [self._python_tag, self._abi, self._platform] + return '-'.join(components) + '.whl' + + def distname(self): + return self._name + '-' + self._version + + def disttags(self): + return ['-'.join([self._python_tag, self._abi, self._platform])] + + def distinfo_path(self, basename): + return self.distname() + '.dist-info/' + basename + + def _serialize_digest(self, hash): + # https://www.python.org/dev/peps/pep-0376/#record + # "base64.urlsafe_b64encode(digest) with trailing = removed" + digest = base64.urlsafe_b64encode(hash.digest()) + digest = b'sha256=' + digest.rstrip(b'=') + return digest + + def add_string(self, filename, contents): + """Add given 'contents' as filename to the distribution.""" + if sys.version_info[0] > 2 and isinstance(contents, str): + contents = contents.encode('utf-8', 'surrogateescape') + self._zipfile.writestr(filename, contents) + hash = hashlib.sha256() + hash.update(contents) + self._add_to_record(filename, self._serialize_digest(hash), + len(contents)) + + def add_file(self, package_filename, real_filename): + """Add given file to the distribution.""" + # Always use unix path separators. + arcname = package_filename.replace(os.path.sep, '/') + self._zipfile.write(real_filename, arcname=arcname) + # Find the hash and length + hash = hashlib.sha256() + size = 0 + with open(real_filename, 'rb') as f: + while True: + block = f.read(2 ** 20) + if not block: + break + hash.update(block) + size += len(block) + self._add_to_record(arcname, self._serialize_digest(hash), size) + + def add_wheelfile(self): + """Write WHEEL file to the distribution""" + # TODO(pstradomski): Support non-purelib wheels. + wheel_contents = """\ +Wheel-Version: 1.0 +Generator: wheelmaker 1.0 +Root-Is-Purelib: true +""" + for tag in self.disttags(): + wheel_contents += "Tag: %s\n" % tag + self.add_string(self.distinfo_path('WHEEL'), wheel_contents) + + def add_metadata(self, extra_headers, description, classifiers, requires, + extra_requires): + """Write METADATA file to the distribution.""" + # https://www.python.org/dev/peps/pep-0566/ + # https://packaging.python.org/specifications/core-metadata/ + metadata = [] + metadata.append("Metadata-Version: 2.1") + metadata.append("Name: %s" % self._name) + metadata.append("Version: %s" % self._version) + metadata.extend(extra_headers) + for classifier in classifiers: + metadata.append("Classifier: %s" % classifier) + for requirement in requires: + metadata.append("Requires-Dist: %s" % requirement) + + extra_requires = sorted(extra_requires.items()) + for option, option_requires in extra_requires: + metadata.append("Provides-Extra: %s" % option) + for requirement in option_requires: + metadata.append( + "Requires-Dist: %s; extra == '%s'" % (requirement, option)) + + metadata = '\n'.join(metadata) + '\n\n' + # setuptools seems to insert UNKNOWN as description when none is + # provided. + metadata += description if description else "UNKNOWN" + metadata += "\n" + self.add_string(self.distinfo_path('METADATA'), metadata) + + def add_entry_points(self, console_scripts): + """Write entry_points.txt file to the distribution.""" + # https://packaging.python.org/specifications/entry-points/ + if not console_scripts: + return + lines = ["[console_scripts]"] + console_scripts + contents = '\n'.join(lines) + self.add_string(self.distinfo_path('entry_points.txt'), contents) + + def add_recordfile(self): + """Write RECORD file to the distribution.""" + record_path = self.distinfo_path('RECORD') + entries = self._record + [(record_path, b'', b'')] + entries.sort() + contents = b'' + for filename, digest, size in entries: + if sys.version_info[0] > 2 and isinstance(filename, str): + filename = filename.encode('utf-8', 'surrogateescape') + contents += b'%s,%s,%s\n' % (filename, digest, size) + self.add_string(record_path, contents) + + def _add_to_record(self, filename, hash, size): + size = str(size).encode('ascii') + self._record.append((filename, hash, size)) + + +def get_files_to_package(input_files): + """Find files to be added to the distribution. + + input_files: list of pairs (package_path, real_path) + """ + files = {} + for package_path, real_path in input_files: + files[package_path] = real_path + return files + + +def main(): + parser = argparse.ArgumentParser(description='Builds a python wheel') + metadata_group = parser.add_argument_group( + "Wheel name, version and platform") + metadata_group.add_argument('--name', required=True, + type=str, + help="Name of the distribution") + metadata_group.add_argument('--version', required=True, + type=str, + help="Version of the distribution") + metadata_group.add_argument('--build_tag', type=str, default='', + help="Optional build tag for the distribution") + metadata_group.add_argument('--python_tag', type=str, default='py3', + help="Python version, e.g. 'py2' or 'py3'") + metadata_group.add_argument('--abi', type=str, default='none') + metadata_group.add_argument('--platform', type=str, default='any', + help="Target platform. ") + + output_group = parser.add_argument_group("Output file location") + output_group.add_argument('--out', type=str, default=None, + help="Override name of ouptut file") + + wheel_group = parser.add_argument_group("Wheel metadata") + wheel_group.add_argument( + '--header', action='append', + help="Additional headers to be embedded in the package metadata. " + "Can be supplied multiple times.") + wheel_group.add_argument('--classifier', action='append', + help="Classifiers to embed in package metadata. " + "Can be supplied multiple times") + wheel_group.add_argument('--description_file', + help="Path to the file with package description") + + contents_group = parser.add_argument_group("Wheel contents") + contents_group.add_argument( + '--input_file', action='append', + help="'package_path;real_path' pairs listing " + "files to be included in the wheel. " + "Can be supplied multiple times.") + contents_group.add_argument( + '--console_script', action='append', + help="Defines a 'console_script' entry point. " + "Can be supplied multiple times.") + + requirements_group = parser.add_argument_group("Package requirements") + requirements_group.add_argument( + '--requires', type=str, action='append', + help="List of package requirements. Can be supplied multiple times.") + requirements_group.add_argument( + '--extra_requires', type=str, action='append', + help="List of optional requirements in a 'requirement;option name'. " + "Can be supplied multiple times.") + arguments = parser.parse_args(sys.argv[1:]) + + # add_wheelfile and add_metadata currently assume pure-Python. + assert arguments.platform == 'any', "Only pure-Python wheels are supported" + + input_files = [i.split(';') for i in arguments.input_file] + all_files = get_files_to_package(input_files) + # Sort the files for reproducible order in the archive. + all_files = sorted(all_files.items()) + + with WheelMaker(name=arguments.name, + version=arguments.version, + build_tag=arguments.build_tag, + python_tag=arguments.python_tag, + abi=arguments.abi, + platform=arguments.platform, + outfile=arguments.out) as maker: + for package_filename, real_filename in all_files: + maker.add_file(package_filename, real_filename) + maker.add_wheelfile() + + description = None + if arguments.description_file: + if sys.version_info[0] == 2: + with open(arguments.description_file, + 'rt') as description_file: + description = description_file.read() + else: + with open(arguments.description_file, 'rt', + encoding='utf-8') as description_file: + description = description_file.read() + + extra_requires = collections.defaultdict(list) + if arguments.extra_requires: + for extra in arguments.extra_requires: + req, option = extra.rsplit(';', 1) + extra_requires[option].append(req) + classifiers = arguments.classifier or [] + requires = arguments.requires or [] + extra_headers = arguments.header or [] + console_scripts = arguments.console_script or [] + + maker.add_metadata(extra_headers=extra_headers, + description=description, + classifiers=classifiers, + requires=requires, + extra_requires=extra_requires) + maker.add_entry_points(console_scripts=console_scripts) + maker.add_recordfile() + + +if __name__ == '__main__': + main() From 4e1c3b6d347672852a585dfb0a9f2a33b009e9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Stradomski?= <44680433+pstradomski@users.noreply.github.com> Date: Tue, 16 Apr 2019 22:06:14 +0200 Subject: [PATCH 4/7] Stop using deprecated depset.union API (#180) Also switch to using Args api for building args instead of flattening the depset. --- experimental/python/wheel.bzl | 60 +++++++++++++++-------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/experimental/python/wheel.bzl b/experimental/python/wheel.bzl index 7603d93d51..75c4711d6d 100644 --- a/experimental/python/wheel.bzl +++ b/experimental/python/wheel.bzl @@ -27,6 +27,10 @@ def _path_inside_wheel(input_file): fail("input_file.path '%s' does not start with expected root '%s'" % (input_file.path, root)) return input_file.path[len(root):] +def _input_file_to_arg(input_file): + """Converts a File object to string for --input_file argument to wheelmaker""" + return "%s;%s" % (_path_inside_wheel(input_file), input_file.path) + def _py_package_impl(ctx): inputs = depset( transitive = [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.deps] + @@ -83,29 +87,23 @@ def _py_wheel_impl(ctx): ctx.attr.platform, ]) + ".whl") - inputs = depset( + inputs_to_package = depset( direct = ctx.files.deps, ) - arguments = [ - "--name", - ctx.attr.distribution, - "--version", - ctx.attr.version, - "--python_tag", - ctx.attr.python_tag, - "--abi", - ctx.attr.abi, - "--platform", - ctx.attr.platform, - "--out", - outfile.path, - ] + # Inputs to this rule which are not to be packaged. + # Currently this is only the description file (if used). + other_inputs = [] + + args = ctx.actions.args() + args.add("--name", ctx.attr.distribution) + args.add("--version", ctx.attr.version) + args.add("--python_tag", ctx.attr.python_tag) + args.add("--abi", ctx.attr.abi) + args.add("--platform", ctx.attr.platform) + args.add("--out", outfile.path) - # TODO: Use args api instead of flattening the depset. - for input_file in inputs.to_list(): - arguments.append("--input_file") - arguments.append("%s;%s" % (_path_inside_wheel(input_file), input_file.path)) + args.add_all(inputs_to_package, format_each = "--input_file=%s", map_each = _input_file_to_arg) extra_headers = [] if ctx.attr.author: @@ -118,36 +116,30 @@ def _py_wheel_impl(ctx): extra_headers.append("License: %s" % ctx.attr.license) for h in extra_headers: - arguments.append("--header") - arguments.append(h) + args.add("--header", h) for c in ctx.attr.classifiers: - arguments.append("--classifier") - arguments.append(c) + args.add("--classifier", c) for r in ctx.attr.requires: - arguments.append("--requires") - arguments.append(r) + args.add("--requires", r) for option, requirements in ctx.attr.extra_requires.items(): for r in requirements: - arguments.append("--extra_requires") - arguments.append(r + ";" + option) + args.add("--extra_requires", r + ";" + option) for name, ref in ctx.attr.console_scripts.items(): - arguments.append("--console_script") - arguments.append(name + " = " + ref) + args.add("--console_script", name + " = " + ref) if ctx.attr.description_file: description_file = ctx.file.description_file - arguments.append("--description_file") - arguments.append(description_file.path) - inputs = inputs.union([description_file]) + args.add("--description_file", description_file) + other_inputs.append(description_file) ctx.actions.run( - inputs = inputs, + inputs = depset(direct = other_inputs, transitive = [inputs_to_package]), outputs = [outfile], - arguments = arguments, + arguments = [args], executable = ctx.executable._wheelmaker, progress_message = "Building wheel", ) From 955f51801ec5693b59b1ec4f8d4b687af5052ab0 Mon Sep 17 00:00:00 2001 From: Philipp Wollermann Date: Tue, 23 Apr 2019 17:39:34 +0200 Subject: [PATCH 5/7] Don't test on Ubuntu 14.04 (#183) Ubuntu 14.04 is about to be end-of-life and Bazel CI will stop supporting it shortly afterwards. Context: https://groups.google.com/d/msg/bazel-dev/_D6XzfNkQQE/8TNKiNmsCAAJ --- .bazelci/presubmit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index dd13f44d9f..4a6248dd11 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -1,11 +1,11 @@ --- platforms: - ubuntu1404: + ubuntu1604: build_targets: - "..." test_targets: - "..." - ubuntu1604: + ubuntu1804: build_targets: - "..." test_targets: From 303ea57e6eed1b3c62bd9048664bf4bcce4fed50 Mon Sep 17 00:00:00 2001 From: Jon Brandvein Date: Tue, 14 May 2019 16:26:30 -0400 Subject: [PATCH 6/7] Update to work with --incompatible_use_python_toolchains (#188) Added `python_version = "PY2"` to some examples, and updated subpar dependency. --- WORKSPACE | 2 +- examples/extras/BUILD | 1 + examples/helloworld/BUILD | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 37decfa5cf..296055f836 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -81,7 +81,7 @@ _piptool_install() git_repository( name = "subpar", remote = "https://github.com/google/subpar", - commit = "0356bef3fbbabec5f0e196ecfacdeb6db62d48c0", # 2019-03-07 + tag = "2.0.0", ) ################################### diff --git a/examples/extras/BUILD b/examples/extras/BUILD index 94880ce47c..04498aded7 100644 --- a/examples/extras/BUILD +++ b/examples/extras/BUILD @@ -26,4 +26,5 @@ py_test( # Make sure that we can resolve the "extra" dependency requirement("googleapis-common-protos[grpc]"), ], + python_version = "PY2", ) diff --git a/examples/helloworld/BUILD b/examples/helloworld/BUILD index f1ee906188..854f70124d 100644 --- a/examples/helloworld/BUILD +++ b/examples/helloworld/BUILD @@ -28,4 +28,5 @@ py_test( name = "helloworld_test", srcs = ["helloworld_test.py"], deps = [":helloworld"], + python_version = "PY2", ) From 3b96a5d5d891934de5aa5704db69b2c71d463c6c Mon Sep 17 00:00:00 2001 From: Tony Liang Date: Tue, 2 Jul 2019 15:04:23 -0700 Subject: [PATCH 7/7] Add pip3 support --- python/pip.bzl | 77 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/python/pip.bzl b/python/pip.bzl index 45629952d2..6d39414631 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -13,7 +13,7 @@ # limitations under the License. """Import pip requirements into Bazel.""" -def _pip_import_impl(repository_ctx): +def _pip_import_impl(repository_ctx, binary_name): """Core implementation of pip_import.""" # Add an empty top-level BUILD file. @@ -25,7 +25,7 @@ def _pip_import_impl(repository_ctx): # To see the output, pass: quiet=False result = repository_ctx.execute( [ - "python", + binary_name, repository_ctx.path(repository_ctx.attr._script), "--name", repository_ctx.attr.name, @@ -42,6 +42,14 @@ def _pip_import_impl(repository_ctx): if result.return_code: fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) +def _pip_system_import_impl(repository_ctx): + """System python implementation.""" + _pip_import_impl(repository_ctx, "python") + +def _pip3_import_impl(repository_ctx): + """Python 3 implementation.""" + _pip_import_impl(repository_ctx, "python3") + pip_import = repository_rule( attrs = { "requirements": attr.label( @@ -54,14 +62,14 @@ pip_import = repository_rule( cfg = "host", ), }, - implementation = _pip_import_impl, + implementation = _pip_system_import_impl, ) """A rule for importing requirements.txt dependencies into Bazel. -This rule imports a requirements.txt file and generates a new -requirements.bzl file. This is used via the WORKSPACE -pattern: +This rule imports a requirements.txt file using the system +python, and generates a new requirements.bzl file. +This is used via the WORKSPACE pattern: 
pip_import(
     name = "foo",
     requirements = ":requirements.txt",
@@ -99,6 +107,63 @@ Args:
   requirements: The label of a requirements.txt file.
 """
 
+pip3_import = repository_rule(
+    attrs = {
+        "requirements": attr.label(
+            mandatory = True,
+            allow_single_file = True,
+        ),
+        "_script": attr.label(
+            executable = True,
+            default = Label("//tools:piptool.par"),
+            cfg = "host",
+        ),
+    },
+    implementation = _pip3_import_impl,
+)
+
+"""A rule for importing requirements.txt dependencies into Bazel.
+
+This rule imports a requirements.txt file using the system
+python3, and generates a new requirements.bzl file.
+This is used via the WORKSPACE pattern:
+
pip3_import(
+    name = "foo",
+    requirements = ":requirements.txt",
+)
+load("@foo//:requirements.bzl", "pip_install")
+pip_install()
+

+
+You can then reference imported dependencies from your BUILD
+file with:
+
load("@foo//:requirements.bzl", "requirement")
+py_library(
+    name = "bar",
+    ...
+    deps = [
+       "//my/other:dep",
+       requirement("futures"),
+       requirement("mock"),
+    ],
+)
+

+
+Or alternatively:
+
load("@foo//:requirements.bzl", "all_requirements")
+py_binary(
+    name = "baz",
+    ...
+    deps = [
+       ":foo",
+    ] + all_requirements,
+)
+

+
+Args:
+    requirements: The label of a requirements.txt file.
+"""
+
 def pip_repositories():
     """Pull in dependencies needed for pulling in pip dependencies.