diff --git a/python/pip.bzl b/python/pip.bzl index 237b7d2baa..6b48d8f1e2 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -14,25 +14,31 @@ """Import pip requirements into Bazel.""" def _pip_import_impl(repository_ctx): - """Core implementation of pip_import.""" + """Core implementation of pip_import.""" - # Add an empty top-level BUILD file. - # This is because Bazel requires BUILD files along all paths accessed - # via //this/sort/of:path and we wouldn't be able to load our generated - # requirements.bzl without it. - repository_ctx.file("BUILD", "") + # Add an empty top-level BUILD file. + # This is because Bazel requires BUILD files along all paths accessed + # via //this/sort/of:path and we wouldn't be able to load our generated + # requirements.bzl without it. + repository_ctx.file("BUILD", "") - # To see the output, pass: quiet=False - result = repository_ctx.execute([ - "python", repository_ctx.path(repository_ctx.attr._script), - "--name", repository_ctx.attr.name, - "--input", repository_ctx.path(repository_ctx.attr.requirements), - "--output", repository_ctx.path("requirements.bzl"), - "--directory", repository_ctx.path(""), - ]) + # To see the output, pass: quiet=False + result = repository_ctx.execute([ + "/usr/bin/env", + "python3.7", + repository_ctx.path(repository_ctx.attr._script), + "--name", + repository_ctx.attr.name, + "--input", + repository_ctx.path(repository_ctx.attr.requirements), + "--output", + repository_ctx.path("requirements.bzl"), + "--directory", + repository_ctx.path(""), + ]) - if result.return_code: - fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) + if result.return_code: + fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) pip_import = repository_rule( attrs = { @@ -93,9 +99,9 @@ Args: """ def pip_repositories(): - """Pull in dependencies needed for pulling in pip dependencies. + """Pull in dependencies needed for pulling in pip dependencies. - A placeholder method that will eventually pull in any dependencies - needed to install pip dependencies. - """ - pass + A placeholder method that will eventually pull in any dependencies + needed to install pip dependencies. + """ + pass diff --git a/python/whl.bzl b/python/whl.bzl index 496755671f..b71f1d073a 100644 --- a/python/whl.bzl +++ b/python/whl.bzl @@ -14,24 +14,27 @@ """Import .whl files into Bazel.""" def _whl_impl(repository_ctx): - """Core implementation of whl_library.""" + """Core implementation of whl_library.""" - args = [ - "python", - repository_ctx.path(repository_ctx.attr._script), - "--whl", repository_ctx.path(repository_ctx.attr.whl), - "--requirements", repository_ctx.attr.requirements, - ] - - if repository_ctx.attr.extras: - args += [ - "--extras=%s" % extra - for extra in repository_ctx.attr.extras + args = [ + "/usr/bin/env", + "python3.7", + repository_ctx.path(repository_ctx.attr._script), + "--whl", + repository_ctx.path(repository_ctx.attr.whl), + "--requirements", + repository_ctx.attr.requirements, ] - result = repository_ctx.execute(args) - if result.return_code: - fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr)) + if repository_ctx.attr.extras: + args += [ + "--extras=%s" % extra + for extra in repository_ctx.attr.extras + ] + + result = repository_ctx.execute(args) + if result.return_code: + fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr)) whl_library = repository_rule( attrs = { diff --git a/rules_python/whl.py b/rules_python/whl.py index 3e9d4f88fa..8ac0601d86 100644 --- a/rules_python/whl.py +++ b/rules_python/whl.py @@ -14,12 +14,17 @@ """The whl modules defines classes for interacting with Python packages.""" import argparse +import collections +import email.parser import json import os import pkg_resources import re import zipfile +EXTRA_RE = re.compile("""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") +MayRequiresKey = collections.namedtuple('MayRequiresKey', ('condition', 'extra')) + class Wheel(object): @@ -107,9 +112,43 @@ def expand(self, directory): # _parse_metadata parses METADATA files according to https://www.python.org/dev/peps/pep-0314/ def _parse_metadata(self, content): - # TODO: handle fields other than just name - name_pattern = re.compile('Name: (.*)') - return { 'name': name_pattern.search(content).group(1) } + metadata = {} + pkg_info = email.parser.Parser().parsestr(content) + metadata['name'] = pkg_info.get('Name') + extras = pkg_info.get_all('Provides-Extra') + if extras: + metadata['extras'] = list(set(extras)) + + reqs_dist = pkg_info.get_all('Requires-Dist') or [] + requires = collections.defaultdict(set) + for value in sorted(reqs_dist): + extra_match = EXTRA_RE.search(value) + if extra_match: + groupdict = extra_match.groupdict() + condition = groupdict['condition'] + extra = groupdict['extra'] + package = groupdict['package'] + if condition.endswith(' and '): + condition = condition[:-5] + else: + condition, extra = None, None + package = value + key = MayRequiresKey(condition, extra) + requires[key].add(package) + + if requires: + metadata['run_requires'] = [] + for key, value in requires.items(): + requirement = { + 'requires': list(value) + } + if key.extra: + requirement['extra'] = key.extra + if key.condition: + requirement['environment'] = key.condition + metadata['run_requires'].append(requirement) + + return metadata parser = argparse.ArgumentParser( diff --git a/tools/piptool.par b/tools/piptool.par index 11ec453cd7..8b6b74dd7c 100755 Binary files a/tools/piptool.par and b/tools/piptool.par differ diff --git a/tools/update_tools/Dockerfile b/tools/update_tools/Dockerfile index 4439eb810c..41e6e8c37d 100644 --- a/tools/update_tools/Dockerfile +++ b/tools/update_tools/Dockerfile @@ -1,14 +1,15 @@ -FROM gcr.io/gcp-runtimes/ubuntu_16_0_4:latest +FROM ubuntu:18.04 # Install Bazel (https://docs.bazel.build/versions/master/install-ubuntu.html) -RUN apt-get update -y && apt-get install openjdk-8-jdk -y +RUN apt-get update -y && apt-get install openjdk-8-jdk curl gnupg -y RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - RUN apt-get update -y && apt-get install bazel -y RUN bazel help info >/dev/null 2>&1 -# Install Python 2.7.12 -RUN apt-get install python -y +# Install Python 3.7 +RUN apt-get install python3.7 python3.7-distutils -y +RUN ln -s /usr/bin/python3.7 /usr/bin/python # Build par files. We have a source and work directory to avoid # stomping on other files as root. diff --git a/tools/whltool.par b/tools/whltool.par index 7cb59c0fbe..dce6f44101 100755 Binary files a/tools/whltool.par and b/tools/whltool.par differ diff --git a/update_tools.sh b/update_tools.sh index 24a655e392..748c9b7e98 100755 --- a/update_tools.sh +++ b/update_tools.sh @@ -22,7 +22,7 @@ usage() { } if [ "$#" -eq 0 ] ; then - docker build --no-cache -f tools/update_tools/Dockerfile --tag rules_python:update_tools . + docker build -f tools/update_tools/Dockerfile --tag rules_python:update_tools . docker run -v"$PWD":/opt/rules_python_source rules_python:update_tools elif [ "$#" -eq 1 -a "$1" == "--nodocker" ] ; then bazel build //rules_python:piptool.par //rules_python:whltool.par