diff --git a/WORKSPACE b/WORKSPACE index 69bfda230a..4eed74f999 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -72,8 +72,7 @@ http_file( # From https://pypi.python.org/pypi/futures url = ("https://pypi.python.org/packages/a6/1c/" + "72a18c8c7502ee1b38a604a5c5243aa8c2a64f4bba4e6631b1b8972235dd/" + - "futures-3.1.1-py2-none-any.whl"), -) + "futures-3.1.1-py2-none-any.whl"),) http_file( name = "futures_2_2_0_whl", @@ -102,6 +101,15 @@ http_file( "google_cloud_language-0.29.0-py2.py3-none-any.whl"), ) +http_file( + name = "pyspark_tar", + sha256 = "8680dcbc98e95a32f89bea70e114e139eae3543c094f063d548863522bdc4370", + # From https://pypi.python.org/pypi/grpcio/1.6.0 + url = ("https://files.pythonhosted.org/packages/65/d7/" + + "2a3b0de1178478fc00201b083d50b3d2d1affe4eac92dad3408219c5c607" + + "/pyspark-2.1.2.tar.gz"), +) + # Imports for examples pip_import( name = "examples_helloworld", diff --git a/rules_python/BUILD b/rules_python/BUILD index 6c23e01ae7..ec6aeddeb7 100644 --- a/rules_python/BUILD +++ b/rules_python/BUILD @@ -35,10 +35,13 @@ py_test( "@google_cloud_language_whl//file", "@grpc_whl//file", "@mock_whl//file", + "@pyspark_tar//file", ], deps = [ ":whl", requirement("mock"), + requirement("pip"), + requirement("wheel"), ], ) diff --git a/rules_python/whl.py b/rules_python/whl.py index 3e9d4f88fa..094377a4f4 100644 --- a/rules_python/whl.py +++ b/rules_python/whl.py @@ -19,7 +19,9 @@ import pkg_resources import re import zipfile +from stat import S_IXUSR +ZIP_UNIX_SYSTEM = 3 class Wheel(object): @@ -103,7 +105,15 @@ def extras(self): def expand(self, directory): with zipfile.ZipFile(self.path(), 'r') as whl: - whl.extractall(directory) + #FROM: https://stackoverflow.com/questions/42326428/zipfile-in-python-file-permission + for info in whl.infolist(): + extracted_path = whl.extract(info, directory) + + if info.create_system == ZIP_UNIX_SYSTEM and os.path.isfile(extracted_path): + unix_attributes = info.external_attr >> 16 + if unix_attributes & S_IXUSR: + os.chmod(extracted_path, os.stat(extracted_path).st_mode | S_IXUSR) + # _parse_metadata parses METADATA files according to https://www.python.org/dev/peps/pep-0314/ def _parse_metadata(self, content): diff --git a/rules_python/whl_test.py b/rules_python/whl_test.py index a63d625088..8a441546d1 100644 --- a/rules_python/whl_test.py +++ b/rules_python/whl_test.py @@ -14,9 +14,10 @@ import os import unittest +import pip -from mock import patch - +from stat import * +from mock import patch from rules_python import whl @@ -116,5 +117,20 @@ def test_google_cloud_language_whl_3_4(self, *args): self.assertEqual(set(wheel.dependencies()), set(expected_deps)) + @patch('platform.python_version', return_value='2.7.13') + def test_expand_executable(self, *args): + td = TestData('pyspark_tar/file/pyspark-2.1.2.tar.gz') + pip.main(["wheel", "-w", os.path.dirname(td), td]) + + td = TestData('pyspark_tar/file/' + + 'pyspark-2.1.2-py2.py3-none-any.whl') + wheel = whl.Wheel(td) + wheel.expand(TestData('pyspark_tar/file/expand')) + + attributes = os.stat(TestData('pyspark_tar/file/expand/pyspark/bin/spark-submit')) + + print attributes + self.assertEqual(attributes[0] & S_IXUSR, S_IXUSR) + if __name__ == '__main__': unittest.main() diff --git a/tools/piptool.par b/tools/piptool.par index 11ec453cd7..1c79768ec3 100755 Binary files a/tools/piptool.par and b/tools/piptool.par differ diff --git a/tools/whltool.par b/tools/whltool.par index 7cb59c0fbe..aa316bf3ed 100755 Binary files a/tools/whltool.par and b/tools/whltool.par differ