From 9db1f56f06dde431eb705f6e66e3214f0be5fa9e Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Wed, 11 May 2022 16:49:02 -0700 Subject: [PATCH 1/8] verify python wheel before publish Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 57 +++++++++++++++++++++++++++++++---- setup.py | 2 +- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 62720a23630..83e1842d1ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,7 @@ on: push: tags: - 'v*.*.*' + workflow_dispatch: jobs: get-version: @@ -140,23 +141,63 @@ jobs: verify-python-wheel: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} needs: [build-python-sdk, build-python-sdk-macos-py310] + strategy: + matrix: + os: [ ubuntu-latest, macos-10.15 ] + python-version: [ "3.8", "3.9", "3.10"] + from-source: [ True, False ] steps: - name: Setup Python id: setup-python uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: ${{ matrix.python-version }} architecture: x64 - uses: actions/download-artifact@v2 with: name: wheels path: dist - name: Install wheel - run: pip install dist/*38*linux*.whl + if: ${{ !matrix.from-source }} + # try to install all wheels; only the current platform wheel should be actually installed + run: | + cd dist/ + for f in *.whl; do pip install $f || true; done - name: Install sdist + if: ${{ matrix.from-source }} run: pip install dist/*tar.gz + - name: Install OS X dependencies + if: ${{ matrix.os == "macos-10.15" }} + run: brew install coreutils + - name: Smoke test + env: + # this script is for testing servers + # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself + TEST_SCRIPT: | + timeout 10s $@ & pid=$! + wait $pid + ret=$? + if [[ $ret -ne 124 ]] + then + exit $ret + else + echo "Succeeded!" + fi + run: | + feast init test_repo + cd test_repo/ + feast apply + echo "$TEST_SCRIPT" > run-and-wait.sh + cat run-and-wait.sh + bash run-and-wait.sh feast serve + bash run-and-wait.sh feast ui + - name: Go lib test + if: ${{ !matrix.from-source }} + run: | + echo "\ngo_feature_retrieval: True" >> feature_store.yaml + bash run-and-wait.sh feast serve build-python-sdk: name: Build wheels on ${{ matrix.os }} @@ -171,6 +212,8 @@ jobs: with: node-version: '17.x' registry-url: 'https://registry.npmjs.org' + - name: Build UI + run: make build-ui - name: Build wheels uses: pypa/cibuildwheel@v2.4.0 env: @@ -178,9 +221,11 @@ jobs: CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" CIBW_ARCHS: "native" CIBW_ENVIRONMENT: > - COMPILE_GO=True + COMPILE_GO=True PATH=$PATH:/usr/local/go/bin CIBW_BEFORE_ALL_LINUX: | - yum install -y golang + curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz + tar -C /usr/local -xzf go.tar.gz + go version CIBW_BEFORE_ALL_MACOS: | curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg sudo installer -pkg python.pkg -target / @@ -188,7 +233,6 @@ jobs: make install-protoc-dependencies make install-go-proto-dependencies make install-go-ci-dependencies - make build-ui - uses: actions/upload-artifact@v2 with: @@ -222,6 +266,7 @@ jobs: make build-ui - name: Build run: | + python3 setup.py build_python_protos python3 setup.py sdist bdist_wheel - uses: actions/upload-artifact@v2 diff --git a/setup.py b/setup.py index 64ea5f0717f..7456de53f2f 100644 --- a/setup.py +++ b/setup.py @@ -355,11 +355,11 @@ class BuildCommand(build_py): def run(self): self.run_command("build_python_protos") - self.run_command("build_ext") if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() self.run_command("build_go_protos") + self.run_command("build_ext") build_py.run(self) From b4ed61329ac33003ca773d2d5122749ba73f3b4b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 09:32:04 -0700 Subject: [PATCH 2/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 83e1842d1ff..e52fc290ae2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -169,7 +169,7 @@ jobs: if: ${{ matrix.from-source }} run: pip install dist/*tar.gz - name: Install OS X dependencies - if: ${{ matrix.os == "macos-10.15" }} + if: matrix.os == 'macos-10.15' run: brew install coreutils - name: Smoke test env: From d99545e7cd5dbed5d18c3c2f77c7789539248e7b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 10:05:19 -0700 Subject: [PATCH 3/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e52fc290ae2..2cb8cc69d4e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -148,6 +148,19 @@ jobs: os: [ ubuntu-latest, macos-10.15 ] python-version: [ "3.8", "3.9", "3.10"] from-source: [ True, False ] + env: + # this script is for testing servers + # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself + TEST_SCRIPT: | + timeout 10s $@ & pid=$! + wait $pid + ret=$? + if [[ $ret -ne 124 ]] + then + exit $ret + else + echo "Succeeded!" + fi steps: - name: Setup Python id: setup-python @@ -172,30 +185,20 @@ jobs: if: matrix.os == 'macos-10.15' run: brew install coreutils - name: Smoke test - env: - # this script is for testing servers - # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself - TEST_SCRIPT: | - timeout 10s $@ & pid=$! - wait $pid - ret=$? - if [[ $ret -ne 124 ]] - then - exit $ret - else - echo "Succeeded!" - fi run: | feast init test_repo cd test_repo/ feast apply echo "$TEST_SCRIPT" > run-and-wait.sh - cat run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui - name: Go lib test if: ${{ !matrix.from-source }} run: | + feast init test_repo + cd test_repo/ + feast apply + echo "$TEST_SCRIPT" > run-and-wait.sh echo "\ngo_feature_retrieval: True" >> feature_store.yaml bash run-and-wait.sh feast serve From 108668dd96d4cd31bcbaf7ce750d973d98e36d0f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 10:41:15 -0700 Subject: [PATCH 4/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2cb8cc69d4e..82e113c2b2e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -180,7 +180,11 @@ jobs: for f in *.whl; do pip install $f || true; done - name: Install sdist if: ${{ matrix.from-source }} - run: pip install dist/*tar.gz + env: + COMPILE_GO=True + run: | + pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.1' + pip install dist/*tar.gz - name: Install OS X dependencies if: matrix.os == 'macos-10.15' run: brew install coreutils @@ -192,13 +196,6 @@ jobs: echo "$TEST_SCRIPT" > run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui - - name: Go lib test - if: ${{ !matrix.from-source }} - run: | - feast init test_repo - cd test_repo/ - feast apply - echo "$TEST_SCRIPT" > run-and-wait.sh echo "\ngo_feature_retrieval: True" >> feature_store.yaml bash run-and-wait.sh feast serve From ca7e40a87c1e762b8919ab50c273b9f86097c6a1 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 10:42:12 -0700 Subject: [PATCH 5/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 82e113c2b2e..f605e93fcb3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -181,7 +181,7 @@ jobs: - name: Install sdist if: ${{ matrix.from-source }} env: - COMPILE_GO=True + COMPILE_GO: "True" run: | pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.1' pip install dist/*tar.gz From 04ba4b5c1cdd3e86d2ea2ccd166fcbd7219aac9c Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 10:47:57 -0700 Subject: [PATCH 6/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f605e93fcb3..d1676a08b8e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -177,6 +177,7 @@ jobs: # try to install all wheels; only the current platform wheel should be actually installed run: | cd dist/ + pip install wheel for f in *.whl; do pip install $f || true; done - name: Install sdist if: ${{ matrix.from-source }} From e5673550b3408948c6d15532c8492bd4b5c53d9d Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 12:27:44 -0700 Subject: [PATCH 7/8] protos build dest Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 3 +- setup.py | 133 +++++++++++++++++++--------------- 2 files changed, 77 insertions(+), 59 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d1676a08b8e..53861cefee3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -184,7 +184,7 @@ jobs: env: COMPILE_GO: "True" run: | - pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.1' + pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' pip install dist/*tar.gz - name: Install OS X dependencies if: matrix.os == 'macos-10.15' @@ -267,7 +267,6 @@ jobs: make build-ui - name: Build run: | - python3 setup.py build_python_protos python3 setup.py sdist bdist_wheel - uses: actions/upload-artifact@v2 diff --git a/setup.py b/setup.py index 7456de53f2f..2044de008fc 100644 --- a/setup.py +++ b/setup.py @@ -114,9 +114,7 @@ "happybase>=1.2.0,<3", ] -GE_REQUIRED = [ - "great_expectations>=0.14.0,<0.15.0" -] +GE_REQUIRED = ["great_expectations>=0.14.0,<0.15.0"] GO_REQUIRED = [ "cffi==1.15.*,<2", @@ -166,15 +164,15 @@ "types-setuptools", "types-tabulate", ] - + GCP_REQUIRED - + REDIS_REQUIRED - + AWS_REQUIRED - + SNOWFLAKE_REQUIRED - + SPARK_REQUIRED - + POSTGRES_REQUIRED - + TRINO_REQUIRED - + GE_REQUIRED - + HBASE_REQUIRED + + GCP_REQUIRED + + REDIS_REQUIRED + + AWS_REQUIRED + + SNOWFLAKE_REQUIRED + + SPARK_REQUIRED + + POSTGRES_REQUIRED + + TRINO_REQUIRED + + GE_REQUIRED + + HBASE_REQUIRED ) DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED @@ -206,7 +204,9 @@ class BuildPythonProtosCommand(Command): description = "Builds the proto files into Python files." - user_options = [] + user_options = [ + ("inplace", "i", "Write generated proto files to source directory."), + ] def initialize_options(self): self.python_protoc = [ @@ -215,13 +215,21 @@ def initialize_options(self): "grpc_tools.protoc", ] # find_executable("protoc") self.proto_folder = os.path.join(repo_root, "protos") - self.python_folder = os.path.join( - os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos" - ) self.sub_folders = PROTO_SUBDIRS + self.build_lib = None + self.inplace = 0 def finalize_options(self): - pass + self.set_undefined_options("build", ("build_lib", "build_lib")) + + @property + def python_folder(self): + if self.inplace: + return os.path.join( + os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos" + ) + + return os.path.join(self.build_lib, "feast/protos") def _generate_python_protos(self, path: str): proto_files = glob.glob(os.path.join(self.proto_folder, path)) @@ -247,13 +255,12 @@ def run(self): # We need the __init__ files for each of the generated subdirs # so that they are regular packages, and don't need the `--namespace-packages` flags # when being typechecked using mypy. - with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'): + with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"): pass - - with open(f"{self.python_folder}/__init__.py", 'w'): + with open(f"{self.python_folder}/__init__.py", "w"): pass - with open(f"{self.python_folder}/feast/__init__.py", 'w'): + with open(f"{self.python_folder}/feast/__init__.py", "w"): pass for path in Path(self.python_folder).rglob("*.py"): @@ -295,12 +302,10 @@ def _ensure_go_and_proto_toolchain(): path_val = _generate_path_with_gopath() try: - subprocess.check_call(["protoc-gen-go", "--version"], env={ - "PATH": path_val - }) - subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={ - "PATH": path_val - }) + subprocess.check_call(["protoc-gen-go", "--version"], env={"PATH": path_val}) + subprocess.check_call( + ["protoc-gen-go-grpc", "--version"], env={"PATH": path_val} + ) except Exception as e: raise RuntimeError("Unable to find go/grpc extensions for protoc") from e @@ -329,15 +334,18 @@ def _generate_go_protos(self, path: str): try: subprocess.check_call( self.go_protoc - + ["-I", self.proto_folder, - "--go_out", self.go_folder, - "--go_opt=module=github.com/feast-dev/feast/go/protos", - "--go-grpc_out", self.go_folder, - "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"] + + [ + "-I", + self.proto_folder, + "--go_out", + self.go_folder, + "--go_opt=module=github.com/feast-dev/feast/go/protos", + "--go-grpc_out", + self.go_folder, + "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos", + ] + proto_files, - env={ - "PATH": self.path_val - } + env={"PATH": self.path_val}, ) except CalledProcessError as e: print(f"Stderr: {e.stderr}") @@ -367,6 +375,7 @@ class DevelopCommand(develop): """Custom develop command.""" def run(self): + self.reinitialize_command("build_python_protos", inplace=1) self.run_command("build_python_protos") if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() @@ -382,7 +391,10 @@ def finalize_options(self) -> None: self.extensions = [e for e in self.extensions if not self._is_go_ext(e)] def _is_go_ext(self, ext: Extension): - return any(source.endswith('.go') or source.startswith('github') for source in ext.sources) + return any( + source.endswith(".go") or source.startswith("github") + for source in ext.sources + ) def build_extension(self, ext: Extension): if not self._is_go_ext(ext): @@ -400,33 +412,34 @@ def build_extension(self, ext: Extension): ) destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call([ - "gopy", - "build", - "-output", - destination, - "-vm", - sys.executable, - "-no-make", - *ext.sources - ], env={ - "PATH": bin_path, - "CGO_LDFLAGS_ALLOW": ".*", - **go_env, - }) + subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"]) + subprocess.check_call(["go", "install", "github.com/go-python/gopy"]) + subprocess.check_call( + [ + "gopy", + "build", + "-output", + destination, + "-vm", + sys.executable, + "-no-make", + *ext.sources, + ], + env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,}, + ) def copy_extensions_to_source(self): - build_py = self.get_finalized_command('build_py') + build_py = self.get_finalized_command("build_py") for ext in self.extensions: fullname = self.get_ext_fullname(ext.name) - modpath = fullname.split('.') - package = '.'.join(modpath[:-1]) + modpath = fullname.split(".") + package = ".".join(modpath[:-1]) package_dir = build_py.get_package_dir(package) src_dir = dest_dir = package_dir if src_dir.startswith(PYTHON_CODE_PREFIX): - src_dir = package_dir[len(PYTHON_CODE_PREFIX):] + src_dir = package_dir[len(PYTHON_CODE_PREFIX) :] src_dir = src_dir.lstrip("/") src_dir = os.path.join(self.build_lib, src_dir) @@ -443,7 +456,9 @@ def copy_extensions_to_source(self): long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")), + packages=find_packages( + where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui") + ), package_dir={"": PYTHON_CODE_PREFIX}, install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements @@ -488,6 +503,10 @@ def copy_extensions_to_source(self): "develop": DevelopCommand, "build_ext": build_ext, }, - ext_modules=[Extension('feast.embedded_go.lib._embedded', - ["github.com/feast-dev/feast/go/embedded"])], + ext_modules=[ + Extension( + "feast.embedded_go.lib._embedded", + ["github.com/feast-dev/feast/go/embedded"], + ) + ], ) From 3c61d93dd722cec045bbe222c69c2e96826ace2b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 13 May 2022 12:43:15 -0700 Subject: [PATCH 8/8] fix Signed-off-by: Oleksii Moskalenko --- .github/workflows/publish.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 53861cefee3..4e73e53ca0d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -146,7 +146,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-10.15 ] - python-version: [ "3.8", "3.9", "3.10"] + python-version: [ "3.7", "3.8", "3.9", "3.10"] from-source: [ True, False ] env: # this script is for testing servers @@ -168,6 +168,9 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 + - uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' - uses: actions/download-artifact@v2 with: name: wheels @@ -185,6 +188,8 @@ jobs: COMPILE_GO: "True" run: | pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 pip install dist/*tar.gz - name: Install OS X dependencies if: matrix.os == 'macos-10.15' @@ -197,7 +202,9 @@ jobs: echo "$TEST_SCRIPT" > run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui - echo "\ngo_feature_retrieval: True" >> feature_store.yaml + + pip install cffi + printf "\ngo_feature_retrieval: True" >> feature_store.yaml bash run-and-wait.sh feast serve build-python-sdk: