diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index fcfb1c66875..e4a45b724ee 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -17,6 +17,10 @@ jobs: export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python - name: Install python run: make install-python - name: Test python diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index c99db076e3a..354ece44949 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -28,6 +28,11 @@ jobs: export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} + - uses: actions/checkout@v2 + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python - name: Install python run: make install-python - name: Test python diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 82adad18692..a6e52c890c8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,13 @@ jobs: container: gcr.io/kf-feast/feast-ci:latest steps: - uses: actions/checkout@v2 - - name: Install python + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python + - name: Install Python run: make install-python - - name: Test python + - name: Test Python run: make test-python unit-test-go: @@ -19,6 +23,8 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install dependencies + run: make install-go-ci-dependencies + - name: Compile protos run: make compile-protos-go - - name: Test go + - name: Test run: make test-go \ No newline at end of file diff --git a/.gitignore b/.gitignore index 546aff03830..b427d8cd12f 100644 --- a/.gitignore +++ b/.gitignore @@ -178,14 +178,15 @@ dmypy.json # This pom should not be committed because it is only used during release / deployment. .flattened-pom.xml -sdk/python/docs/html - -# Generated python code -*_pb2.py -*_pb2.pyi -*_pb2_grpc.py # VSCode .bloop .metals *.code-workspace + +# Protos +sdk/python/docs/source +sdk/python/docs/html +sdk/python/feast/protos/ +sdk/python/tensorflow_metadata +sdk/go/protos/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7cf514e580..47ed7b479c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,10 @@ +default_stages: + - push repos: - repo: local hooks: - id: lint name: Lint - stages: [commit] + stages: [ push ] language: system entry: make lint \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000..d01f2eebff8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,14 @@ +version: 2 + +sphinx: + configuration: sdk/python/docs/source/conf.py + +formats: + - pdf + +python: + version: 3.7 + install: + - requirements: sdk/python/requirements-ci.txt + - path: sdk/python/ + method: setuptools \ No newline at end of file diff --git a/Makefile b/Makefile index 0cde416a1c1..c2205898a9d 100644 --- a/Makefile +++ b/Makefile @@ -44,12 +44,14 @@ install-python-ci-dependencies: package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos -compile-protos-python: install-python-ci-dependencies +compile-protos-python: + printenv + pip install --ignore-installed mypy-protobuf || echo "Mypy could not be installed" @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto -install-python: compile-protos-python +install-python: python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver test-python: @@ -74,7 +76,7 @@ install-go-ci-dependencies: go get -u github.com/golang/protobuf/protoc-gen-go go get -u golang.org/x/lint/golint -compile-protos-go: install-go-ci-dependencies +compile-protos-go: cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ tensorflow_metadata/proto/v0/*.proto $(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;) @@ -124,22 +126,9 @@ install-dependencies-proto-docs: mv protoc3/include/* $$HOME/include compile-protos-docs: - cd ${ROOT_DIR}/protos; - mkdir -p ../dist/grpc - protoc --docs_out=../dist/grpc feast/*/*.proto || \ - cd ${ROOT_DIR}; $(MAKE) install-dependencies-proto-docs && cd ${ROOT_DIR}/protos; PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto + rm -rf $(ROOT_DIR)/dist/grpc + mkdir -p dist/grpc; + cd ${ROOT_DIR}/protos && protoc --docs_out=../dist/grpc feast/*/*.proto -clean-html: - rm -rf $(ROOT_DIR)/dist - -build-html: clean-html - mkdir -p $(ROOT_DIR)/dist/python - mkdir -p $(ROOT_DIR)/dist/grpc - - # Build Protobuf documentation - $(MAKE) compile-protos-docs - - # Build Python SDK documentation - $(MAKE) compile-protos-python - cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) html - cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python +build-sphinx: compile-protos-python + cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source diff --git a/sdk/python/.gitignore b/sdk/python/.gitignore index 503c2784600..f4e09ad9d62 100644 --- a/sdk/python/.gitignore +++ b/sdk/python/.gitignore @@ -115,5 +115,4 @@ dmypy.json .pyre/ .vscode/* -test.py playground \ No newline at end of file diff --git a/sdk/python/docs/Makefile b/sdk/python/docs/Makefile index 947df397e84..59ffa954e73 100644 --- a/sdk/python/docs/Makefile +++ b/sdk/python/docs/Makefile @@ -15,15 +15,17 @@ help: .PHONY: help Makefile source -clean: - rm -rf $(PROJECT_ROOT)/sdk/python/docs/html +clean-source: rm -rf $(PROJECT_ROOT)/sdk/python/docs/source -build-api-source: +clean-html: + rm -rf $(PROJECT_ROOT)/sdk/python/docs/html + +build-api-source: clean-source sphinx-apidoc -f -o source $(PROJECT_ROOT)/sdk/python/feast cp conf.py index.rst source/ -html: clean build-api-source +html: clean-html build-api-source sphinx-build -b html source html rm -rf $(PROJECT_ROOT)/sdk/python/docs/source diff --git a/sdk/python/feast/protos/tensorflow_metadata/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/feast/protos/tensorflow_metadata/proto/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/proto/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/feast/protos/tensorflow_metadata/proto/v0/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/proto/v0/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index f7865912727..57af0bc814e 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -11,12 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import logging import os import re import subprocess -from setuptools import find_packages, setup +from setuptools import find_packages + +try: + from setuptools import setup + from setuptools.command.install import install + from setuptools.command.develop import develop + from setuptools.command.egg_info import egg_info + from setuptools.command.sdist import sdist + +except ImportError: + from distutils.core import setup + from distutils.command.install import install NAME = "feast" DESCRIPTION = "Python SDK for Feast" @@ -67,6 +78,33 @@ r"^(?:[\/\w-]+)?(?P[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$" ) +proto_dirs = [x for x in os.listdir(repo_root + "/protos/feast")] +proto_dirs.remove("third_party") + + +def pre_install_build(): + subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") + subprocess.check_call("make build-sphinx", shell=True, cwd=f"{repo_root}") + + +class CustomInstallCommand(install): + def do_egg_install(self): + pre_install_build() + install.do_egg_install(self) + + +class CustomDevelopCommand(develop): + def run(self): + pre_install_build() + develop.run(self) + + +class CustomEggInfoCommand(egg_info): + def run(self): + pre_install_build() + egg_info.run(self) + + setup( name=NAME, author=AUTHOR, @@ -75,13 +113,14 @@ long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(exclude=("tests",)), + packages=find_packages(exclude=("tests",)) + ['.'], install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] extras_require={ "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"], "validation": ["great_expectations==0.13.2", "pyspark==3.0.1"], + "docs": ["grpcio-tools"], }, include_package_data=True, license="Apache", @@ -95,12 +134,19 @@ ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version={"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}, - setup_requires=["setuptools_scm"], + setup_requires=["setuptools_scm", "grpcio-tools", "mypy-protobuf", "sphinx"], package_data={ "": [ "protos/feast/**/*.proto", "protos/feast/third_party/grpc/health/v1/*.proto", "protos/tensorflow_metadata/proto/v0/*.proto", + "feast/protos/feast/**/*.py", + "tensorflow_metadata/proto/v0/*.py" ], }, + cmdclass={ + "install": CustomInstallCommand, + "develop": CustomDevelopCommand, + "egg_info": CustomEggInfoCommand, + }, )