From ac607963db3e2a335b01980f928402c535e95034 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 17:34:00 -0700 Subject: [PATCH 01/12] Update documentation and proto building Signed-off-by: Willem Pienaar --- .gitignore | 12 +- .pre-commit-config.yaml | 9 +- .readthedocs.yml | 14 ++ Makefile | 27 +-- sdk/python/.gitignore | 1 - sdk/python/docs/Makefile | 10 +- sdk/python/docs/source/conf.py | 178 +++++++++++++++++ sdk/python/docs/source/feast.infra.rst | 45 +++++ sdk/python/docs/source/feast.loaders.rst | 45 +++++ sdk/python/docs/source/feast.protos.feast.rst | 10 + sdk/python/docs/source/feast.protos.rst | 19 ++ ...feast.protos.tensorflow_metadata.proto.rst | 18 ++ ...st.protos.tensorflow_metadata.proto.v0.rst | 10 + .../feast.protos.tensorflow_metadata.rst | 18 ++ sdk/python/docs/source/feast.rst | 184 ++++++++++++++++++ sdk/python/docs/source/feast.staging.rst | 29 +++ sdk/python/docs/source/index.rst | 43 ++++ sdk/python/docs/source/modules.rst | 7 + .../protos/tensorflow_metadata/__init__.py | 0 .../tensorflow_metadata/proto/__init__.py | 0 .../tensorflow_metadata/proto/v0/__init__.py | 0 sdk/python/setup.py | 74 ++++++- 22 files changed, 714 insertions(+), 39 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 sdk/python/docs/source/conf.py create mode 100644 sdk/python/docs/source/feast.infra.rst create mode 100644 sdk/python/docs/source/feast.loaders.rst create mode 100644 sdk/python/docs/source/feast.protos.feast.rst create mode 100644 sdk/python/docs/source/feast.protos.rst create mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst create mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst create mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.rst create mode 100644 sdk/python/docs/source/feast.rst create mode 100644 sdk/python/docs/source/feast.staging.rst create mode 100644 sdk/python/docs/source/index.rst create mode 100644 sdk/python/docs/source/modules.rst delete mode 100644 sdk/python/feast/protos/tensorflow_metadata/__init__.py delete mode 100644 sdk/python/feast/protos/tensorflow_metadata/proto/__init__.py delete mode 100644 sdk/python/feast/protos/tensorflow_metadata/proto/v0/__init__.py diff --git a/.gitignore b/.gitignore index 546aff03830..9d4928a9284 100644 --- a/.gitignore +++ b/.gitignore @@ -178,14 +178,14 @@ dmypy.json # This pom should not be committed because it is only used during release / deployment. .flattened-pom.xml -sdk/python/docs/html - -# Generated python code -*_pb2.py -*_pb2.pyi -*_pb2_grpc.py # VSCode .bloop .metals *.code-workspace + +# Protos +sdk/python/docs/html +sdk/python/feast/protos/ +sdk/python/tensorflow_metadata +sdk/go/protos/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7cf514e580..fe949cee286 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,11 @@ repos: hooks: - id: lint name: Lint - stages: [commit] + stages: [ push ] language: system - entry: make lint \ No newline at end of file + entry: make lint + - id: compile-protos-python + name: Compile Python Protos + stages: [ push ] + language: system + entry: make compile-protos-python \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000..d01f2eebff8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,14 @@ +version: 2 + +sphinx: + configuration: sdk/python/docs/source/conf.py + +formats: + - pdf + +python: + version: 3.7 + install: + - requirements: sdk/python/requirements-ci.txt + - path: sdk/python/ + method: setuptools \ No newline at end of file diff --git a/Makefile b/Makefile index 0cde416a1c1..7ad80052447 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ install-python-ci-dependencies: package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos -compile-protos-python: install-python-ci-dependencies +compile-protos-python: @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto @@ -74,7 +74,7 @@ install-go-ci-dependencies: go get -u github.com/golang/protobuf/protoc-gen-go go get -u golang.org/x/lint/golint -compile-protos-go: install-go-ci-dependencies +compile-protos-go: cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ tensorflow_metadata/proto/v0/*.proto $(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;) @@ -124,22 +124,9 @@ install-dependencies-proto-docs: mv protoc3/include/* $$HOME/include compile-protos-docs: - cd ${ROOT_DIR}/protos; - mkdir -p ../dist/grpc - protoc --docs_out=../dist/grpc feast/*/*.proto || \ - cd ${ROOT_DIR}; $(MAKE) install-dependencies-proto-docs && cd ${ROOT_DIR}/protos; PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto + rm -rf $(ROOT_DIR)/dist/grpc + mkdir -p dist/grpc; + cd ${ROOT_DIR}/protos && protoc --docs_out=../dist/grpc feast/*/*.proto -clean-html: - rm -rf $(ROOT_DIR)/dist - -build-html: clean-html - mkdir -p $(ROOT_DIR)/dist/python - mkdir -p $(ROOT_DIR)/dist/grpc - - # Build Protobuf documentation - $(MAKE) compile-protos-docs - - # Build Python SDK documentation - $(MAKE) compile-protos-python - cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) html - cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python +build-sphinx: compile-protos-python + cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source \ No newline at end of file diff --git a/sdk/python/.gitignore b/sdk/python/.gitignore index 503c2784600..f4e09ad9d62 100644 --- a/sdk/python/.gitignore +++ b/sdk/python/.gitignore @@ -115,5 +115,4 @@ dmypy.json .pyre/ .vscode/* -test.py playground \ No newline at end of file diff --git a/sdk/python/docs/Makefile b/sdk/python/docs/Makefile index 947df397e84..59ffa954e73 100644 --- a/sdk/python/docs/Makefile +++ b/sdk/python/docs/Makefile @@ -15,15 +15,17 @@ help: .PHONY: help Makefile source -clean: - rm -rf $(PROJECT_ROOT)/sdk/python/docs/html +clean-source: rm -rf $(PROJECT_ROOT)/sdk/python/docs/source -build-api-source: +clean-html: + rm -rf $(PROJECT_ROOT)/sdk/python/docs/html + +build-api-source: clean-source sphinx-apidoc -f -o source $(PROJECT_ROOT)/sdk/python/feast cp conf.py index.rst source/ -html: clean build-api-source +html: clean-html build-api-source sphinx-build -b html source html rm -rf $(PROJECT_ROOT)/sdk/python/docs/source diff --git a/sdk/python/docs/source/conf.py b/sdk/python/docs/source/conf.py new file mode 100644 index 00000000000..8f873d21b61 --- /dev/null +++ b/sdk/python/docs/source/conf.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# +# Feast documentation build configuration file, created by +# sphinx-quickstart on Sat Nov 30 15:06:53 2019. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath("../../feast")) +sys.path.insert(0, os.path.abspath("../..")) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + "sphinx.ext.autodoc", + "sphinx_rtd_theme", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Feast" +copyright = "2021, Feast Authors" +author = "Feast Authors" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. + +# TODO: Add the below versions back to documentation building. +# version = ( +# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() +# ) +# The full version, including alpha/beta/rc tags. +# release = ( +# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() +# ) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "Feastdoc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "Feast.tex", "Feast Documentation", "Feast Authors", "manual") +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "feast", "Feast Documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "Feast", + "Feast Documentation", + author, + "Feast", + "One line description of project.", + "Miscellaneous", + ) +] + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst new file mode 100644 index 00000000000..9e0322c950a --- /dev/null +++ b/sdk/python/docs/source/feast.infra.rst @@ -0,0 +1,45 @@ +feast.infra package +=================== + +Submodules +---------- + +feast.infra.gcp module +---------------------- + +.. automodule:: feast.infra.gcp + :members: + :undoc-members: + :show-inheritance: + +feast.infra.key\_encoding\_utils module +--------------------------------------- + +.. automodule:: feast.infra.key_encoding_utils + :members: + :undoc-members: + :show-inheritance: + +feast.infra.local\_sqlite module +-------------------------------- + +.. automodule:: feast.infra.local_sqlite + :members: + :undoc-members: + :show-inheritance: + +feast.infra.provider module +--------------------------- + +.. automodule:: feast.infra.provider + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.loaders.rst b/sdk/python/docs/source/feast.loaders.rst new file mode 100644 index 00000000000..da3cbc13ad5 --- /dev/null +++ b/sdk/python/docs/source/feast.loaders.rst @@ -0,0 +1,45 @@ +feast.loaders package +===================== + +Submodules +---------- + +feast.loaders.abstract\_producer module +--------------------------------------- + +.. automodule:: feast.loaders.abstract_producer + :members: + :undoc-members: + :show-inheritance: + +feast.loaders.file module +------------------------- + +.. automodule:: feast.loaders.file + :members: + :undoc-members: + :show-inheritance: + +feast.loaders.ingest module +--------------------------- + +.. automodule:: feast.loaders.ingest + :members: + :undoc-members: + :show-inheritance: + +feast.loaders.yaml module +------------------------- + +.. automodule:: feast.loaders.yaml + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.loaders + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.rst b/sdk/python/docs/source/feast.protos.feast.rst new file mode 100644 index 00000000000..db5d26413dc --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.rst @@ -0,0 +1,10 @@ +feast.protos.feast package +========================== + +Module contents +--------------- + +.. automodule:: feast.protos.feast + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.rst b/sdk/python/docs/source/feast.protos.rst new file mode 100644 index 00000000000..47015b8cbd2 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.rst @@ -0,0 +1,19 @@ +feast.protos package +==================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.protos.feast + feast.protos.tensorflow_metadata + +Module contents +--------------- + +.. automodule:: feast.protos + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst new file mode 100644 index 00000000000..fe0d5b1707a --- /dev/null +++ b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst @@ -0,0 +1,18 @@ +feast.protos.tensorflow\_metadata.proto package +=============================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.protos.tensorflow_metadata.proto.v0 + +Module contents +--------------- + +.. automodule:: feast.protos.tensorflow_metadata.proto + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst new file mode 100644 index 00000000000..d6076aababf --- /dev/null +++ b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst @@ -0,0 +1,10 @@ +feast.protos.tensorflow\_metadata.proto.v0 package +================================================== + +Module contents +--------------- + +.. automodule:: feast.protos.tensorflow_metadata.proto.v0 + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst new file mode 100644 index 00000000000..433eeab6407 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst @@ -0,0 +1,18 @@ +feast.protos.tensorflow\_metadata package +========================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.protos.tensorflow_metadata.proto + +Module contents +--------------- + +.. automodule:: feast.protos.tensorflow_metadata + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst new file mode 100644 index 00000000000..3eaa3eae827 --- /dev/null +++ b/sdk/python/docs/source/feast.rst @@ -0,0 +1,184 @@ +feast package +============= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra + feast.loaders + feast.protos + feast.staging + +Submodules +---------- + +feast.cli module +---------------- + +.. automodule:: feast.cli + :members: + :undoc-members: + :show-inheritance: + +feast.client module +------------------- + +.. automodule:: feast.client + :members: + :undoc-members: + :show-inheritance: + +feast.config module +------------------- + +.. automodule:: feast.config + :members: + :undoc-members: + :show-inheritance: + +feast.constants module +---------------------- + +.. automodule:: feast.constants + :members: + :undoc-members: + :show-inheritance: + +feast.data\_format module +------------------------- + +.. automodule:: feast.data_format + :members: + :undoc-members: + :show-inheritance: + +feast.data\_source module +------------------------- + +.. automodule:: feast.data_source + :members: + :undoc-members: + :show-inheritance: + +feast.entity module +------------------- + +.. automodule:: feast.entity + :members: + :undoc-members: + :show-inheritance: + +feast.feature module +-------------------- + +.. automodule:: feast.feature + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_store module +--------------------------- + +.. automodule:: feast.feature_store + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_table module +--------------------------- + +.. automodule:: feast.feature_table + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_view module +-------------------------- + +.. automodule:: feast.feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.offline\_store module +--------------------------- + +.. automodule:: feast.offline_store + :members: + :undoc-members: + :show-inheritance: + +feast.online\_response module +----------------------------- + +.. automodule:: feast.online_response + :members: + :undoc-members: + :show-inheritance: + +feast.registry module +--------------------- + +.. automodule:: feast.registry + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_config module +------------------------- + +.. automodule:: feast.repo_config + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_operations module +----------------------------- + +.. automodule:: feast.repo_operations + :members: + :undoc-members: + :show-inheritance: + +feast.telemetry module +---------------------- + +.. automodule:: feast.telemetry + :members: + :undoc-members: + :show-inheritance: + +feast.type\_map module +---------------------- + +.. automodule:: feast.type_map + :members: + :undoc-members: + :show-inheritance: + +feast.value\_type module +------------------------ + +.. automodule:: feast.value_type + :members: + :undoc-members: + :show-inheritance: + +feast.wait module +----------------- + +.. automodule:: feast.wait + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.staging.rst b/sdk/python/docs/source/feast.staging.rst new file mode 100644 index 00000000000..5ad63281a6a --- /dev/null +++ b/sdk/python/docs/source/feast.staging.rst @@ -0,0 +1,29 @@ +feast.staging package +===================== + +Submodules +---------- + +feast.staging.entities module +----------------------------- + +.. automodule:: feast.staging.entities + :members: + :undoc-members: + :show-inheritance: + +feast.staging.storage\_client module +------------------------------------ + +.. automodule:: feast.staging.storage_client + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.staging + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/index.rst b/sdk/python/docs/source/index.rst new file mode 100644 index 00000000000..782ec9c83e2 --- /dev/null +++ b/sdk/python/docs/source/index.rst @@ -0,0 +1,43 @@ +Feast Python API Documentation +============================= + +Client +================== + +.. automodule:: feast.client + :members: + +Data Source +================== + +.. automodule:: feast.data_source + :members: + + +Entity +================== + +.. automodule:: feast.entity + :inherited-members: + :members: + + +Feature Table +================== + +.. automodule:: feast.feature_table + :members: + +Feature +================== + +.. automodule:: feast.feature + :inherited-members: + :members: + +Constants +================== + +.. automodule:: feast.constants + :members: + :exclude-members: AuthProvider, ConfigMeta diff --git a/sdk/python/docs/source/modules.rst b/sdk/python/docs/source/modules.rst new file mode 100644 index 00000000000..3a6f8333abd --- /dev/null +++ b/sdk/python/docs/source/modules.rst @@ -0,0 +1,7 @@ +feast +===== + +.. toctree:: + :maxdepth: 4 + + feast diff --git a/sdk/python/feast/protos/tensorflow_metadata/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/feast/protos/tensorflow_metadata/proto/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/proto/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/feast/protos/tensorflow_metadata/proto/v0/__init__.py b/sdk/python/feast/protos/tensorflow_metadata/proto/v0/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index f7865912727..ecad24e8057 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -11,12 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import logging import os import re import subprocess -from setuptools import find_packages, setup +from setuptools import find_packages + +try: + from setuptools import setup + from setuptools.command.install import install + from setuptools.command.develop import develop + from setuptools.command.egg_info import egg_info +except ImportError: + from distutils.core import setup + from distutils.command.install import install NAME = "feast" DESCRIPTION = "Python SDK for Feast" @@ -52,9 +61,9 @@ # README file from Feast repo root directory repo_root = ( subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) - .communicate()[0] - .rstrip() - .decode("utf-8") + .communicate()[0] + .rstrip() + .decode("utf-8") ) README_FILE = os.path.join(repo_root, "README.md") with open(os.path.join(README_FILE), "r") as f: @@ -67,6 +76,53 @@ r"^(?:[\/\w-]+)?(?P[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$" ) + +proto_dirs = [x for x in os.listdir(repo_root + "/protos/feast")] +proto_dirs.remove("third_party") + + +def pre_install_build(): + """ + Build Python protos when installing Feast using setup.py + """ + for directory in proto_dirs: + from grpc_tools import protoc + subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") + # args = f"--proto_path=. --python_out={repo_root}/sdk/python/feast/protos/ --grpc_python_out={repo_root}/sdk/python/feast/protos/ feast/{directory}/*.proto" + # subprocess.check_call( + # "python -m grpc_tools.protoc " + args, shell=True, cwd=f"{repo_root}/protos" + # ) + # + # subprocess.check_call( + # f"python -m grpc_tools.protoc -I. --python_out={repo_root}/sdk/python/ tensorflow_metadata/proto/v0/*.proto", + # shell=True, + # cwd=f"{repo_root}/protos", + # ) + # subprocess.check_call( + # f"grep -rli 'from feast.{directory}' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.{directory}/from feast.protos.feast.{directory}/g' @", + # shell=True, + # cwd=f"{repo_root}", + # ) + +# Classes used to inject pre_install_build() +class CustomInstallCommand(install): + def do_egg_install(self): + pre_install_build() + install.do_egg_install(self) + + +class CustomDevelopCommand(develop): + def run(self): + pre_install_build() + develop.run(self) + + +class CustomEggInfoCommand(egg_info): + def run(self): + pre_install_build() + egg_info.run(self) + + setup( name=NAME, author=AUTHOR, @@ -82,6 +138,7 @@ extras_require={ "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"], "validation": ["great_expectations==0.13.2", "pyspark==3.0.1"], + "docs": ["grpcio-tools"], }, include_package_data=True, license="Apache", @@ -95,7 +152,7 @@ ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version={"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}, - setup_requires=["setuptools_scm"], + setup_requires=["setuptools_scm", "grpcio-tools", "mypy-protobuf"], package_data={ "": [ "protos/feast/**/*.proto", @@ -103,4 +160,9 @@ "protos/tensorflow_metadata/proto/v0/*.proto", ], }, + cmdclass={ + "install": CustomInstallCommand, + "develop": CustomDevelopCommand, + "egg_info": CustomEggInfoCommand, + }, ) From f6d14918549ea71a0c0ac441911f4c1c4f617dd9 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 17:44:18 -0700 Subject: [PATCH 02/12] Automatically build API docs Signed-off-by: Willem Pienaar --- sdk/python/docs/source/conf.py | 178 ----------------- sdk/python/docs/source/feast.infra.rst | 45 ----- sdk/python/docs/source/feast.loaders.rst | 45 ----- sdk/python/docs/source/feast.protos.feast.rst | 10 - sdk/python/docs/source/feast.protos.rst | 19 -- ...feast.protos.tensorflow_metadata.proto.rst | 18 -- ...st.protos.tensorflow_metadata.proto.v0.rst | 10 - .../feast.protos.tensorflow_metadata.rst | 18 -- sdk/python/docs/source/feast.rst | 184 ------------------ sdk/python/docs/source/feast.staging.rst | 29 --- sdk/python/docs/source/index.rst | 43 ---- sdk/python/docs/source/modules.rst | 7 - sdk/python/setup.py | 22 +-- 13 files changed, 3 insertions(+), 625 deletions(-) delete mode 100644 sdk/python/docs/source/conf.py delete mode 100644 sdk/python/docs/source/feast.infra.rst delete mode 100644 sdk/python/docs/source/feast.loaders.rst delete mode 100644 sdk/python/docs/source/feast.protos.feast.rst delete mode 100644 sdk/python/docs/source/feast.protos.rst delete mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst delete mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst delete mode 100644 sdk/python/docs/source/feast.protos.tensorflow_metadata.rst delete mode 100644 sdk/python/docs/source/feast.rst delete mode 100644 sdk/python/docs/source/feast.staging.rst delete mode 100644 sdk/python/docs/source/index.rst delete mode 100644 sdk/python/docs/source/modules.rst diff --git a/sdk/python/docs/source/conf.py b/sdk/python/docs/source/conf.py deleted file mode 100644 index 8f873d21b61..00000000000 --- a/sdk/python/docs/source/conf.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Feast documentation build configuration file, created by -# sphinx-quickstart on Sat Nov 30 15:06:53 2019. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -import sphinx_rtd_theme - -sys.path.insert(0, os.path.abspath("../../feast")) -sys.path.insert(0, os.path.abspath("../..")) - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.mathjax", - "sphinx.ext.ifconfig", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", - "sphinx.ext.napoleon", - "sphinx.ext.autodoc", - "sphinx_rtd_theme", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Feast" -copyright = "2021, Feast Authors" -author = "Feast Authors" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. - -# TODO: Add the below versions back to documentation building. -# version = ( -# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() -# ) -# The full version, including alpha/beta/rc tags. -# release = ( -# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() -# ) - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = "Feastdoc" - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, "Feast.tex", "Feast Documentation", "Feast Authors", "manual") -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "feast", "Feast Documentation", [author], 1)] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "Feast", - "Feast Documentation", - author, - "Feast", - "One line description of project.", - "Miscellaneous", - ) -] - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst deleted file mode 100644 index 9e0322c950a..00000000000 --- a/sdk/python/docs/source/feast.infra.rst +++ /dev/null @@ -1,45 +0,0 @@ -feast.infra package -=================== - -Submodules ----------- - -feast.infra.gcp module ----------------------- - -.. automodule:: feast.infra.gcp - :members: - :undoc-members: - :show-inheritance: - -feast.infra.key\_encoding\_utils module ---------------------------------------- - -.. automodule:: feast.infra.key_encoding_utils - :members: - :undoc-members: - :show-inheritance: - -feast.infra.local\_sqlite module --------------------------------- - -.. automodule:: feast.infra.local_sqlite - :members: - :undoc-members: - :show-inheritance: - -feast.infra.provider module ---------------------------- - -.. automodule:: feast.infra.provider - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: feast.infra - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.loaders.rst b/sdk/python/docs/source/feast.loaders.rst deleted file mode 100644 index da3cbc13ad5..00000000000 --- a/sdk/python/docs/source/feast.loaders.rst +++ /dev/null @@ -1,45 +0,0 @@ -feast.loaders package -===================== - -Submodules ----------- - -feast.loaders.abstract\_producer module ---------------------------------------- - -.. automodule:: feast.loaders.abstract_producer - :members: - :undoc-members: - :show-inheritance: - -feast.loaders.file module -------------------------- - -.. automodule:: feast.loaders.file - :members: - :undoc-members: - :show-inheritance: - -feast.loaders.ingest module ---------------------------- - -.. automodule:: feast.loaders.ingest - :members: - :undoc-members: - :show-inheritance: - -feast.loaders.yaml module -------------------------- - -.. automodule:: feast.loaders.yaml - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: feast.loaders - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.rst b/sdk/python/docs/source/feast.protos.feast.rst deleted file mode 100644 index db5d26413dc..00000000000 --- a/sdk/python/docs/source/feast.protos.feast.rst +++ /dev/null @@ -1,10 +0,0 @@ -feast.protos.feast package -========================== - -Module contents ---------------- - -.. automodule:: feast.protos.feast - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.rst b/sdk/python/docs/source/feast.protos.rst deleted file mode 100644 index 47015b8cbd2..00000000000 --- a/sdk/python/docs/source/feast.protos.rst +++ /dev/null @@ -1,19 +0,0 @@ -feast.protos package -==================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - feast.protos.feast - feast.protos.tensorflow_metadata - -Module contents ---------------- - -.. automodule:: feast.protos - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst deleted file mode 100644 index fe0d5b1707a..00000000000 --- a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.rst +++ /dev/null @@ -1,18 +0,0 @@ -feast.protos.tensorflow\_metadata.proto package -=============================================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - feast.protos.tensorflow_metadata.proto.v0 - -Module contents ---------------- - -.. automodule:: feast.protos.tensorflow_metadata.proto - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst deleted file mode 100644 index d6076aababf..00000000000 --- a/sdk/python/docs/source/feast.protos.tensorflow_metadata.proto.v0.rst +++ /dev/null @@ -1,10 +0,0 @@ -feast.protos.tensorflow\_metadata.proto.v0 package -================================================== - -Module contents ---------------- - -.. automodule:: feast.protos.tensorflow_metadata.proto.v0 - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst b/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst deleted file mode 100644 index 433eeab6407..00000000000 --- a/sdk/python/docs/source/feast.protos.tensorflow_metadata.rst +++ /dev/null @@ -1,18 +0,0 @@ -feast.protos.tensorflow\_metadata package -========================================= - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - feast.protos.tensorflow_metadata.proto - -Module contents ---------------- - -.. automodule:: feast.protos.tensorflow_metadata - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst deleted file mode 100644 index 3eaa3eae827..00000000000 --- a/sdk/python/docs/source/feast.rst +++ /dev/null @@ -1,184 +0,0 @@ -feast package -============= - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - feast.infra - feast.loaders - feast.protos - feast.staging - -Submodules ----------- - -feast.cli module ----------------- - -.. automodule:: feast.cli - :members: - :undoc-members: - :show-inheritance: - -feast.client module -------------------- - -.. automodule:: feast.client - :members: - :undoc-members: - :show-inheritance: - -feast.config module -------------------- - -.. automodule:: feast.config - :members: - :undoc-members: - :show-inheritance: - -feast.constants module ----------------------- - -.. automodule:: feast.constants - :members: - :undoc-members: - :show-inheritance: - -feast.data\_format module -------------------------- - -.. automodule:: feast.data_format - :members: - :undoc-members: - :show-inheritance: - -feast.data\_source module -------------------------- - -.. automodule:: feast.data_source - :members: - :undoc-members: - :show-inheritance: - -feast.entity module -------------------- - -.. automodule:: feast.entity - :members: - :undoc-members: - :show-inheritance: - -feast.feature module --------------------- - -.. automodule:: feast.feature - :members: - :undoc-members: - :show-inheritance: - -feast.feature\_store module ---------------------------- - -.. automodule:: feast.feature_store - :members: - :undoc-members: - :show-inheritance: - -feast.feature\_table module ---------------------------- - -.. automodule:: feast.feature_table - :members: - :undoc-members: - :show-inheritance: - -feast.feature\_view module --------------------------- - -.. automodule:: feast.feature_view - :members: - :undoc-members: - :show-inheritance: - -feast.offline\_store module ---------------------------- - -.. automodule:: feast.offline_store - :members: - :undoc-members: - :show-inheritance: - -feast.online\_response module ------------------------------ - -.. automodule:: feast.online_response - :members: - :undoc-members: - :show-inheritance: - -feast.registry module ---------------------- - -.. automodule:: feast.registry - :members: - :undoc-members: - :show-inheritance: - -feast.repo\_config module -------------------------- - -.. automodule:: feast.repo_config - :members: - :undoc-members: - :show-inheritance: - -feast.repo\_operations module ------------------------------ - -.. automodule:: feast.repo_operations - :members: - :undoc-members: - :show-inheritance: - -feast.telemetry module ----------------------- - -.. automodule:: feast.telemetry - :members: - :undoc-members: - :show-inheritance: - -feast.type\_map module ----------------------- - -.. automodule:: feast.type_map - :members: - :undoc-members: - :show-inheritance: - -feast.value\_type module ------------------------- - -.. automodule:: feast.value_type - :members: - :undoc-members: - :show-inheritance: - -feast.wait module ------------------ - -.. automodule:: feast.wait - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: feast - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/feast.staging.rst b/sdk/python/docs/source/feast.staging.rst deleted file mode 100644 index 5ad63281a6a..00000000000 --- a/sdk/python/docs/source/feast.staging.rst +++ /dev/null @@ -1,29 +0,0 @@ -feast.staging package -===================== - -Submodules ----------- - -feast.staging.entities module ------------------------------ - -.. automodule:: feast.staging.entities - :members: - :undoc-members: - :show-inheritance: - -feast.staging.storage\_client module ------------------------------------- - -.. automodule:: feast.staging.storage_client - :members: - :undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: feast.staging - :members: - :undoc-members: - :show-inheritance: diff --git a/sdk/python/docs/source/index.rst b/sdk/python/docs/source/index.rst deleted file mode 100644 index 782ec9c83e2..00000000000 --- a/sdk/python/docs/source/index.rst +++ /dev/null @@ -1,43 +0,0 @@ -Feast Python API Documentation -============================= - -Client -================== - -.. automodule:: feast.client - :members: - -Data Source -================== - -.. automodule:: feast.data_source - :members: - - -Entity -================== - -.. automodule:: feast.entity - :inherited-members: - :members: - - -Feature Table -================== - -.. automodule:: feast.feature_table - :members: - -Feature -================== - -.. automodule:: feast.feature - :inherited-members: - :members: - -Constants -================== - -.. automodule:: feast.constants - :members: - :exclude-members: AuthProvider, ConfigMeta diff --git a/sdk/python/docs/source/modules.rst b/sdk/python/docs/source/modules.rst deleted file mode 100644 index 3a6f8333abd..00000000000 --- a/sdk/python/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -feast -===== - -.. toctree:: - :maxdepth: 4 - - feast diff --git a/sdk/python/setup.py b/sdk/python/setup.py index ecad24e8057..5f447508ed7 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -85,24 +85,8 @@ def pre_install_build(): """ Build Python protos when installing Feast using setup.py """ - for directory in proto_dirs: - from grpc_tools import protoc - subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") - # args = f"--proto_path=. --python_out={repo_root}/sdk/python/feast/protos/ --grpc_python_out={repo_root}/sdk/python/feast/protos/ feast/{directory}/*.proto" - # subprocess.check_call( - # "python -m grpc_tools.protoc " + args, shell=True, cwd=f"{repo_root}/protos" - # ) - # - # subprocess.check_call( - # f"python -m grpc_tools.protoc -I. --python_out={repo_root}/sdk/python/ tensorflow_metadata/proto/v0/*.proto", - # shell=True, - # cwd=f"{repo_root}/protos", - # ) - # subprocess.check_call( - # f"grep -rli 'from feast.{directory}' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.{directory}/from feast.protos.feast.{directory}/g' @", - # shell=True, - # cwd=f"{repo_root}", - # ) + subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") + subprocess.check_call("make build-sphinx", shell=True, cwd=f"{repo_root}") # Classes used to inject pre_install_build() class CustomInstallCommand(install): @@ -152,7 +136,7 @@ def run(self): ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version={"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}, - setup_requires=["setuptools_scm", "grpcio-tools", "mypy-protobuf"], + setup_requires=["setuptools_scm", "grpcio-tools", "mypy-protobuf", "sphinx"], package_data={ "": [ "protos/feast/**/*.proto", From 96d8ff7c7c7d440a87ef22219e3599478aa2e1c1 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 17:45:02 -0700 Subject: [PATCH 03/12] Update formatting Signed-off-by: Willem Pienaar --- sdk/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 5f447508ed7..45ef3350b06 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -88,6 +88,7 @@ def pre_install_build(): subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") subprocess.check_call("make build-sphinx", shell=True, cwd=f"{repo_root}") + # Classes used to inject pre_install_build() class CustomInstallCommand(install): def do_egg_install(self): From 2e57db6e7ba1974bf60858571af0147a0c886c70 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 18:51:23 -0700 Subject: [PATCH 04/12] Fix SDK release building Signed-off-by: Willem Pienaar --- .gitignore | 1 + .pre-commit-config.yaml | 9 +++------ sdk/python/setup.py | 17 ++++++++--------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 9d4928a9284..b427d8cd12f 100644 --- a/.gitignore +++ b/.gitignore @@ -185,6 +185,7 @@ dmypy.json *.code-workspace # Protos +sdk/python/docs/source sdk/python/docs/html sdk/python/feast/protos/ sdk/python/tensorflow_metadata diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe949cee286..47ed7b479c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +default_stages: + - push repos: - repo: local hooks: @@ -5,9 +7,4 @@ repos: name: Lint stages: [ push ] language: system - entry: make lint - - id: compile-protos-python - name: Compile Python Protos - stages: [ push ] - language: system - entry: make compile-protos-python \ No newline at end of file + entry: make lint \ No newline at end of file diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 45ef3350b06..57af0bc814e 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -23,6 +23,8 @@ from setuptools.command.install import install from setuptools.command.develop import develop from setuptools.command.egg_info import egg_info + from setuptools.command.sdist import sdist + except ImportError: from distutils.core import setup from distutils.command.install import install @@ -61,9 +63,9 @@ # README file from Feast repo root directory repo_root = ( subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) - .communicate()[0] - .rstrip() - .decode("utf-8") + .communicate()[0] + .rstrip() + .decode("utf-8") ) README_FILE = os.path.join(repo_root, "README.md") with open(os.path.join(README_FILE), "r") as f: @@ -76,20 +78,15 @@ r"^(?:[\/\w-]+)?(?P[vV]?\d+(?:\.\d+){0,2}[^\+]*)(?:\+.*)?$" ) - proto_dirs = [x for x in os.listdir(repo_root + "/protos/feast")] proto_dirs.remove("third_party") def pre_install_build(): - """ - Build Python protos when installing Feast using setup.py - """ subprocess.check_call("make compile-protos-python", shell=True, cwd=f"{repo_root}") subprocess.check_call("make build-sphinx", shell=True, cwd=f"{repo_root}") -# Classes used to inject pre_install_build() class CustomInstallCommand(install): def do_egg_install(self): pre_install_build() @@ -116,7 +113,7 @@ def run(self): long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, url=URL, - packages=find_packages(exclude=("tests",)), + packages=find_packages(exclude=("tests",)) + ['.'], install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] @@ -143,6 +140,8 @@ def run(self): "protos/feast/**/*.proto", "protos/feast/third_party/grpc/health/v1/*.proto", "protos/tensorflow_metadata/proto/v0/*.proto", + "feast/protos/feast/**/*.py", + "tensorflow_metadata/proto/v0/*.py" ], }, cmdclass={ From d730066f2702accfd7935ea30b4687f348f63214 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 18:57:08 -0700 Subject: [PATCH 05/12] Remove explicit compilation step from proto building Signed-off-by: Willem Pienaar --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7ad80052447..3445bf3fdb2 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ compile-protos-python: @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto -install-python: compile-protos-python +install-python: python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver test-python: From 850c595267e57a56819c34b3138a0f55f85df414 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 18:58:50 -0700 Subject: [PATCH 06/12] Fix Proto CI for Go Signed-off-by: Willem Pienaar --- .github/workflows/unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 82adad18692..00c927d950f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,6 +19,8 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install dependencies + run: make install-go-ci-dependencies + - name: Compile Protos run: make compile-protos-go - - name: Test go + - name: Test run: make test-go \ No newline at end of file From 8e600327ba1d39b489a79d7ea99c10cbcfcbc60a Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 19:04:47 -0700 Subject: [PATCH 07/12] Fix Proto CI for Python Signed-off-by: Willem Pienaar --- .github/workflows/unit_tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 00c927d950f..a6e52c890c8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,13 @@ jobs: container: gcr.io/kf-feast/feast-ci:latest steps: - uses: actions/checkout@v2 - - name: Install python + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python + - name: Install Python run: make install-python - - name: Test python + - name: Test Python run: make test-python unit-test-go: @@ -20,7 +24,7 @@ jobs: - uses: actions/checkout@v2 - name: Install dependencies run: make install-go-ci-dependencies - - name: Compile Protos + - name: Compile protos run: make compile-protos-go - name: Test run: make test-go \ No newline at end of file From 95a27304fbd62b61a44e728226c5f51afdf4caa2 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 19:07:06 -0700 Subject: [PATCH 08/12] Fix Python Integration Test Dependencies Signed-off-by: Willem Pienaar --- .github/workflows/integration_tests.yml | 4 ++++ .github/workflows/pr_integration_tests.yml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index fcfb1c66875..e4a45b724ee 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -17,6 +17,10 @@ jobs: export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python - name: Install python run: make install-python - name: Test python diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index c99db076e3a..354ece44949 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -28,6 +28,11 @@ jobs: export_default_credentials: true project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} + - uses: actions/checkout@v2 + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Compile protos + run: make compile-protos-python - name: Install python run: make install-python - name: Test python From 8768fa0bb9797da3d4bb9aa620e2cf6352001678 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 20:57:04 -0700 Subject: [PATCH 09/12] Add printenv to compile protos Signed-off-by: Willem Pienaar --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3445bf3fdb2..91bfdf70f89 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos compile-protos-python: + printenv @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto From 8dc00a385b1fda7a11d6ccf2c2bc281377357519 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 20 Mar 2021 22:04:04 -0700 Subject: [PATCH 10/12] Add mypy-protobuf installation to compile-protos-python Signed-off-by: Willem Pienaar --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 91bfdf70f89..3011a100038 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ package-protos: compile-protos-python: printenv + pip install mypy-protobuf || true @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto From 1a1116e39f85e800b78a52b305d72fd9dbc428e2 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 21 Mar 2021 09:23:01 -0700 Subject: [PATCH 11/12] Install Mypy Signed-off-by: Willem Pienaar --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3011a100038..358fe399178 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ package-protos: compile-protos-python: printenv - pip install mypy-protobuf || true + pip install --ignore-installed mypy-protobuf @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto @@ -131,4 +131,4 @@ compile-protos-docs: cd ${ROOT_DIR}/protos && protoc --docs_out=../dist/grpc feast/*/*.proto build-sphinx: compile-protos-python - cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source \ No newline at end of file + cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source From 0ba8dc712d6faec5b074b00f3816687eec74a2c2 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 21 Mar 2021 09:38:20 -0700 Subject: [PATCH 12/12] Fix telemetry test Signed-off-by: Willem Pienaar --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 358fe399178..c2205898a9d 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ package-protos: compile-protos-python: printenv - pip install --ignore-installed mypy-protobuf + pip install --ignore-installed mypy-protobuf || echo "Mypy could not be installed" @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto