Skip to content

Commit beb30b7

Browse files
woopwoop
authored
Fix documentation building for Feast SDK (feast-dev#1400)
* Update documentation and proto building Signed-off-by: Willem Pienaar <git@willem.co> * Automatically build API docs Signed-off-by: Willem Pienaar <git@willem.co> * Update formatting Signed-off-by: Willem Pienaar <git@willem.co> * Fix SDK release building Signed-off-by: Willem Pienaar <git@willem.co> * Remove explicit compilation step from proto building Signed-off-by: Willem Pienaar <git@willem.co> * Fix Proto CI for Go Signed-off-by: Willem Pienaar <git@willem.co> * Fix Proto CI for Python Signed-off-by: Willem Pienaar <git@willem.co> * Fix Python Integration Test Dependencies Signed-off-by: Willem Pienaar <git@willem.co> * Add printenv to compile protos Signed-off-by: Willem Pienaar <git@willem.co> * Add mypy-protobuf installation to compile-protos-python Signed-off-by: Willem Pienaar <git@willem.co> * Install Mypy Signed-off-by: Willem Pienaar <git@willem.co> * Fix telemetry test Signed-off-by: Willem Pienaar <git@willem.co>
1 parent 046e8e2 commit beb30b7

13 files changed

Lines changed: 108 additions & 40 deletions

File tree

‎.github/workflows/integration_tests.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ jobs:
1717
export_default_credentials: true
1818
project_id: ${{ secrets.GCP_PROJECT_ID }}
1919
service_account_key: ${{ secrets.GCP_SA_KEY }}
20+
- name: Install dependencies
21+
run: make install-python-ci-dependencies
22+
- name: Compile protos
23+
run: make compile-protos-python
2024
- name: Install python
2125
run: make install-python
2226
- name: Test python

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ jobs:
2828
export_default_credentials: true
2929
project_id: ${{ secrets.GCP_PROJECT_ID }}
3030
service_account_key: ${{ secrets.GCP_SA_KEY }}
31+
- uses: actions/checkout@v2
32+
- name: Install dependencies
33+
run: make install-python-ci-dependencies
34+
- name: Compile protos
35+
run: make compile-protos-python
3136
- name: Install python
3237
run: make install-python
3338
- name: Test python

‎.github/workflows/unit_tests.yml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@ jobs:
88
container: gcr.io/kf-feast/feast-ci:latest
99
steps:
1010
- uses: actions/checkout@v2
11-
- name: Install python
11+
- name: Install dependencies
12+
run: make install-python-ci-dependencies
13+
- name: Compile protos
14+
run: make compile-protos-python
15+
- name: Install Python
1216
run: make install-python
13-
- name: Test python
17+
- name: Test Python
1418
run: make test-python
1519

1620
unit-test-go:
@@ -19,6 +23,8 @@ jobs:
1923
steps:
2024
- uses: actions/checkout@v2
2125
- name: Install dependencies
26+
run: make install-go-ci-dependencies
27+
- name: Compile protos
2228
run: make compile-protos-go
23-
- name: Test go
29+
- name: Test
2430
run: make test-go

‎.gitignore‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -178,14 +178,15 @@ dmypy.json
178178
# This pom should not be committed because it is only used during release / deployment.
179179
.flattened-pom.xml
180180

181-
sdk/python/docs/html
182-
183-
# Generated python code
184-
*_pb2.py
185-
*_pb2.pyi
186-
*_pb2_grpc.py
187181

188182
# VSCode
189183
.bloop
190184
.metals
191185
*.code-workspace
186+
187+
# Protos
188+
sdk/python/docs/source
189+
sdk/python/docs/html
190+
sdk/python/feast/protos/
191+
sdk/python/tensorflow_metadata
192+
sdk/go/protos/

‎.pre-commit-config.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1+
default_stages:
2+
- push
13
repos:
24
- repo: local
35
hooks:
46
- id: lint
57
name: Lint
6-
stages: [commit]
8+
stages: [ push ]
79
language: system
810
entry: make lint

‎.readthedocs.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
version: 2
2+
3+
sphinx:
4+
configuration: sdk/python/docs/source/conf.py
5+
6+
formats:
7+
- pdf
8+
9+
python:
10+
version: 3.7
11+
install:
12+
- requirements: sdk/python/requirements-ci.txt
13+
- path: sdk/python/
14+
method: setuptools

‎Makefile‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,14 @@ install-python-ci-dependencies:
4444
package-protos:
4545
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
4646

47-
compile-protos-python: install-python-ci-dependencies
47+
compile-protos-python:
48+
printenv
49+
pip install --ignore-installed mypy-protobuf || echo "Mypy could not be installed"
4850
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
4951
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -i@ sed -i 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
5052
cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto
5153

52-
install-python: compile-protos-python
54+
install-python:
5355
python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver
5456

5557
test-python:
@@ -74,7 +76,7 @@ install-go-ci-dependencies:
7476
go get -u github.com/golang/protobuf/protoc-gen-go
7577
go get -u golang.org/x/lint/golint
7678

77-
compile-protos-go: install-go-ci-dependencies
79+
compile-protos-go:
7880
cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ tensorflow_metadata/proto/v0/*.proto
7981
$(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;)
8082

@@ -124,22 +126,9 @@ install-dependencies-proto-docs:
124126
mv protoc3/include/* $$HOME/include
125127

126128
compile-protos-docs:
127-
cd ${ROOT_DIR}/protos;
128-
mkdir -p ../dist/grpc
129-
protoc --docs_out=../dist/grpc feast/*/*.proto || \
130-
cd ${ROOT_DIR}; $(MAKE) install-dependencies-proto-docs && cd ${ROOT_DIR}/protos; PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto
129+
rm -rf $(ROOT_DIR)/dist/grpc
130+
mkdir -p dist/grpc;
131+
cd ${ROOT_DIR}/protos && protoc --docs_out=../dist/grpc feast/*/*.proto
131132

132-
clean-html:
133-
rm -rf $(ROOT_DIR)/dist
134-
135-
build-html: clean-html
136-
mkdir -p $(ROOT_DIR)/dist/python
137-
mkdir -p $(ROOT_DIR)/dist/grpc
138-
139-
# Build Protobuf documentation
140-
$(MAKE) compile-protos-docs
141-
142-
# Build Python SDK documentation
143-
$(MAKE) compile-protos-python
144-
cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) html
145-
cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python
133+
build-sphinx: compile-protos-python
134+
cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source

‎sdk/python/.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,5 +115,4 @@ dmypy.json
115115
.pyre/
116116

117117
.vscode/*
118-
test.py
119118
playground

‎sdk/python/docs/Makefile‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,17 @@ help:
1515

1616
.PHONY: help Makefile source
1717

18-
clean:
19-
rm -rf $(PROJECT_ROOT)/sdk/python/docs/html
18+
clean-source:
2019
rm -rf $(PROJECT_ROOT)/sdk/python/docs/source
2120

22-
build-api-source:
21+
clean-html:
22+
rm -rf $(PROJECT_ROOT)/sdk/python/docs/html
23+
24+
build-api-source: clean-source
2325
sphinx-apidoc -f -o source $(PROJECT_ROOT)/sdk/python/feast
2426
cp conf.py index.rst source/
2527

26-
html: clean build-api-source
28+
html: clean-html build-api-source
2729
sphinx-build -b html source html
2830
rm -rf $(PROJECT_ROOT)/sdk/python/docs/source
2931

‎sdk/python/feast/protos/tensorflow_metadata/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)