Skip to content

Commit 3b6d993

Browse files
woopwoop
authored
Refactor tests and protobuf building for Python SDK (#1418)
* Fix Makefile driven proto building Signed-off-by: Willem Pienaar <git@willem.co> * Remove commented out files Signed-off-by: Willem Pienaar <git@willem.co> * Use Python 3.7 for unit tests Signed-off-by: Willem Pienaar <git@willem.co> * Refactor dependency management Signed-off-by: Willem Pienaar <git@willem.co> * Fix Telemetry tests Signed-off-by: Willem Pienaar <git@willem.co> * Remove unused test Signed-off-by: Willem Pienaar <git@willem.co>
1 parent 8e8558c commit 3b6d993

21 files changed

+5774
-113
lines changed

‎.github/workflows/integration_tests.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88
jobs:
99
integration-test-python:
1010
runs-on: ubuntu-latest
11-
container: gcr.io/kf-feast/feast-ci:latest
11+
container: python:3.7
1212
steps:
1313
- uses: actions/checkout@v2
1414
- name: Set up Cloud SDK
@@ -21,9 +21,5 @@ jobs:
2121
run: gcloud info
2222
- name: Install dependencies
2323
run: make install-python-ci-dependencies
24-
- name: Compile protos
25-
run: make compile-protos-python
26-
- name: Install python
27-
run: make install-python
2824
- name: Test python
2925
run: FEAST_TELEMETRY=False pytest --verbose --color=yes sdk/python/tests --integration

‎.github/workflows/linter.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,12 @@ on: [push, pull_request]
44

55
jobs:
66
lint-python:
7-
container: gcr.io/kf-feast/feast-ci:latest
7+
container: python:3.7
88
runs-on: [ubuntu-latest]
99
steps:
1010
- uses: actions/checkout@v2
1111
- name: Install dependencies
1212
run: make install-python-ci-dependencies
13-
- name: Compile protos
14-
run: make compile-protos-python
1513
- name: Lint python
1614
run: make lint-python
1715

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test'))
1414
|| (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))
1515
runs-on: ubuntu-latest
16-
container: gcr.io/kf-feast/feast-ci:latest
16+
container: python:3.7
1717
steps:
1818
- uses: actions/checkout@v2
1919
with:
@@ -32,9 +32,5 @@ jobs:
3232
run: gcloud info
3333
- name: Install dependencies
3434
run: make install-python-ci-dependencies
35-
- name: Compile protos
36-
run: make compile-protos-python
37-
- name: Install python
38-
run: make install-python
3935
- name: Test python
4036
run: FEAST_TELEMETRY=False pytest --verbose --color=yes sdk/python/tests --integration

‎.github/workflows/unit_tests.yml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,11 @@ on: [push, pull_request]
55
jobs:
66
unit-test-python:
77
runs-on: ubuntu-latest
8-
container: gcr.io/kf-feast/feast-ci:latest
8+
container: python:3.7
99
steps:
1010
- uses: actions/checkout@v2
1111
- name: Install dependencies
1212
run: make install-python-ci-dependencies
13-
- name: Compile protos
14-
run: make compile-protos-python
15-
- name: Install Python
16-
run: make install-python
1713
- name: Test Python
1814
run: make test-python
1915

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,5 +188,4 @@ dmypy.json
188188
sdk/python/docs/source
189189
sdk/python/docs/html
190190
sdk/python/feast/protos/
191-
sdk/python/tensorflow_metadata
192191
sdk/go/protos/

‎.prow.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,6 @@ presubmits:
6464
branches:
6565
- ^v0\.(3|4)-branch$
6666

67-
- name: test-python-sdk
68-
decorate: true
69-
spec:
70-
containers:
71-
- image: python:3.7
72-
command: ["infra/scripts/test-python-sdk.sh"]
73-
7467
- name: test-telemetry
7568
decorate: true
7669
run_if_changed: "sdk/python/.*"

‎.readthedocs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@ formats:
99
python:
1010
version: 3.7
1111
install:
12-
- requirements: sdk/python/requirements-ci.txt
12+
- requirements: sdk/python/docs/requirements.txt
1313
- path: sdk/python/
1414
method: setuptools

‎Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,12 @@ install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependenci
3939
# Python SDK
4040

4141
install-python-ci-dependencies:
42-
pip install --no-cache-dir -r sdk/python/requirements-ci.txt
42+
pip install -e "sdk/python[ci]"
4343

4444
package-protos:
4545
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
4646

4747
compile-protos-python:
48-
pip install --ignore-installed mypy-protobuf || echo "Mypy could not be installed"
4948
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
5049
@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -I@ sed -i.bak 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
5150
cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto

‎infra/scripts/test-python-sdk.sh‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎infra/scripts/test-telemetry.sh‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,7 @@ set -e
55
# Default artifact location setting in Prow jobs
66
LOGS_ARTIFACT_PATH=/logs/artifacts
77

8-
pip install -r sdk/python/requirements-ci.txt
9-
make compile-protos-python
10-
118
cd sdk/python/
12-
pip install -e .
9+
pip install -e ".[ci]"
1310
cd telemetry_tests/
1411
pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml

0 commit comments

Comments
 (0)