Skip to content

Commit 3c274f6

Browse files
author
zhilingc
committed
Add integration test for redis-bq-dataflow
1 parent b9e724d commit 3c274f6

52 files changed

Lines changed: 1432 additions & 40 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 119 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,122 @@ vendor
4646
.direnv
4747

4848
.terraform/
49-
*.tfvars
49+
*.tfvars
50+
51+
# python
52+
# Byte-compiled / optimized / DLL files
53+
__pycache__/
54+
*.py[cod]
55+
*$py.class
56+
57+
# C extensions
58+
*.so
59+
60+
# Distribution / packaging
61+
.Python
62+
build/
63+
develop-eggs/
64+
dist/
65+
downloads/
66+
eggs/
67+
.eggs/
68+
lib/
69+
lib64/
70+
parts/
71+
sdist/
72+
var/
73+
wheels/
74+
share/python-wheels/
75+
*.egg-info/
76+
.installed.cfg
77+
*.egg
78+
MANIFEST
79+
80+
# PyInstaller
81+
# Usually these files are written by a python script from a template
82+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
83+
*.manifest
84+
*.spec
85+
86+
# Installer logs
87+
pip-log.txt
88+
pip-delete-this-directory.txt
89+
90+
# Unit test / coverage reports
91+
htmlcov/
92+
.tox/
93+
.nox/
94+
.coverage
95+
.coverage.*
96+
.cache
97+
nosetests.xml
98+
coverage.xml
99+
*.cover
100+
.hypothesis/
101+
.pytest_cache/
102+
103+
# Translations
104+
*.mo
105+
*.pot
106+
107+
# Django stuff:
108+
*.log
109+
local_settings.py
110+
db.sqlite3
111+
112+
# Flask stuff:
113+
instance/
114+
.webassets-cache
115+
116+
# Scrapy stuff:
117+
.scrapy
118+
119+
# Sphinx documentation
120+
docs/_build/
121+
122+
# PyBuilder
123+
target/
124+
125+
# Jupyter Notebook
126+
.ipynb_checkpoints
127+
128+
# IPython
129+
profile_default/
130+
ipython_config.py
131+
132+
# pyenv
133+
.python-version
134+
135+
# celery beat schedule file
136+
celerybeat-schedule
137+
138+
# SageMath parsed files
139+
*.sage.py
140+
141+
# Environments
142+
.env
143+
.venv
144+
env/
145+
venv/
146+
ENV/
147+
env.bak/
148+
venv.bak/
149+
150+
# Spyder project settings
151+
.spyderproject
152+
.spyproject
153+
154+
# Rope project settings
155+
.ropeproject
156+
157+
# mkdocs documentation
158+
/site
159+
160+
# mypy
161+
.mypy_cache/
162+
.dmypy.json
163+
dmypy.json
164+
165+
# Pyre type checker
166+
.pyre/
167+
.vscode

‎Makefile‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,28 @@
1-
VERSION_FILE=VERSION
2-
FEAST_VERSION=`cat $(VERSION_FILE)`
1+
#
2+
# Copyright 2019 The Feast Authors
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
VERSION_FILE = VERSION
18+
FEAST_VERSION = `cat $(VERSION_FILE)`
319

420
test:
521
mvn test
622

23+
test-integration:
24+
$(MAKE) -C testing/integration test-integration ID=$(ID)
25+
726
build-deps:
827
$(MAKE) -C protos gen-go
928
dep ensure

‎core/src/test/java/feast/core/CoreApplicationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
*/
4949
@RunWith(SpringRunner.class)
5050
@SpringBootTest(properties = {
51-
"feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
51+
"feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
5252
"spring.datasource.url=jdbc:h2:mem:testdb",
5353
"feast.store.warehouse.type=file.json",
5454
"feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}",

‎core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
*/
5050
@RunWith(SpringRunner.class)
5151
@SpringBootTest(properties = {
52-
"feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
52+
"feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
5353
"spring.datasource.url=jdbc:h2:mem:testdb",
5454
"feast.store.warehouse.type=file.json",
5555
"feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}",

‎core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
*/
5050
@RunWith(SpringRunner.class)
5151
@SpringBootTest(properties = {
52-
"feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
52+
"feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
5353
"spring.datasource.url=jdbc:h2:mem:testdb",
5454
"feast.store.serving.type=redis",
5555
"feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}",

‎testing/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,6 @@ update-config: get-cluster-credentials
3232
update-plugins: get-cluster-credentials
3333
kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f -
3434

35-
build-push:
36-
docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION)
35+
build-push-test-image:
36+
docker build -f docker-images/test-image/Dockerfile -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) .
3737
$(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)"

‎testing/README.md‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ This folder contains the Feast test infrastructure.
44

55
## Components
66

7-
* test-image/ - Base docker image and script for running tests.
7+
* docker-images/ - Docker images for running tests
88
* prow/ - Prow configuration (plugins and jobs)
9-
* tf/ - Terraform modules to provision the base testing infrastructure on GCP
9+
* infrastructure/ - Terraform modules to provision the base testing infrastructure on GCP
1010

1111
## Set up
1212

@@ -23,7 +23,7 @@ mv tf/gcs
2323
terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state
2424
```
2525

26-
4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated.
26+
4. Ensure that all variables are set correctly in `infrastructure/terraform.tfvars`. It is likely that the GCP project will need to be updated.
2727

2828
5. Create the primary Kubernetes cluster which will host Prow and Argo
2929

@@ -51,7 +51,11 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o
5151

5252
- `prow/config.yaml`
5353
- `prow/plugins.yaml`
54-
- `test-image/Dockerfile`
55-
- `test-image/run.sh`
54+
- `docker-images/test-image/Dockerfile`
55+
- `docker-images/test-image/run.sh`
5656

57-
After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry.
57+
After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry.
58+
59+
## Installing Argo
60+
61+
Argo is used to run integration tests. It follows the [standard installation process described in the argo quickstart](https://github.com/argoproj/argo/blob/master/demo.md), with the artifact repository [configured to use gcs](https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md).

‎testing/docker-images/docker-builder-image/Dockerfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM docker:18.09
2+
3+
# from https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/alpine/Dockerfile
4+
ENV CLOUD_SDK_VERSION=236.0.0
5+
6+
ENV PATH /google-cloud-sdk/bin:$PATH
7+
RUN apk --no-cache add \
8+
curl \
9+
python \
10+
py-crcmod \
11+
bash \
12+
libc6-compat \
13+
openssh-client \
14+
git \
15+
gnupg \
16+
&& curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
17+
tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
18+
rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
19+
ln -s /lib /lib64 && \
20+
gcloud config set core/disable_usage_reporting true && \
21+
gcloud config set component_manager/disable_update_check true && \
22+
gcloud config set metrics/environment github_docker_image && \
23+
gcloud --version
24+
25+
RUN gcloud auth configure-docker

‎testing/docker-images/terraform-image/Dockerfile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM debian:stretch
2+
3+
RUN apt-get update && apt-get install -y jq wget python zip netcat
4+
5+
RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \
6+
&& /google-cloud-sdk/bin/gcloud -q components install kubectl \
7+
&& /google-cloud-sdk/bin/gcloud -q components install beta \
8+
&& /google-cloud-sdk/bin/gcloud -q components install bq
9+
10+
ENV PATH="/google-cloud-sdk/bin:${PATH}"
11+
12+
RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \
13+
&& unzip terraform_0.11.11_linux_amd64.zip \
14+
&& mv terraform /bin/terraform
15+
16+
RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \
17+
&& mv linux-arm64/helm /bin/helm
18+
19+
RUN helm init --client-only

testing/test-image/Dockerfile renamed to testing/docker-images/test-image/Dockerfile

File renamed without changes.

0 commit comments

Comments
 (0)