Skip to content

Commit bb960a7

Browse files
Yansonfeast-ci-bot
Yanson
authored and
feast-ci-bot
committed
Parameterize end to end test scripts. (#433)
1 parent eefdc35 commit bb960a7

2 files changed

Lines changed: 94 additions & 54 deletions

File tree

‎.prow/scripts/test-end-to-end-batch.sh‎

Lines changed: 59 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,32 @@ set -e
44
set -o pipefail
55

66
if ! cat /etc/*release | grep -q stretch; then
7-
echo ${BASH_SOURCE} only supports Debian stretch.
7+
echo ${BASH_SOURCE} only supports Debian stretch.
88
echo Please change your operating system to use this script.
99
exit 1
1010
fi
1111

12+
test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json"
13+
test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
14+
test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
15+
test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
16+
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
17+
test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT"
18+
1219
echo "
1320
This script will run end-to-end tests for Feast Core and Batch Serving.
1421
1522
1. Install gcloud SDK
1623
2. Install Redis as the job store for Feast Batch Serving.
1724
4. Install Postgres for persisting Feast metadata.
1825
5. Install Kafka and Zookeeper as the Source in Feast.
19-
6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
26+
6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
2027
tests/e2e via pytest.
2128
"
2229

30+
apt-get -qq update
31+
apt-get -y install wget netcat kafkacat
32+
2333

2434
echo "
2535
============================================================
@@ -31,8 +41,8 @@ if [[ ! $(command -v gsutil) ]]; then
3141
. "${CURRENT_DIR}"/install-google-cloud-sdk.sh
3242
fi
3343

34-
export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json
35-
gcloud auth activate-service-account --key-file /etc/service-account/service-account.json
44+
export GOOGLE_APPLICATION_CREDENTIALS
45+
gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
3646

3747

3848

@@ -41,10 +51,9 @@ echo "
4151
Installing Redis at localhost:6379
4252
============================================================
4353
"
44-
apt-get -qq update
4554
# Allow starting serving in this Maven Docker image. Default set to not allowed.
4655
echo "exit 0" > /usr/sbin/policy-rc.d
47-
apt-get -y install redis-server wget > /var/log/redis.install.log
56+
apt-get -y install redis-server > /var/log/redis.install.log
4857
redis-server --daemonize yes
4958
redis-cli ping
5059

@@ -73,24 +82,32 @@ Installing Kafka at localhost:9092
7382
wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz
7483
mv kafka_2.12-2.3.0/ /tmp/kafka
7584
nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 &
76-
sleep 10
85+
sleep 5
7786
tail -n10 /var/log/zookeeper.log
7887
nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 &
79-
sleep 30
88+
sleep 20
8089
tail -n10 /var/log/kafka.log
81-
82-
echo "
83-
============================================================
84-
Building jars for Feast
85-
============================================================
86-
"
87-
88-
.prow/scripts/download-maven-cache.sh \
89-
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
90-
--output-dir /root/
91-
92-
# Build jars for Feast
93-
mvn --quiet --batch-mode --define skipTests=true clean package
90+
kafkacat -b localhost:9092 -L
91+
92+
if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
93+
echo "
94+
============================================================
95+
Building jars for Feast
96+
============================================================
97+
"
98+
99+
.prow/scripts/download-maven-cache.sh \
100+
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
101+
--output-dir /root/
102+
103+
# Build jars for Feast
104+
mvn --quiet --batch-mode --define skipTests=true clean package
105+
106+
ls -lh core/target/*jar
107+
ls -lh serving/target/*jar
108+
else
109+
echo "[DEBUG] Skipping building jars"
110+
fi
94111

95112
echo "
96113
============================================================
@@ -142,11 +159,13 @@ management:
142159
enabled: false
143160
EOF
144161

145-
nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
162+
nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \
146163
--spring.config.location=file:///tmp/core.application.yml \
147164
&> /var/log/feast-core.log &
148165
sleep 35
149166
tail -n10 /var/log/feast-core.log
167+
nc -w2 localhost 6565 < /dev/null
168+
150169
echo "
151170
============================================================
152171
Starting Feast Warehouse Serving
@@ -155,18 +174,18 @@ Starting Feast Warehouse Serving
155174

156175
DATASET_NAME=feast_$(date +%s)
157176

158-
bq --location=US --project_id=kf-feast mk \
177+
bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
159178
--dataset \
160179
--default_table_expiration 86400 \
161-
kf-feast:$DATASET_NAME
180+
${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
162181

163182
# Start Feast Online Serving in background
164183
cat <<EOF > /tmp/serving.store.bigquery.yml
165184
name: warehouse
166185
type: BIGQUERY
167186
bigquery_config:
168-
projectId: kf-feast
169-
datasetId: $DATASET_NAME
187+
projectId: ${GOOGLE_CLOUD_PROJECT}
188+
datasetId: ${DATASET_NAME}
170189
subscriptions:
171190
- name: "*"
172191
version: "*"
@@ -183,24 +202,27 @@ feast:
183202
store:
184203
config-path: /tmp/serving.store.bigquery.yml
185204
jobs:
186-
staging-location: gs://feast-templocation-kf-feast/staging-location
205+
staging-location: ${JOBS_STAGING_LOCATION}
187206
store-type: REDIS
188207
store-options:
189-
host: $REMOTE_HOST
208+
host: localhost
190209
port: 6379
191210
grpc:
192211
port: 6566
193212
enable-reflection: true
213+
194214
spring:
195215
main:
196216
web-environment: false
217+
197218
EOF
198219

199-
nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
220+
nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \
200221
--spring.config.location=file:///tmp/serving.warehouse.application.yml \
201222
&> /var/log/feast-serving-warehouse.log &
202223
sleep 15
203224
tail -n100 /var/log/feast-serving-warehouse.log
225+
nc -w2 localhost 6566 < /dev/null
204226

205227
echo "
206228
============================================================
@@ -230,9 +252,15 @@ ORIGINAL_DIR=$(pwd)
230252
cd tests/e2e
231253

232254
set +e
233-
pytest bq-batch-retrieval.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
255+
pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
234256
TEST_EXIT_CODE=$?
235257

258+
if [[ ${TEST_EXIT_CODE} != 0 ]]; then
259+
echo "[DEBUG] Printing logs"
260+
ls -ltrh /var/log/feast*
261+
cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
262+
fi
263+
236264
cd ${ORIGINAL_DIR}
237265
exit ${TEST_EXIT_CODE}
238266

@@ -242,4 +270,4 @@ Cleaning up
242270
============================================================
243271
"
244272

245-
bq rm -r -f kf-feast:$DATASET_NAME
273+
bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}

‎.prow/scripts/test-end-to-end.sh‎

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,25 @@ set -e
44
set -o pipefail
55

66
if ! cat /etc/*release | grep -q stretch; then
7-
echo ${BASH_SOURCE} only supports Debian stretch.
7+
echo ${BASH_SOURCE} only supports Debian stretch.
88
echo Please change your operating system to use this script.
99
exit 1
1010
fi
1111

12+
test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json"
13+
test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
14+
test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
15+
test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
16+
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
17+
test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT"
18+
1219
echo "
1320
This script will run end-to-end tests for Feast Core and Online Serving.
1421
1522
1. Install Redis as the store for Feast Online Serving.
1623
2. Install Postgres for persisting Feast metadata.
1724
3. Install Kafka and Zookeeper as the Source in Feast.
18-
4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
25+
4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
1926
tests/e2e via pytest.
2027
"
2128

@@ -27,7 +34,6 @@ echo "
2734
Installing Redis at localhost:6379
2835
============================================================
2936
"
30-
3137
# Allow starting serving in this Maven Docker image. Default set to not allowed.
3238
echo "exit 0" > /usr/sbin/policy-rc.d
3339
apt-get -y install redis-server > /var/log/redis.install.log
@@ -66,21 +72,25 @@ sleep 20
6672
tail -n10 /var/log/kafka.log
6773
kafkacat -b localhost:9092 -L
6874

69-
echo "
70-
============================================================
71-
Building jars for Feast
72-
============================================================
73-
"
75+
if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
76+
echo "
77+
============================================================
78+
Building jars for Feast
79+
============================================================
80+
"
7481

75-
.prow/scripts/download-maven-cache.sh \
76-
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
77-
--output-dir /root/
82+
.prow/scripts/download-maven-cache.sh \
83+
--archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
84+
--output-dir /root/
7885

79-
# Build jars for Feast
80-
mvn --quiet --batch-mode --define skipTests=true clean package
86+
# Build jars for Feast
87+
mvn --quiet --batch-mode --define skipTests=true clean package
8188

82-
ls -lh core/target/*jar
83-
ls -lh serving/target/*jar
89+
ls -lh core/target/*jar
90+
ls -lh serving/target/*jar
91+
else
92+
echo "[DEBUG] Skipping building jars"
93+
fi
8494

8595
echo "
8696
============================================================
@@ -118,7 +128,6 @@ spring:
118128
event.merge.entity_copy_observer: allow
119129
hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
120130
hibernate.ddl-auto: update
121-
122131
datasource:
123132
url: jdbc:postgresql://localhost:5432/postgres
124133
username: postgres
@@ -133,7 +142,7 @@ management:
133142
enabled: false
134143
EOF
135144

136-
nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
145+
nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \
137146
--spring.config.location=file:///tmp/core.application.yml \
138147
&> /var/log/feast-core.log &
139148
sleep 35
@@ -163,17 +172,14 @@ feast:
163172
version: 0.3
164173
core-host: localhost
165174
core-grpc-port: 6565
166-
167175
tracing:
168176
enabled: false
169-
170177
store:
171178
config-path: /tmp/serving.store.redis.yml
172179
redis-pool-max-size: 128
173180
redis-pool-max-idle: 16
174-
175181
jobs:
176-
staging-location: gs://feast-templocation-kf-feast/staging-location
182+
staging-location: ${JOBS_STAGING_LOCATION}
177183
store-type:
178184
store-options: {}
179185
@@ -187,11 +193,11 @@ spring:
187193
188194
EOF
189195

190-
nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
196+
nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \
191197
--spring.config.location=file:///tmp/serving.online.application.yml \
192198
&> /var/log/feast-serving-online.log &
193199
sleep 15
194-
tail -n10 /var/log/feast-serving-online.log
200+
tail -n100 /var/log/feast-serving-online.log
195201
nc -w2 localhost 6566 < /dev/null
196202

197203
echo "
@@ -225,5 +231,11 @@ set +e
225231
pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
226232
TEST_EXIT_CODE=$?
227233

234+
if [[ ${TEST_EXIT_CODE} != 0 ]]; then
235+
echo "[DEBUG] Printing logs"
236+
ls -ltrh /var/log/feast*
237+
cat /var/log/feast-serving-online.log /var/log/feast-core.log
238+
fi
239+
228240
cd ${ORIGINAL_DIR}
229241
exit ${TEST_EXIT_CODE}

0 commit comments

Comments
 (0)