Skip to content

Commit 6b7a844

Browse files
pyalexpyalex
authored
Fix flaky tests (feast-dev#953)
* debug flaky tests * trailing slash * trailing slash * debug * correct log path * testcontainers for ingestion unit tests * fix rounding error * cleanup if success * close resources * unique feast topic per e2e test * remove debug * f-string in python unit tests * lint * round event timestamp to seconds * add line about event_timestamp to limitation * add line about event_timestamp to limitation
1 parent e37c2c0 commit 6b7a844

12 files changed

Lines changed: 160 additions & 320 deletions

File tree

‎docs/reference/limitations.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,5 @@
4444
| Limitation | Motivation |
4545
| :--- | :--- |
4646
| Once data has been ingested into Feast, there is currently no way to delete the data without manually going to the database and deleting it. However, during retrieval only the latest rows will be returned for a specific key \(`event_timestamp`, `entity`\) based on its `created_timestamp`. | This functionality simply doesn't exist yet as a Feast API |
47+
| During the ingestion of data into BigQuery, `event_timestamp` is rounded down to seconds. E.g., `2020-08-21T08:40:19.906 -> 2020-08-21T08:40:19.000` | This ensures that floating point rounding errors do not occur during the retrieval of feature data, since this step requires time based joins |
4748

‎infra/scripts/test-end-to-end-batch-dataflow.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ test -z ${HELM_RELEASE_NAME} && HELM_RELEASE_NAME="pr-$PULL_NUMBER"
1515
test -z ${HELM_COMMON_NAME} && HELM_COMMON_NAME="deps"
1616
test -z ${DATASET_NAME} && DATASET_NAME=feast_e2e_$(date +%s)
1717
test -z ${SPECS_TOPIC} && SPECS_TOPIC=feast-specs-$(date +%s)
18+
test -z ${FEATURES_TOPIC} && FEATURES_TOPIC=feast-$(date +%s)
1819

1920

2021
feast_kafka_1_ip_name="feast-kafka-1"
@@ -212,6 +213,7 @@ export GCLOUD_REGION=$GCLOUD_REGION
212213
export HELM_COMMON_NAME=$HELM_COMMON_NAME
213214
export IMAGE_TAG=$PULL_PULL_SHA
214215
export SPECS_TOPIC=$SPECS_TOPIC
216+
export FEATURES_TOPIC=$FEATURES_TOPIC
215217

216218
export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
217219
export SCRIPTS_DIR=${PROJECT_ROOT_DIR}/infra/scripts
@@ -220,7 +222,7 @@ source ${SCRIPTS_DIR}/setup-common-functions.sh
220222
wait_for_docker_image gcr.io/kf-feast/feast-core:"${IMAGE_TAG}"
221223
wait_for_docker_image gcr.io/kf-feast/feast-serving:"${IMAGE_TAG}"
222224

223-
envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC \
225+
envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC $FEATURES_TOPIC \
224226
$GCLOUD_SUBNET $GCLOUD_REGION $IMAGE_TAG $HELM_COMMON_NAME $feast_kafka_1_ip
225227
$feast_kafka_2_ip $feast_kafka_3_ip $feast_redis_ip $feast_statsd_ip' < $ORIGINAL_DIR/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml > $ORIGINAL_DIR/infra/charts/feast/values-end-to-end-batch-dataflow-updated.yaml
226228

@@ -288,7 +290,7 @@ jobcontroller_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}
288290

289291
set +e
290292
pytest -s -v bq/bq-batch-retrieval.py -m dataflow_runner --core_url "$core_ip:6565" --serving_url "$serving_ip:6566" \
291-
--jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
293+
--jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
292294
TEST_EXIT_CODE=$?
293295

294296
if [[ ${TEST_EXIT_CODE} != 0 ]]; then

‎infra/scripts/test-end-to-end-batch.sh‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/et
2121
test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
2222
test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
2323
test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
24-
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
24+
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location/$(date +%s)"
2525

2626
# Get the current build version using maven (and pom.xml)
2727
export FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
@@ -56,6 +56,12 @@ else
5656
echo "[DEBUG] Skipping building jars"
5757
fi
5858

59+
DATASET_NAME=feast_$(date +%s)
60+
bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
61+
--dataset \
62+
--default_table_expiration 86400 \
63+
${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
64+
5965
# Start Feast Core in background
6066
cat <<EOF > /tmp/jc.warehouse.application.yml
6167
feast:
@@ -73,16 +79,6 @@ feast:
7379
7480
EOF
7581

76-
start_feast_core
77-
start_feast_jobcontroller /tmp/jc.warehouse.application.yml
78-
79-
DATASET_NAME=feast_$(date +%s)
80-
bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
81-
--dataset \
82-
--default_table_expiration 86400 \
83-
${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
84-
85-
# Start Feast Online Serving in background
8682
cat <<EOF > /tmp/serving.warehouse.application.yml
8783
feast:
8884
# GRPC service address for Feast Core
@@ -122,6 +118,10 @@ server:
122118
123119
EOF
124120

121+
cat /tmp/jc.warehouse.application.yml /tmp/serving.warehouse.application.yml
122+
123+
start_feast_core
124+
start_feast_jobcontroller /tmp/jc.warehouse.application.yml
125125
start_feast_serving /tmp/serving.warehouse.application.yml
126126

127127
install_python_with_miniconda_and_feast_sdk
@@ -134,21 +134,20 @@ ORIGINAL_DIR=$(pwd)
134134
cd tests/e2e
135135

136136
set +e
137-
pytest bq/* -m ${PYTEST_MARK} --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
137+
pytest bq/* -v -m ${PYTEST_MARK} --gcs_path ${JOBS_STAGING_LOCATION} --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
138138
TEST_EXIT_CODE=$?
139139

140140
if [[ ${TEST_EXIT_CODE} != 0 ]]; then
141141
echo "[DEBUG] Printing logs"
142142
ls -ltrh /var/log/feast*
143-
cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
143+
cat /var/log/feast-serving-online.log /var/log/feast-core.log /var/log/feast-jobcontroller.log
144144

145145
echo "[DEBUG] Printing Python packages list"
146146
pip list
147-
fi
148-
149-
cd ${ORIGINAL_DIR}
147+
else
148+
print_banner "Cleaning up"
150149

151-
print_banner "Cleaning up"
150+
bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
151+
fi
152152

153-
bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
154153
exit ${TEST_EXIT_CODE}

‎infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ feast-core:
1515
stream:
1616
options:
1717
bootstrapServers: $feast_kafka_1_ip:31090
18+
topic: $FEATURES_TOPIC
1819

1920
feast-jobcontroller:
2021
enabled: true

‎ingestion/pom.xml‎

Lines changed: 7 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@
144144
<version>${project.version}</version>
145145
</dependency>
146146

147+
<dependency>
148+
<groupId>dev.feast</groupId>
149+
<artifactId>feast-common-test</artifactId>
150+
<version>${project.version}</version>
151+
<scope>test</scope>
152+
</dependency>
153+
147154
<dependency>
148155
<groupId>com.google.auto.value</groupId>
149156
<artifactId>auto-value-annotations</artifactId>
@@ -155,20 +162,6 @@
155162
<artifactId>google-cloud-bigquery</artifactId>
156163
</dependency>
157164

158-
<dependency>
159-
<groupId>org.hamcrest</groupId>
160-
<artifactId>hamcrest-core</artifactId>
161-
</dependency>
162-
<dependency>
163-
<groupId>org.hamcrest</groupId>
164-
<artifactId>hamcrest-library</artifactId>
165-
</dependency>
166-
167-
<dependency>
168-
<groupId>org.mockito</groupId>
169-
<artifactId>mockito-core</artifactId>
170-
</dependency>
171-
172165
<dependency>
173166
<groupId>com.google.protobuf</groupId>
174167
<artifactId>protobuf-java</artifactId>
@@ -178,13 +171,6 @@
178171
<artifactId>protobuf-java-util</artifactId>
179172
</dependency>
180173

181-
<dependency>
182-
<groupId>junit</groupId>
183-
<artifactId>junit</artifactId>
184-
<version>4.12</version>
185-
<scope>test</scope>
186-
</dependency>
187-
188174
<dependency>
189175
<groupId>org.apache.kafka</groupId>
190176
<artifactId>kafka-clients</artifactId>
@@ -223,19 +209,6 @@
223209
<artifactId>slf4j-api</artifactId>
224210
</dependency>
225211

226-
<!-- To run actual Redis for ingestion integration test -->
227-
<dependency>
228-
<groupId>com.github.kstyrc</groupId>
229-
<artifactId>embedded-redis</artifactId>
230-
</dependency>
231-
232-
<!-- To run actual Kafka for ingestion integration test -->
233-
<dependency>
234-
<groupId>org.apache.kafka</groupId>
235-
<artifactId>kafka_2.12</artifactId>
236-
<scope>test</scope>
237-
</dependency>
238-
239212
<dependency>
240213
<groupId>org.slf4j</groupId>
241214
<artifactId>slf4j-simple</artifactId>

0 commit comments

Comments
 (0)