Skip to content

Commit e543bfb

Browse files
khorshuhengkhorshuheng
authored
Migrate sparkop e2e tests to kf-feast cluster (#62)
* Migrate sparkop e2e tests to kf-feast cluster Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Run tests within pod Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Add resource to test spark job Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Update to spark 3.0.2 Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Add java options to enable netty reflection Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>
1 parent b8f0313 commit e543bfb

File tree

5 files changed

+136
-51
lines changed

5 files changed

+136
-51
lines changed

‎.prow.yaml‎

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,11 @@ presubmits:
6666
always_run: true
6767
max_concurrency: 1
6868
spec:
69+
metadata:
70+
namespace: sparkop-e2e
6971
containers:
7072
- image: gcr.io/kf-feast/feast-ci:latest
71-
command: [ "infra/scripts/aws-runner.sh", "--location-from-prow", "--project-name", "feast-ci-sparkop-project" ]
73+
command: [ "infra/scripts/test-end-to-end-sparkop.sh"]
7274
resources:
7375
requests:
7476
cpu: "2"
@@ -78,18 +80,6 @@ presubmits:
7880
value: /etc/gcloud/service-account.json
7981
- name: DOCKER_REPOSITORY
8082
value: gcr.io/kf-feast
81-
- name: AWS_ACCESS_KEY_ID
82-
valueFrom:
83-
secretKeyRef:
84-
name: feast-aws-creds
85-
key: AWS_ACCESS_KEY_ID
86-
- name: AWS_SECRET_ACCESS_KEY
87-
valueFrom:
88-
secretKeyRef:
89-
name: feast-aws-creds
90-
key: AWS_SECRET_ACCESS_KEY
91-
- name: AWS_DEFAULT_REGION
92-
value: us-west-2
9383
volumeMounts:
9484
- mountPath: /etc/gcloud/service-account.json
9585
name: service-account
@@ -235,9 +225,11 @@ postsubmits:
235225
branches:
236226
- ^master$
237227
spec:
228+
metadata:
229+
namespace: sparkop-e2e
238230
containers:
239231
- image: gcr.io/kf-feast/feast-ci:latest
240-
command: [ "infra/scripts/aws-runner.sh", "--location-from-prow", "--project-name", "feast-ci-sparkop-project" ]
232+
command: [ "infra/scripts/test-end-to-end-sparkop.sh"]
241233
resources:
242234
requests:
243235
cpu: "2"
@@ -247,18 +239,6 @@ postsubmits:
247239
value: /etc/gcloud/service-account.json
248240
- name: DOCKER_REPOSITORY
249241
value: gcr.io/kf-feast
250-
- name: AWS_ACCESS_KEY_ID
251-
valueFrom:
252-
secretKeyRef:
253-
name: feast-aws-creds
254-
key: AWS_ACCESS_KEY_ID
255-
- name: AWS_SECRET_ACCESS_KEY
256-
valueFrom:
257-
secretKeyRef:
258-
name: feast-aws-creds
259-
key: AWS_SECRET_ACCESS_KEY
260-
- name: AWS_DEFAULT_REGION
261-
value: us-west-2
262242
volumeMounts:
263243
- mountPath: /etc/gcloud/service-account.json
264244
name: service-account

‎infra/scripts/helm/k8s-jobservice.tpl.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
feast-jobservice:
2+
envOverrides:
3+
FEAST_CORE_URL: feast-release-feast-core:6565
4+
FEAST_SPARK_LAUNCHER: k8s
5+
FEAST_SPARK_K8S_NAMESPACE: sparkop-e2e
6+
FEAST_SPARK_K8S_USE_INCLUSTER_CONFIG: True
7+
FEAST_TELEMETRY: False
8+
FEAST_SPARK_STAGING_LOCATION: gs://feast-templocation-kf-feast
9+
FEAST_REDIS_HOST: feast-release-redis-master
10+
FEAST_REDIS_PORT: 6379
11+
FEAST_JOB_SERVICE_ENABLE_CONTROL_LOOP: False
12+
FEAST_SPARK_INGESTION_JAR: local:///opt/spark/jars/feast-ingestion-spark-${IMAGE_TAG}.jar
13+
14+
sparkOperator:
15+
enabled: true
16+
jobTemplate:
17+
apiVersion: "sparkoperator.k8s.io/v1beta2"
18+
kind: SparkApplication
19+
spec:
20+
type: Scala
21+
mode: cluster
22+
image: "gcr.io/kf-feast/feast-spark:${IMAGE_TAG}"
23+
hadoopConf:
24+
"fs.gs.project.id": "kf-feast"
25+
"google.cloud.auth.service.account.enable": "true"
26+
"google.cloud.auth.service.account.json.keyfile": "/mnt/secrets/credentials.json"
27+
sparkVersion: "3.0.2"
28+
timeToLiveSeconds: 3600
29+
pythonVersion: "3"
30+
restartPolicy:
31+
type: Never
32+
driver:
33+
cores: 1
34+
coreLimit: "1200m"
35+
memory: "600m"
36+
labels:
37+
version: 3.0.2
38+
javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
39+
secrets:
40+
- name: feast-gcp-service-account
41+
path: /mnt/secrets
42+
secretType: GCPServiceAccount
43+
executor:
44+
cores: 1
45+
instances: 1
46+
memory: "800m"
47+
labels:
48+
version: 3.0.2
49+
javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
50+
secrets:
51+
- name: feast-gcp-service-account
52+
path: /mnt/secrets
53+
secretType: GCPServiceAccount
54+

‎infra/scripts/k8s-common-functions.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,18 @@ function helm_install {
9191
exit 1
9292
fi
9393

94+
if [ -z ${JOBSERVICE_HELM_VALUES:-} ]; then
95+
export HELM_ARGS="--set feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice \
96+
--set feast-jobservice.image.tag=${JOBSERVICE_GIT_TAG:-$GIT_TAG} \
97+
--set prometheus-statsd-exporter.enabled=false $@";
98+
else
99+
export HELM_ARGS="--values ${JOBSERVICE_HELM_VALUES}";
100+
fi
101+
94102
if ! time helm install --wait "$RELEASE" "${HELM_CHART_LOCATION:-./infra/charts/feast-spark}" \
95103
--timeout 5m \
96-
--set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
97-
--set "feast-jobservice.image.tag=${JOBSERVICE_GIT_TAG:-$GIT_TAG}" \
98-
--set "prometheus-statsd-exporter.enabled=false" \
99104
--namespace "$NAMESPACE" \
100-
"$@" ; then
105+
${HELM_ARGS}; then
101106

102107
echo "Error during helm install. "
103108
kubectl -n "$NAMESPACE" get pods
@@ -142,4 +147,4 @@ subjects:
142147
- kind: ServiceAccount
143148
name: default
144149
EOF
145-
}
150+
}

‎infra/scripts/test-end-to-end-sparkop.sh‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,69 @@
11
#!/usr/bin/env bash
22

3-
set -euo pipefail
3+
set -e
44

5-
pip install "s3fs" "aiobotocore==1.2.2" "boto3" "urllib3>=1.25.4"
5+
export GIT_TAG=${PULL_PULL_SHA:-${PULL_BASE_SHA}}
6+
export GIT_REMOTE_URL=https://github.com/feast-dev/feast-spark.git
7+
export DOCKER_REPOSITORY=gcr.io/kf-feast
8+
export JOBSERVICE_HELM_VALUES=infra/scripts/helm/k8s-jobservice.yaml
69

7-
export DISABLE_FEAST_SERVICE_FIXTURES=1
8-
export DISABLE_SERVICE_FIXTURES=1
9-
export FEAST_TELEMETRY="False"
10+
test -z ${GCLOUD_PROJECT} && GCLOUD_PROJECT="kf-feast"
11+
test -z ${GCLOUD_REGION} && GCLOUD_REGION="us-central1"
12+
test -z ${GCLOUD_NETWORK} && GCLOUD_NETWORK="default"
13+
test -z ${GCLOUD_SUBNET} && GCLOUD_SUBNET="default"
14+
test -z ${KUBE_CLUSTER} && KUBE_CLUSTER="feast-e2e-dataflow"
1015

11-
export FEAST_SPARK_K8S_NAMESPACE=sparkop
16+
gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
17+
gcloud -q auth configure-docker
1218

13-
PYTHONPATH=sdk/python pytest tests/e2e/ \
14-
--feast-version develop \
15-
--core-url feast-release-feast-core:6565 \
16-
--serving-url feast-release-feast-online-serving:6566 \
17-
--env k8s \
18-
--staging-path $STAGING_PATH \
19-
--redis-url feast-release-redis-master.sparkop.svc.cluster.local:6379 \
20-
--kafka-brokers feast-release-kafka.sparkop.svc.cluster.local:9092 \
21-
-m "not bq"
19+
gcloud config set project ${GCLOUD_PROJECT}
20+
gcloud config set compute/region ${GCLOUD_REGION}
21+
gcloud config list
22+
23+
gcloud container clusters get-credentials ${KUBE_CLUSTER} --region ${GCLOUD_REGION} --project ${GCLOUD_PROJECT}
24+
25+
source infra/scripts/k8s-common-functions.sh
26+
27+
NAMESPACE="sparkop-e2e"
28+
29+
k8s_cleanup "feast-release" "$NAMESPACE"
30+
k8s_cleanup "js" "$NAMESPACE"
31+
32+
kubectl delete sparkapplication --all -n $NAMESPACE
33+
kubectl delete scheduledsparkapplication --all -n $NAMESPACE
34+
35+
wait_for_image "${DOCKER_REPOSITORY}" feast-jobservice "${GIT_TAG}"
36+
wait_for_image "${DOCKER_REPOSITORY}" feast-spark "${GIT_TAG}"
37+
38+
sed s/\$\{IMAGE_TAG\}/${JOBSERVICE_GIT_TAG:-$GIT_TAG}/g infra/scripts/helm/k8s-jobservice.tpl.yaml > $JOBSERVICE_HELM_VALUES
39+
40+
helm_install "js" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \
41+
--set 'feast-online-serving.application-override\.yaml.feast.stores[0].type=REDIS' \
42+
--set 'feast-online-serving.application-override\.yaml.feast.stores[0].name=online' \
43+
--set 'feast-online-serving.application-override\.yaml.feast.stores[0].config.host=feast-release-redis-master' \
44+
--set 'feast-online-serving.application-override\.yaml.feast.stores[0].config.port=6379' \
45+
46+
CMD=$(printf '%s' \
47+
"mkdir src && cd src && git clone --recursive ${GIT_REMOTE_URL} && cd feast-spark && " \
48+
"git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && " \
49+
"git fetch -q && git checkout ${GIT_TAG} && " \
50+
"make install-python && " \
51+
"python -m pip install -qr tests/requirements.txt && " \
52+
"pytest -v tests/e2e/ --env k8s " \
53+
"--staging-path gs://feast-templocation-kf-feast/ " \
54+
"--core-url feast-release-feast-core:6565 " \
55+
"--serving-url feast-release-feast-online-serving:6566 " \
56+
"--job-service-url js-feast-jobservice:6568 " \
57+
"--kafka-brokers feast-release-kafka-headless:9092 --bq-project kf-feast --feast-version dev")
58+
59+
# Delete old test running pod if it exists
60+
kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true
61+
62+
kubectl run -n "$NAMESPACE" -i ci-test-runner \
63+
--pod-running-timeout=5m \
64+
--restart=Never \
65+
--image="${DOCKER_REPOSITORY}/feast-ci:latest" \
66+
--env="FEAST_TELEMETRY=false" \
67+
--env="DISABLE_FEAST_SERVICE_FIXTURES=1" \
68+
--env="DISABLE_SERVICE_FIXTURES=1" \
69+
-- bash -c "$CMD"

‎tests/e2e/fixtures/client.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,12 @@ def feast_client(
9595
return Client(
9696
core_url=f"{feast_core[0]}:{feast_core[1]}",
9797
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
98-
spark_launcher="k8s",
99-
spark_staging_location=os.path.join(local_staging_path, "k8s"),
100-
spark_ingestion_jar=ingestion_job_jar,
101-
redis_host=pytestconfig.getoption("redis_url").split(":")[0],
102-
redis_port=pytestconfig.getoption("redis_url").split(":")[1],
10398
historical_feature_output_location=os.path.join(
10499
local_staging_path, "historical_output"
105100
),
101+
spark_staging_location=os.path.join(local_staging_path, "k8s"),
106102
enable_auth=pytestconfig.getoption("enable_auth"),
103+
**job_service_env,
107104
)
108105
else:
109106
raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")
@@ -185,6 +182,7 @@ def tfrecord_feast_client(
185182
historical_feature_output_location=os.path.join(
186183
local_staging_path, "historical_output"
187184
),
185+
**job_service_env,
188186
)
189187
else:
190188
raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")

0 commit comments

Comments
 (0)