Skip to content

Commit 7226bde

Browse files
jklegarjklegar
authored
fix azure blob storage access in e2e tests (#1253)
* fix azure blob storage access Signed-off-by: Jacob Klegar <jacob@tecton.ai> * lint Signed-off-by: Jacob Klegar <jacob@tecton.ai> * bugfix Signed-off-by: Jacob Klegar <jacob@tecton.ai>
1 parent 7885b08 commit 7226bde

File tree

7 files changed

+26
-17
lines changed

7 files changed

+26
-17
lines changed

‎.prow/config.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,16 @@ presubmits:
312312
secretKeyRef:
313313
name: feast-az-creds
314314
key: AZ_SERVICE_PRINCIPAL_TENANT_ID
315+
- name: AZURE_BLOB_ACCOUNT_NAME
316+
valueFrom:
317+
secretKeyRef:
318+
name: feast-az-creds
319+
key: AZURE_BLOB_ACCOUNT_NAME
320+
- name: AZURE_BLOB_ACCOUNT_ACCESS_KEY
321+
valueFrom:
322+
secretKeyRef:
323+
name: feast-az-creds
324+
key: AZURE_BLOB_ACCOUNT_ACCESS_KEY
315325
volumeMounts:
316326
- mountPath: /etc/gcloud/service-account.json
317327
name: service-account

‎infra/scripts/azure-runner.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ time kubectl run -n "$NAMESPACE" -i ci-test-runner \
5353
--restart=Never \
5454
--image="${DOCKER_REPOSITORY}/feast-ci:${GIT_TAG}" \
5555
--env="STAGING_PATH=${STAGING_PATH}" \
56+
--env="FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
57+
--env="FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" \
5658
-- \
5759
bash -c "mkdir src && cd src && git clone ${GIT_REMOTE_URL} && cd feast && git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && git fetch -q && git checkout ${GIT_TAG} && ./infra/scripts/setup-e2e-env-sparkop.sh && ./infra/scripts/test-end-to-end-sparkop.sh"
5860

‎infra/scripts/runner-helper.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ function helm_install {
5252
--set "feast-online-serving.image.tag=${GIT_TAG}" \
5353
--set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
5454
--set "feast-jobservice.image.tag=${GIT_TAG}" \
55+
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
56+
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" \
5557
--set "feast-core.image.repository=${DOCKER_REPOSITORY}/feast-core" \
5658
--set "feast-core.image.tag=${GIT_TAG}" \
5759
--set "prometheus-statsd-exporter.enabled=false" \

‎sdk/python/feast/contrib/validation/ge.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ def apply_validation(
103103
"/"
104104
)
105105
staging_scheme = urlparse(staging_location).scheme
106-
staging_client = get_staging_client(staging_scheme)
106+
staging_client = get_staging_client(staging_scheme, client._config)
107107

108108
pickled_code_fp = io.BytesIO(udf.pickled_code)
109109
remote_path = f"{staging_location}/udfs/{udf.name}.pickle"

‎sdk/python/feast/pyspark/launcher.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,7 @@ def _get_optional(option):
6868
def _k8s_launcher(config: Config) -> JobLauncher:
6969
from feast.pyspark.launchers import k8s
7070

71-
return k8s.KubernetesJobLauncher(
72-
namespace=config.get(opt.SPARK_K8S_NAMESPACE),
73-
resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None),
74-
staging_location=config.get(opt.SPARK_STAGING_LOCATION),
75-
incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG),
76-
)
71+
return k8s.KubernetesJobLauncher(config=config)
7772

7873

7974
_launchers = {

‎sdk/python/feast/pyspark/launchers/k8s/k8s.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import yaml
1010
from kubernetes.client.api import CustomObjectsApi
1111

12+
from feast.config import Config
13+
from feast.constants import ConfigOptions as opt
1214
from feast.pyspark.abc import (
1315
BQ_SPARK_PACKAGE,
1416
BatchIngestionJob,
@@ -140,16 +142,13 @@ class KubernetesJobLauncher(JobLauncher):
140142
Submits spark jobs to a spark cluster. Currently supports only historical feature retrieval jobs.
141143
"""
142144

143-
def __init__(
144-
self,
145-
namespace: str,
146-
incluster: bool,
147-
staging_location: str,
148-
resource_template_path: Optional[Path],
149-
):
150-
self._namespace = namespace
145+
def __init__(self, config: Config):
146+
self._config = config
147+
self._namespace = config.get(opt.SPARK_K8S_NAMESPACE)
148+
incluster = config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG)
151149
self._api = _get_api(incluster=incluster)
152-
self._staging_location = staging_location
150+
self._staging_location = config.get(opt.SPARK_STAGING_LOCATION)
151+
resource_template_path = config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None)
153152
if resource_template_path is not None:
154153
self._resource_template = _load_resource_template(resource_template_path)
155154
else:
@@ -183,7 +182,7 @@ def _job_from_job_info(self, job_info: JobInfo) -> SparkJob:
183182

184183
def _get_staging_client(self):
185184
uri = urlparse(self._staging_location)
186-
return get_staging_client(uri.scheme)
185+
return get_staging_client(uri.scheme, self._config)
187186

188187
def historical_feature_retrieval(
189188
self, job_params: RetrievalJobParameters

‎sdk/python/requirements-dev.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,4 @@ moto
4141
pyspark==3.0.1
4242
pyspark-stubs==3.0.0.post1
4343
kubernetes==12.0.*
44+
azure-storage-blob==12.6.0

0 commit comments

Comments
 (0)