Skip to content

Commit 7561970

Browse files
authored
allow https url for spark ingestion jar (#1266)
* allow https url for spark ingestion jar Signed-off-by: Jacob Klegar <jacob@tecton.ai> * lint Signed-off-by: Jacob Klegar <jacob@tecton.ai>
1 parent cd57909 commit 7561970

2 files changed

Lines changed: 11 additions & 7 deletions

File tree

sdk/python/feast/pyspark/launchers/aws/emr_utils.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,13 @@ def _random_string(length) -> str:
8080
return "".join(random.choice(string.ascii_lowercase) for _ in range(length))
8181

8282

83-
def _upload_jar(jar_s3_prefix: str, local_path: str) -> str:
84-
with open(local_path, "rb") as f:
85-
uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(local_path)))
83+
def _upload_jar(jar_s3_prefix: str, jar_path: str) -> str:
84+
if jar_path.startswith("https://"):
85+
return jar_path
86+
with open(jar_path, "rb") as f:
87+
uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(jar_path)))
8688
return urlunparse(
87-
get_staging_client(uri.scheme).upload_fileobj(
88-
f, local_path, remote_uri=uri,
89-
)
89+
get_staging_client(uri.scheme).upload_fileobj(f, jar_path, remote_uri=uri)
9090
)
9191

9292

sdk/python/feast/pyspark/launchers/k8s/k8s.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,11 @@ def historical_feature_retrieval(
276276
return cast(RetrievalJob, self._job_from_job_info(job_info))
277277

278278
def _upload_jar(self, jar_path: str) -> str:
279-
if jar_path.startswith("s3://") or jar_path.startswith("s3a://"):
279+
if (
280+
jar_path.startswith("s3://")
281+
or jar_path.startswith("s3a://")
282+
or jar_path.startswith("https://")
283+
):
280284
return jar_path
281285
elif jar_path.startswith("file://"):
282286
local_jar_path = urlparse(jar_path).path

0 commit comments

Comments
 (0)