Skip to content

Commit 698ec10

Browse files
committed
allow https url for spark ingestion jar
Signed-off-by: Jacob Klegar <jacob@tecton.ai>
1 parent cd57909 commit 698ec10

2 files changed

Lines changed: 7 additions & 5 deletions

File tree

sdk/python/feast/pyspark/launchers/aws/emr_utils.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,14 @@ def _random_string(length) -> str:
8080
return "".join(random.choice(string.ascii_lowercase) for _ in range(length))
8181

8282

83-
def _upload_jar(jar_s3_prefix: str, local_path: str) -> str:
84-
with open(local_path, "rb") as f:
85-
uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(local_path)))
83+
def _upload_jar(jar_s3_prefix: str, jar_path: str) -> str:
84+
if jar_path.startswith("https://"):
85+
return jar_path
86+
with open(jar_path, "rb") as f:
87+
uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(jar_path)))
8688
return urlunparse(
8789
get_staging_client(uri.scheme).upload_fileobj(
88-
f, local_path, remote_uri=uri,
90+
f, jar_path, remote_uri=uri,
8991
)
9092
)
9193

sdk/python/feast/pyspark/launchers/k8s/k8s.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ def historical_feature_retrieval(
276276
return cast(RetrievalJob, self._job_from_job_info(job_info))
277277

278278
def _upload_jar(self, jar_path: str) -> str:
279-
if jar_path.startswith("s3://") or jar_path.startswith("s3a://"):
279+
if jar_path.startswith("s3://") or jar_path.startswith("s3a://") or jar_path.startswith("https://"):
280280
return jar_path
281281
elif jar_path.startswith("file://"):
282282
local_jar_path = urlparse(jar_path).path

0 commit comments

Comments
 (0)