Skip to content

Add Synapse Launcher #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xiaoyongzhu wants to merge 23 commits into
base: master
Choose a base branch
from
Open

Add Synapse Launcher #61

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
8a1a310
Add Synapse Launcher
xiaoyongzhu Apr 25, 2021
8c06c45
Remove unnecessary printout
xiaoyongzhu Apr 26, 2021
b3fd983
Add eventhub support
xiaoyongzhu May 8, 2021
24f0986
Merge branch 'master' of https://github.com/xiaoyongzhu/feast-spark
xiaoyongzhu May 19, 2021
89f48a3
Add EventHub support and Redis Auth support
xiaoyongzhu May 19, 2021
f2cd8be
Adding EventHub support in Spark jobs
xiaoyongzhu May 19, 2021
53d7e20
add ScheduledBatchIngestionJobParameters
xiaoyongzhu Aug 31, 2021
79866e9
Merge pull request #1 from feast-dev/master
xiaoyongzhu Aug 31, 2021
c469ee7
Add Azure specific dependencies
xiaoyongzhu Sep 2, 2021
91c6822
Change azure storage dependencies
xiaoyongzhu Sep 2, 2021
0a7a56c
Commen for removing/adding spaces between brackets
xiaoyongzhu Sep 9, 2021
f4c0d5a
Delete feature_store_debug.py
xiaoyongzhumsft Sep 9, 2021
6bc9260
Update StreamingPipeline.scala
xiaoyongzhumsft Sep 14, 2021
dd53a53
Merge branch 'feast-dev:master' into master
xiaoyongzhumsft Oct 1, 2021
9fde235
Update synapse.py
xiaoyongzhumsft Oct 1, 2021
bb3d6be
Update synapse.py
xiaoyongzhumsft Oct 1, 2021
6885f31
Merge branch 'feast-dev:master' into master
xiaoyongzhumsft Oct 4, 2021
08da84f
Fix Redis auth issue
xiaoyongzhu Oct 12, 2021
0ddbcef
Update Ingestion jobs and add supporting files
xiaoyongzhu Oct 12, 2021
48a1c44
Fix build issues
xiaoyongzhu Oct 12, 2021
762386e
Add support for Kafka ingestion
xiaoyongzhu Oct 12, 2021
41fc406
Add build and push instructions
xiaoyongzhu Oct 12, 2021
0f7d433
Adding License
xiaoyongzhu Oct 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix Redis auth issue
  • Loading branch information
@xiaoyongzhu
xiaoyongzhu committed Oct 12, 2021
commit 08da84f6286a04d678ce436ee17d0978057b93de
7 changes: 6 additions & 1 deletion python/feast_spark/pyspark/abc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ def __init__(
redis_host: str,
redis_port: int,
redis_ssl: bool,
redis_auth: str,
statsd_host: Optional[str] = None,
statsd_port: Optional[int] = None,
deadletter_path: Optional[str] = None,
Expand All @@ -355,14 +356,15 @@ def __init__(
self._redis_host = redis_host
self._redis_port = redis_port
self._redis_ssl = redis_ssl
self._redis_auth = redis_auth
self._statsd_host = statsd_host
self._statsd_port = statsd_port
self._deadletter_path = deadletter_path
self._stencil_url = stencil_url
self._drop_invalid_rows = drop_invalid_rows

def _get_redis_config(self):
return dict(host=self._redis_host, port=self._redis_port, ssl=self._redis_ssl)
return dict(host=self._redis_host, port=self._redis_port, ssl=self._redis_ssl, auth=self._redis_auth)

def _get_statsd_config(self):
return (
Expand Down Expand Up @@ -424,6 +426,7 @@ def __init__(
redis_host: str,
redis_port: int,
redis_ssl: bool,
redis_auth: str,
bigtable_project: Optional[str] = None,
bigtable_instance: Optional[str] = None,
cassandra_host: Optional[str] = None,
Expand All @@ -440,6 +443,7 @@ def __init__(
redis_host,
redis_port,
redis_ssl,
redis_auth,
statsd_host,
statsd_port,
deadletter_path,
Expand Down Expand Up @@ -552,6 +556,7 @@ def __init__(
redis_host,
redis_port,
redis_ssl,
redis_auth,
statsd_host,
statsd_port,
deadletter_path,
Expand Down
1 change: 1 addition & 0 deletions python/feast_spark/pyspark/launcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ def start_offline_to_online_ingestion(
redis_port=bool(client.config.get(opt.REDIS_HOST))
and client.config.getint(opt.REDIS_PORT),
redis_ssl=client.config.getboolean(opt.REDIS_SSL),
redis_auth=client.config.get(opt.REDIS_AUTH),
bigtable_project=client.config.get(opt.BIGTABLE_PROJECT),
bigtable_instance=client.config.get(opt.BIGTABLE_INSTANCE),
cassandra_host=client.config.get(opt.CASSANDRA_HOST),
Expand Down
4 changes: 3 additions & 1 deletion python/feast_spark/pyspark/launchers/synapse/synapse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import List, Optional, cast

from azure.synapse.spark.models import SparkBatchJob
from azure.identity import DefaultAzureCredential, DeviceCodeCredential
from azure.identity import DefaultAzureCredential, DeviceCodeCredential, ChainedTokenCredential, ManagedIdentityCredential,EnvironmentCredential

from feast_spark.pyspark.abc import (
BatchIngestionJob,
Expand Down Expand Up @@ -239,6 +239,7 @@ def offline_to_online_ingestion(
job_info = _submit_job(self._api, ingestion_job_params.get_project(), main_file,
main_class = ingestion_job_params.get_class_name(),
arguments = ingestion_job_params.get_arguments(),
reference_files=[main_file],
tags = _prepare_job_tags(ingestion_job_params, OFFLINE_TO_ONLINE_JOB_TYPE),configuration=None)

return cast(BatchIngestionJob, self._job_from_job_info(job_info))
Expand Down Expand Up @@ -269,6 +270,7 @@ def start_stream_to_online_ingestion(
main_class = ingestion_job_params.get_class_name(),
arguments = ingestion_job_params.get_arguments(),
reference_files = extra_jar_paths,
configuration=None,
tags = tags)

return cast(StreamIngestionJob, self._job_from_job_info(job_info))
Expand Down