Skip to content

Make demo notebook work on AWS #1097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
khorshuheng merged 1 commit into from
Oct 27, 2020
Merged

Make demo notebook work on AWS #1097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions examples/minimal/Feast 101.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,10 @@
"metadata": {},
"outputs": [],
"source": [
"# Replace this with a GCS bucket accessible by the service account associated with this notebook\n",
"# datalake_gcs_bucket = \"<your gcs bucket name>\""
"# This is the location we're using for the offline feature store.\n",
"\n",
"import os\n",
"demo_data_location = os.path.join(os.getenv(\"FEAST_SPARK_STAGING_LOCATION\", \"file:///home/jovyan/\"), \"test_data\")\n"
]
},
{
Expand All @@ -268,11 +270,7 @@
"metadata": {},
"outputs": [],
"source": [
"# GCS Source\n",
"# driver_statistics_source_uri = f\"gs://{datalake_gcs_bucket}/driver_statistics\"\n",
"\n",
"# Local File System Source\n",
"driver_statistics_source_uri = f\"file:///home/jovyan/driver_statistics\"\n",
"driver_statistics_source_uri = os.path.join(demo_data_location, \"driver_statistics\")\n",
"\n",
"driver_statistics = FeatureTable(\n",
" name = \"driver_statistics\",\n",
Expand All @@ -298,11 +296,7 @@
"metadata": {},
"outputs": [],
"source": [
"# GCS Source\n",
"# driver_trips_source_uri = f\"gs://{datalake_gcs_bucket}/driver_trips\"\n",
"\n",
"# Local File System Source\n",
"driver_trips_source_uri = f\"file:///home/jovyan/driver_trips\"\n",
"driver_trips_source_uri = os.path.join(demo_data_location, \"driver_trips\")\n",
"\n",
"\n",
"driver_trips = FeatureTable(\n",
Expand Down Expand Up @@ -568,11 +562,17 @@
" return pd.read_parquet(parsed_uri.path)\n",
" elif parsed_uri.scheme == \"gs\":\n",
" fs = gcsfs.GCSFileSystem()\n",
" files = [\"gs://\" + path for path in gcsfs.GCSFileSystem().glob(uri + '/part-*')]\n",
" files = [\"gs://\" + path for path in fs.glob(uri + '/part-*')]\n",
" ds = ParquetDataset(files, filesystem=fs)\n",
" return ds.read().to_pandas()\n",
" elif parsed_uri.scheme == 's3':\n",
" import s3fs\n",
" fs = s3fs.S3FileSystem()\n",
" files = [\"s3://\" + path for path in fs.glob(uri + '/part-*')]\n",
" ds = ParquetDataset(files, filesystem=fs)\n",
" return ds.read().to_pandas()\n",
" else:\n",
" raise ValueError(\"Unsupported scheme\")"
" raise ValueError(f\"Unsupported URL scheme {uri}\")"
]
},
{
Expand Down Expand Up @@ -1178,7 +1178,7 @@
"outputs": [],
"source": [
"# Change this to any Kafka broker addresses which is accessible by the spark cluster\n",
"KAFKA_BROKER = \"kafka:9092\""
"KAFKA_BROKER = os.getenv(\"DEMO_KAFKA_BROKERS\", \"kafka:9092\")"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion infra/charts/feast/charts/feast-jupyter/templates/deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ spec:
env:
- name: FEAST_CORE_URL
value: "{{ .Release.Name }}-feast-core:6565"
- name: FEAST_ONLINE_SERVING_URL
- name: FEAST_SERVING_URL
value: "{{ .Release.Name }}-feast-online-serving:6566"
- name: FEAST_HISTORICAL_SERVING_URL
value: "{{ .Release.Name }}-feast-batch-serving:6566"
Expand Down
2 changes: 1 addition & 1 deletion infra/docker-compose/docker-compose.dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ services:
- core
environment:
FEAST_CORE_URL: core:6565
FEAST_ONLINE_SERVING_URL: online_serving:6566
FEAST_SERVING_URL: online_serving:6566
FEAST_HISTORICAL_SERVING_URL: historical_serving:6567
GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
ports:
Expand Down