5959 " ### Configuration"
6060 ]
6161 },
62- {
63- "cell_type" : " code"
64- "execution_count" : null ,
65- "metadata" : {},
66- "outputs" : [],
67- "source" : [
68- " import os"
69- ]
70- },
71- {
72- "cell_type" : " code"
73- "execution_count" : 54 ,
74- "metadata" : {},
75- "outputs" : [],
76- "source" : [
77- " # os.environ['FEAST_SPARK_LAUNCHER'] = 'standalone'\n "
78- " # os.environ['FEAST_SPARK_STANDALONE_MASTER'] = 'local[*]'\n "
79- " # os.environ['FEAST_SPARK_HOME'] = os.path.dirname(pyspark.__file__)\n "
80- " # os.environ['FEAST_SPARK_EXTRA_OPTIONS'] = '--jars https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar'\\\n "
81- " # ' --conf spark.hadoop.fs.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem'"
82- ]
83- },
84- {
85- "cell_type" : " code"
86- "execution_count" : null ,
87- "metadata" : {},
88- "outputs" : [],
89- "source" : [
90- " os.environ['FEAST_SPARK_STAGING_LOCATION'] = \" gs://feast-templocation-kf-feast/demo/staging/\"\n "
91- " os.environ['FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION'] = \" gs://feast-templocation-kf-feast/demo/output\" "
92- ]
93- },
94- {
95- "cell_type" : " code"
96- "execution_count" : 4 ,
97- "metadata" : {},
98- "outputs" : [],
99- "source" : [
100- " # os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \" /path/to/key\" "
101- ]
102- },
10362 {
10463 "cell_type" : " markdown"
10564 "metadata" : {},
12685 "metadata" : {},
12786 "outputs" : [],
12887 "source" : [
129- " CORE_HOST = os.getenv(\" DEMO_FEAST_CORE_SERVICE_HOST \" , \" localhost \" )\n "
130- " SERVING_HOST = os.getenv(\" DEMO_FEAST_ONLINE_SERVING_SERVICE_HOST \" , \" localhost \" )\n "
131- " REDIS_HOST = os.getenv('DEMO_REDIS_MASTER_SERVICE_HOST', 'localhost') \n "
88+ " REDIS_HOST = os.getenv('DEMO_REDIS_MASTER_SERVICE_HOST', os.getenv('FEAST_REDIS_HOST') )\n "
89+ " DEMO_DATA_LOCATION = os.getenv(\" DEMO_DATA_LOCATION \" )\n "
90+ " # os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \" /path/to/key \" \n"
13291 " \n "
13392 " client = Client(\n "
134- " core_url=f \" {CORE_HOST} :6565\" ,\n "
135- " serving_url=f \" {SERVING_HOST} :6566\" , \n "
93+ " core_url=os.getenv('FEAST_CORE_URL', \" localhost :6565\" ) ,\n "
94+ " serving_url=os.getenv('FEAST_ONLINE_SERVING_URL', \" localhost :6566\" ), \n "
13695 " redis_host=REDIS_HOST\n "
13796 " )"
13897 ]
251210 " event_timestamp_column=\" datetime\" ,\n "
252211 " created_timestamp_column=\" created\" ,\n "
253212 " file_format=ParquetFormat(),\n "
254- " file_url=\" gs://feast-demo-data-lake/ driver_statistics\" ,\n "
213+ " file_url=os.path.join(DEMO_DATA_LOCATION, \" driver_statistics\" ) ,\n "
255214 " date_partition_column=\" date\"\n "
256215 " )\n "
257216 " )"
273232 " event_timestamp_column=\" datetime\" ,\n "
274233 " created_timestamp_column=\" created\" ,\n "
275234 " file_format=ParquetFormat(),\n "
276- " file_url=\" gs://feast-demo-data-lake/ driver_trips\" ,\n "
235+ " file_url=os.path.join(DEMO_DATA_LOCATION, \" driver_trips\" ) ,\n "
277236 " date_partition_column=\" date\"\n "
278237 " )\n "
279238 " )"
563522 "metadata" : {},
564523 "outputs" : [],
565524 "source" : [
566- " import gcsfs \n "
525+ " \n "
567526 " from pyarrow.parquet import ParquetDataset"
568527 ]
569528 },
574533 "outputs" : [],
575534 "source" : [
576535 " def read_remote_parquet(path):\n "
577- " fs = gcsfs.GCSFileSystem()\n "
578- " files = [\" gs://\" + path for path in gcsfs.GCSFileSystem().glob(path)]\n "
536+ " prefix = path[:5]\n "
537+ " if prefix == 'gs://':\n "
538+ " import gcsfs\n "
539+ " fs = gcsfs.GCSFileSystem()\n "
540+ " elif prefix == 's3://':\n "
541+ " import s3fs\n "
542+ " fs = s3fs.S3FileSystem()\n "
543+ " else:\n "
544+ " raise Exception(f'Unsupported fs {prefix}')\n "
545+ " files = [prefix + path for path in fs.glob(path)]\n "
579546 " ds = ParquetDataset(files, filesystem=fs)\n "
580547 " return ds.read().to_pandas()"
581548 ]
11381105 "metadata" : {},
11391106 "outputs" : [],
11401107 "source" : [
1141- " KAFKA_BROKER = \" kafka:9092\" "
1108+ " KAFKA_BROKER = os.getenv( \" DEMO_KAFKA_BROKERS \" , \" kafka:9092\" ) "
11421109 ]
11431110 },
11441111 {
12351202 "metadata" : {},
12361203 "outputs" : [],
12371204 "source" : [
1238- " entities_sample = np.random.choice(entities, 10)\n "
1205+ " entities_sample = np.random.choice(entities, 10, replace=False )\n "
12391206 " entities_sample = [{\" driver_id\" : e} for e in entities_sample]\n "
12401207 " entities_sample"
12411208 ]
12981265 },
12991266 "nbformat" : 4 ,
13001267 "nbformat_minor" : 4
1301- }
1268+ }
0 commit comments