feat: Support GCS filesystem for bytewax engine (#3774)

sudohainguyen · web-flow · commit fb6b807f8b32 · 2023-10-20T15:13:49.000-07:00
* fix: Support param timeout when persisting

Signed-off-by: Hai Nguyen &lt;quanghai.ng1512@gmail.com&gt;

* fix: fix java integration test

Signed-off-by: Hai Nguyen &lt;quanghai.ng1512@gmail.com&gt;

---------

Signed-off-by: Hai Nguyen &lt;quanghai.ng1512@gmail.com&gt;
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py
@@ -2,7 +2,6 @@
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-import s3fs
 from bytewax.dataflow import Dataflow  # type: ignore
 from bytewax.execution import cluster_main
 from bytewax.inputs import ManualInputConfig, distribute
@@ -29,8 +28,7 @@ def __init__(
         self._run_dataflow()
 
     def process_path(self, path):
-        fs = s3fs.S3FileSystem()
-        dataset = pq.ParquetDataset(path, filesystem=fs, use_legacy_dataset=False)
+        dataset = pq.ParquetDataset(path, use_legacy_dataset=False)
         batches = []
         for fragment in dataset.fragments:
             for batch in fragment.to_table().to_batches():
diff --git a/setup.py b/setup.py
@@ -91,6 +91,7 @@
     "google-cloud-datastore>=2.1.0,<3",
     "google-cloud-storage>=1.34.0,<3",
     "google-cloud-bigtable>=2.11.0,<3",
+    "gcsfs",
 ]
 
 REDIS_REQUIRED = [
@@ -158,7 +159,7 @@
         "moto",
         "mypy>=0.981,<0.990",
         "avro==1.10.0",
-        "gcsfs>=0.4.0,<=2022.01.0",
+        "gcsfs",
         "urllib3>=1.25.4,<2",
         "psutil==5.9.0",
         "py>=1.11.0",  # https://github.com/pytest-dev/pytest/issues/10420
