SAASMLOPS-809 fix bytewax workers so they only process a single file (#6)

james-crabtree-sp · james-crabtree-sp · commit 2313c1dc5bfd · 2023-10-23T11:41:28.000-05:00
* SAASMLOPS-809 fix bytewax workers so they only process a single file

* SAASMLOPS-809 fix newlines

Signed-off-by: James Crabtree &lt;james.crabtree@sailpoint.com&gt;
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py
@@ -5,7 +5,7 @@
 import pyarrow.parquet as pq
 from bytewax.dataflow import Dataflow  # type: ignore
 from bytewax.execution import cluster_main
-from bytewax.inputs import ManualInputConfig, distribute
+from bytewax.inputs import ManualInputConfig
 from bytewax.outputs import ManualOutputConfig
 from tqdm import tqdm
 
@@ -21,17 +21,25 @@ def __init__(
         config: RepoConfig,
         feature_view: FeatureView,
         paths: List[str],
+        worker_index: int
     ):
         self.config = config
         self.feature_store = FeatureStore(config=config)
 
         self.feature_view = feature_view
+        self.worker_index = worker_index
         self.paths = paths
 
         self._run_dataflow()
 
     def process_path(self, path):
+<<<<<<< HEAD
         dataset = pq.ParquetDataset(path, use_legacy_dataset=False)
+=======
+        fs = s3fs.S3FileSystem()
+        logger.info(f"Processing path {path}")
+        dataset = pq.ParquetDataset(path, filesystem=fs, use_legacy_dataset=False)
+>>>>>>> 15c523a2 (SAASMLOPS-809 fix bytewax workers so they only process a single file (#6))
         batches = []
         for fragment in dataset.fragments:
             for batch in fragment.to_table().to_batches():
@@ -40,11 +48,7 @@ def process_path(self, path):
         return batches
 
     def input_builder(self, worker_index, worker_count, _state):
-        worker_paths = distribute(self.paths, worker_index, worker_count)
-        for path in worker_paths:
-            yield None, path
-
-        return
+        return [(None, self.paths[self.worker_index])]
 
     def output_builder(self, worker_index, worker_count):
         def yield_batch(iterable, batch_size):
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py
@@ -205,7 +205,7 @@ def _materialize_one(
                 try:
                     self.batch_v1.delete_namespaced_job(job.job_id(), self.namespace)
                 except ApiException as de:
-                    logger.warning(f"Could not delete job due to API Error: {ae.body}")
+                    logger.warning(f"Could not delete job due to API Error: {de.body}")
                 raise e
             self._print_pod_logs(job.job_id(), feature_view)
         return job
@@ -293,7 +293,7 @@ def _create_job_definition(self, job_id, namespace, pods, env):
             },
             {
                 "name": "BYTEWAX_REPLICAS",
-                "value": "1",
+                "value": f"{pods}",
             },
             {
                 "name": "BYTEWAX_KEEP_CONTAINER_ALIVE",
@@ -331,8 +331,8 @@ def _create_job_definition(self, job_id, namespace, pods, env):
             "spec": {
                 "ttlSecondsAfterFinished": 3600,
                 "backoffLimit": self.batch_engine_config.retry_limit,
-                "completions": 1,
-                "parallelism": 1,
+                "completions": pods,
+                "parallelism": min(pods, self.batch_engine_config.max_parallelism),
                 "completionMode": "Indexed",
                 "template": {
                     "metadata": {
@@ -349,7 +349,7 @@ def _create_job_definition(self, job_id, namespace, pods, env):
                                 "env": [
                                     {
                                         "name": "BYTEWAX_REPLICAS",
-                                        "value": "1",
+                                        "value": f"{pods}",
                                     }
                                 ],
                                 "image": "busybox",
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py
@@ -1,3 +1,4 @@
+import os
 import yaml
 
 from feast import FeatureStore, RepoConfig
@@ -19,4 +20,5 @@
                 config,
                 store.get_feature_view(bytewax_config["feature_view"]),
                 bytewax_config["paths"],
+                int(os.environ["JOB_COMPLETION_INDEX"])
             )

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+import os`
`1`	`2`	`import yaml`
`2`	`3`
`3`	`4`	`from feast import FeatureStore, RepoConfig`
`@@ -19,4 +20,5 @@`
`19`	`20`	`config,`
`20`	`21`	`store.get_feature_view(bytewax_config["feature_view"]),`
`21`	`22`	`bytewax_config["paths"],`
	`23`	`+ int(os.environ["JOB_COMPLETION_INDEX"])`
`22`	`24`	`)`