fix: Bytewax engine create configmap from object (feast-dev#3821)

sudohainguyen · tqtensor · commit d2a6c276bed3 · 2024-03-11T02:46:22.000Z
fix: bytewax engine create configmap from object

Signed-off-by: Hai Nguyen &lt;quanghai.ng1512@gmail.com&gt;
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py
@@ -111,6 +111,7 @@ def __init__(
         self.offline_store = offline_store
         self.online_store = online_store
 
+        # TODO: Configure k8s here
         k8s_config.load_config()
 
         self.k8s_client = client.api_client.ApiClient()
@@ -298,9 +299,6 @@ def _create_kubernetes_job(self, job_id, paths, feature_view):
                 len(paths),  # Create a pod for each parquet file
                 self.batch_engine_config.env,
             )
-            logger.info(
-                f"Created job `dataflow-{job_id}` on namespace `{self.namespace}`"
-            )
         except FailToCreateError as failures:
             return BytewaxMaterializationJob(job_id, self.namespace, error=failures)
 
@@ -309,11 +307,7 @@ def _create_kubernetes_job(self, job_id, paths, feature_view):
     def _create_configuration_map(self, job_id, paths, feature_view, namespace):
         """Create a Kubernetes configmap for this job"""
 
-        if self.repo_config.repo_path:
-            feature_store_path = get_default_yaml_file_path(self.repo_config.repo_path)
-            feature_store_configuration = feature_store_path.read_text()
-        else:
-            feature_store_configuration = yaml.dump(self.repo_config.dict())
+        feature_store_configuration = yaml.dump(self.repo_config.dict())
 
         materialization_config = yaml.dump(
             {"paths": paths, "feature_view": feature_view.name}
@@ -364,7 +358,7 @@ def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0):
             },
             {
                 "name": "BYTEWAX_REPLICAS",
-                "value": "1",
+                "value": f"{pods}",
             },
             {
                 "name": "BYTEWAX_KEEP_CONTAINER_ALIVE",
