SAASMLOPS-769 make max parallelism configurable

james-crabtree-sp · james-crabtree-sp · commit 3ec2d3ef3462 · 2023-09-14T11:11:40.000-05:00
Signed-off-by: James Crabtree &lt;james.crabtree@sailpoint.com&gt;
diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py
@@ -64,6 +64,9 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel):
     labels: dict = {}
     """ (optional) additional labels to append to kubernetes objects """
 
+    max_parallelism: int = 10
+    """ (optional) Maximum number of pods  (default 10) allowed to run in parallel per job"""
+
 
 class BytewaxMaterializationEngine(BatchMaterializationEngine):
     def __init__(
@@ -275,7 +278,7 @@ def _create_job_definition(self, job_id, namespace, pods, env):
             "spec": {
                 "ttlSecondsAfterFinished": 3600,
                 "completions": pods,
-                "parallelism": pods,
+                "parallelism": min(pods, self.batch_engine_config.max_parallelism),
                 "completionMode": "Indexed",
                 "template": {
                     "metadata": {
