feat: Snowflake skip materialization if no table change (feast-dev#3404)

sfc-gh-khaynie · web-flow · commit 0ab3942e5fa0 · 2022-12-20T07:32:29.000-08:00
* feat: Snowflake skip materialization if no table change

Signed-off-by: Katy Haynie &lt;74935161+katyhaynie-snowflake@users.noreply.github.com&gt;

* removing print statement from offline snowflake.py

Signed-off-by: Katy Haynie &lt;74935161+katyhaynie-snowflake@users.noreply.github.com&gt;

Signed-off-by: Katy Haynie &lt;74935161+katyhaynie-snowflake@users.noreply.github.com&gt;
diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py
@@ -8,6 +8,7 @@
 import pandas as pd
 from colorama import Fore, Style
 from pydantic import Field, StrictStr
+from pytz import utc
 from tqdm import tqdm
 
 import feast
@@ -256,6 +257,18 @@ def _materialize_one(
                 end_date=end_date,
             )
 
+            # Lets check and see if we can skip this query, because the table hasnt changed
+            # since before the start date of this query
+            with get_snowflake_conn(self.repo_config.offline_store) as conn:
+                query = f"""SELECT SYSTEM$LAST_CHANGE_COMMIT_TIME('{feature_view.batch_source.get_table_query_string()}') AS last_commit_change_time"""
+                last_commit_change_time = (
+                    conn.cursor().execute(query).fetchall()[0][0] / 1_000_000_000
+                )
+            if last_commit_change_time < start_date.astimezone(tz=utc).timestamp():
+                return SnowflakeMaterializationJob(
+                    job_id=job_id, status=MaterializationJobStatus.SUCCEEDED
+                )
+
             fv_latest_values_sql = offline_job.to_sql()
 
             if feature_view.batch_source.field_mapping is not None:
