From 5050298a97ef184d952c6c13dc2c5d08356e238d Mon Sep 17 00:00:00 2001 From: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:08:44 -0700 Subject: [PATCH] fix: Keep null values of the latest row in deduplicate_keep_latest Signed-off-by: ShryukGrandhi <85529990+ShryukGrandhi@users.noreply.github.com> --- .../aggregation/tiling/tile_subtraction.py | 4 +-- sdk/python/tests/unit/test_aggregation_ops.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/aggregation/tiling/tile_subtraction.py b/sdk/python/feast/aggregation/tiling/tile_subtraction.py index 4493cf0392f..2baab23c0ae 100644 --- a/sdk/python/feast/aggregation/tiling/tile_subtraction.py +++ b/sdk/python/feast/aggregation/tiling/tile_subtraction.py @@ -190,6 +190,6 @@ def deduplicate_keep_latest( return ( df.sort_values(by=timestamp_col, ascending=False) - .groupby(entity_keys, as_index=False) - .first() + .drop_duplicates(subset=entity_keys, keep="first") + .reset_index(drop=True) ) diff --git a/sdk/python/tests/unit/test_aggregation_ops.py b/sdk/python/tests/unit/test_aggregation_ops.py index fade3ee4b1a..69ddedb087d 100644 --- a/sdk/python/tests/unit/test_aggregation_ops.py +++ b/sdk/python/tests/unit/test_aggregation_ops.py @@ -288,3 +288,30 @@ def windowed(frame): assert len(expected) == 8 assert result["event_timestamp"].tolist() == expected["event_timestamp"].tolist() assert result[feature_name].tolist() == expected[feature_name].tolist() + + +def test_deduplicate_keep_latest_keeps_null_values_of_latest_row(): + """A null in the latest row is kept rather than filled from an older row.""" + from feast.aggregation.tiling.tile_subtraction import deduplicate_keep_latest + + start = pd.Timestamp("2024-01-01") + df = pd.DataFrame( + { + "driver_id": [1, 1, 2], + "event_timestamp": [ + start, + start + timedelta(minutes=5), + start, + ], + # std over a window with a single event is NaN + "std_trips_1200s": [2.5, float("nan"), 1.0], + } + ) + + result = deduplicate_keep_latest(df, ["driver_id"], "event_timestamp") + result = result.set_index("driver_id") + + assert len(result) == 2 + assert result.loc[1, "event_timestamp"] == start + timedelta(minutes=5) + assert pd.isna(result.loc[1, "std_trips_1200s"]) + assert result.loc[2, "std_trips_1200s"] == 1.0