Test vector_length updates and related Fixes

Signed-off-by: jyejare <jyejare@redhat.com>
feast-dev · franciscojavierarceo · Apr 23, 2025 · Apr 16, 2025 · Apr 16, 2025 · Apr 21, 2025
commit df15f5ae2d5f5e536eb38e715f640c9844a224e4
@@ -90,7 +90,7 @@
 from feast.stream_feature_view import StreamFeatureView
 from feast.transformation.pandas_transformation import PandasTransformation
 from feast.transformation.python_transformation import PythonTransformation
-from feast.utils import _utc_now, _get_feature_view_vector_field_metadata
+from feast.utils import _get_feature_view_vector_field_metadata, _utc_now
 
 warnings.simplefilter("once", DeprecationWarning)
 
@@ -1559,11 +1559,11 @@ def _get_feature_view_and_df_for_online_write(
             df_vector_feature_index = df.columns.get_loc(fv_vector_feature_name)
             if feature_view.features[0].vector_len != 0:
                 if (
-                    feature_view.features[0].vector_len
-                    != df.shape[df_vector_feature_index]
+                    df.shape[df_vector_feature_index]
+                    > feature_view.features[0].vector_len
                 ):
                     raise ValueError(
-                        f"The dataframe for {fv_vector_feature_name} column has {df.shape[1]} vectors, but the feature view {feature_view.name} expects {feature_view.features[0].vector_len}"
+                        f"The dataframe for {fv_vector_feature_name} column has {df.shape[1]} vectors which is greater than expected (i.e {feature_view.features[0].vector_len}) by feature view {feature_view.name}."
                     )
 
         # # Apply transformations if this is an OnDemandFeatureView with write_to_online_store=True
@@ -2513,4 +2513,3 @@ def _validate_data_sources(data_sources: List[DataSource]):
             raise DataSourceRepeatNamesException(case_insensitive_ds_name)
         else:
             ds_names.add(case_insensitive_ds_name)
-
@@ -18,7 +18,11 @@
 from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
 from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 from feast.repo_config import FeastConfigBaseModel
-from feast.utils import _build_retrieve_online_document_record, to_naive_utc, _get_feature_view_vector_field_metadata
+from feast.utils import (
+    _build_retrieve_online_document_record,
+    _get_feature_view_vector_field_metadata,
+    to_naive_utc,
+)
 
 
 class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig):
@@ -161,7 +165,9 @@ def create_index(self, config: RepoConfig, table: FeatureView):
             config: Feast repo configuration object.
             table: FeatureView table for which the index needs to be created.
         """
-        vector_field_length = _get_feature_view_vector_field_metadata(table).vector_len or 512
+        vector_field_length = getattr(
+            _get_feature_view_vector_field_metadata(table), "vector_len", 512
+        )
 
         index_mapping = {
             "properties": {

@@ -19,7 +19,11 @@
 from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
 from feast.protos.feast.types.Value_pb2 import Value as ValueProto
 from feast.repo_config import FeastConfigBaseModel
-from feast.utils import _build_retrieve_online_document_record, to_naive_utc, _get_feature_view_vector_field_metadata
+from feast.utils import (
+    _build_retrieve_online_document_record,
+    _get_feature_view_vector_field_metadata,
+    to_naive_utc,
+)
 
 SCROLL_SIZE = 1000
 
@@ -198,7 +202,9 @@ def create_collection(self, config: RepoConfig, table: FeatureView):
             table: FeatureView table for which the index needs to be created.
         """
 
-        vector_field_length = _get_feature_view_vector_field_metadata(table) or 512
+        vector_field_length = getattr(
+            _get_feature_view_vector_field_metadata(table), "vector_len", 512
+        )
 
         client: QdrantClient = self._get_client(config)
 

@@ -43,9 +43,9 @@
 from feast.types import FEAST_VECTOR_TYPES, PrimitiveFeastType
 from feast.utils import (
     _build_retrieve_online_document_record,
+    _get_feature_view_vector_field_metadata,
     _serialize_vector_to_float_list,
     to_naive_utc,
-    _get_feature_view_vector_field_metadata,
 )
 
 
@@ -172,7 +172,11 @@ def online_write_batch(
                             feature_type_dict.get(feature_name, None)
                             in FEAST_VECTOR_TYPES
                         ):
-                            vector_field_length = _get_feature_view_vector_field_metadata(table).vector_len or 512
+                            vector_field_length = getattr(
+                                _get_feature_view_vector_field_metadata(table),
+                                "vector_len",
+                                512,
+                            )
                             val_bin = serialize_f32(
                                 val.float_list_val.val, vector_field_length
                             )  # type: ignore
@@ -356,7 +360,9 @@ def retrieve_online_documents(
         conn = self._get_conn(config)
         cur = conn.cursor()
 
-        vector_field_length = _get_feature_view_vector_field_metadata(table).vector_len or 512
+        vector_field_length = getattr(
+            _get_feature_view_vector_field_metadata(table), "vector_len", 512
+        )
 
         # Convert the embedding to a binary format instead of using SerializeToString()
         query_embedding_bin = serialize_f32(embedding, vector_field_length)
@@ -480,18 +486,19 @@ def retrieve_online_documents_v2(
         conn = self._get_conn(config)
         cur = conn.cursor()
 
-        if online_store.vector_enabled and not online_store.vector_len:
-            raise ValueError("vector_len is not configured in the online store config")
+        vector_field_length = getattr(
+            _get_feature_view_vector_field_metadata(table), "vector_len", 512
+        )
 
         table_name = _table_id(config.project, table)
         vector_field = _get_vector_field(table)
 
         if online_store.vector_enabled:
-            query_embedding_bin = serialize_f32(query, online_store.vector_len)  # type: ignore
+            query_embedding_bin = serialize_f32(query, vector_field_length)  # type: ignore
             cur.execute(
                 f"""
                 CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0(
-                    vector_value float[{online_store.vector_len}]
+                    vector_value float[{vector_field_length}]
                 );
                 """
             )

@@ -32,6 +32,7 @@
     FeatureViewNotFoundException,
     RequestDataNotFoundInEntityRowsException,
 )
+from feast.field import Field
 from feast.infra.key_encoding_utils import deserialize_entity_key
 from feast.protos.feast.serving.ServingService_pb2 import (
     FieldStatus,
@@ -45,7 +46,6 @@
 from feast.types import ComplexFeastType, PrimitiveFeastType, from_feast_to_pyarrow_type
 from feast.value_type import ValueType
 from feast.version import get_version
-from feast.field import Field
 
 if typing.TYPE_CHECKING:
     from feast.base_feature_view import BaseFeatureView
@@ -1409,7 +1409,7 @@ def _build_retrieve_online_document_record(
 
 
 def _get_feature_view_vector_field_metadata(
-    feature_view: FeatureView,
+    feature_view,
 ) -> Optional[Field]:
     vector_fields = [field for field in feature_view.schema if field.vector_index]
     if len(vector_fields) > 1:

@@ -122,6 +122,7 @@
             name="Embeddings",
             dtype=Array(Float32),
             vector_index=True,
+            vector_len=8,
             vector_search_metric="L2",
         ),
         Field(name="item_id", dtype=String),

@@ -633,7 +633,6 @@ def test_sqlite_get_online_documents() -> None:
         get_example_repo("example_feature_repo_1.py"), "file"
     ) as store:
         store.config.online_store.vector_enabled = True
-        store.config.online_store.vector_len = vector_length
         # Write some data to two tables
         document_embeddings_fv = store.get_feature_view(name="document_embeddings")
 
@@ -870,7 +869,6 @@ def test_sqlite_get_online_documents_v2() -> None:
         get_example_repo("example_feature_repo_1.py"), "file"
     ) as store:
         store.config.online_store.vector_enabled = True
-        store.config.online_store.vector_len = vector_length
         store.config.entity_key_serialization_version = 3
         document_embeddings_fv = store.get_feature_view(name="document_embeddings")
 

@@ -1178,6 +1178,7 @@ def test_stored_writes_with_explode(self):
                         name="vector",
                         dtype=Array(Float32),
                         vector_index=True,
+                        vector_length=5,
                         vector_search_metric="L2",
                     ),
                 ],