SQLite vector length

Signed-off-by: jyejare <jyejare@redhat.com>
feast-dev · franciscojavierarceo · Apr 23, 2025 · Apr 16, 2025 · Apr 16, 2025 · Apr 21, 2025
commit 1c6590680baa80234cde5da008582e4e2d716e4c
@@ -62,7 +62,6 @@
 from feast.feast_object import FeastObject
 from feast.feature_service import FeatureService
 from feast.feature_view import DUMMY_ENTITY, DUMMY_ENTITY_NAME, FeatureView
-from feast.field import Field
 from feast.inference import (
     update_data_sources_with_inferred_event_timestamp_col,
     update_feature_views_with_inferred_features_and_entities,
@@ -91,7 +90,7 @@
 from feast.stream_feature_view import StreamFeatureView
 from feast.transformation.pandas_transformation import PandasTransformation
 from feast.transformation.python_transformation import PythonTransformation
-from feast.utils import _utc_now
+from feast.utils import _utc_now, _get_feature_view_vector_field_metadata
 
 warnings.simplefilter("once", DeprecationWarning)
 
@@ -2515,15 +2514,3 @@ def _validate_data_sources(data_sources: List[DataSource]):
         else:
             ds_names.add(case_insensitive_ds_name)
 
-
-def _get_feature_view_vector_field_metadata(
-    feature_view: FeatureView,
-) -> Optional[Field]:
-    vector_fields = [field for field in feature_view.schema if field.vector_index]
-    if len(vector_fields) > 1:
-        raise ValueError(
-            f"Feature view {feature_view.name} has multiple vector fields. Only one vector field per feature view is supported."
-        )
-    if not vector_fields:
-        return None
-    return vector_fields[0]
@@ -45,6 +45,7 @@
     _build_retrieve_online_document_record,
     _serialize_vector_to_float_list,
     to_naive_utc,
+    _get_feature_view_vector_field_metadata,
 )
 
 
@@ -171,8 +172,9 @@ def online_write_batch(
                             feature_type_dict.get(feature_name, None)
                             in FEAST_VECTOR_TYPES
                         ):
+                            vector_field_length = _get_feature_view_vector_field_metadata(table).vector_len or 512
                             val_bin = serialize_f32(
-                                val.float_list_val.val, config.online_store.vector_len
+                                val.float_list_val.val, vector_field_length
                             )  # type: ignore
                         else:
                             val_bin = feast_value_type_to_python_type(val)
@@ -354,15 +356,17 @@ def retrieve_online_documents(
         conn = self._get_conn(config)
         cur = conn.cursor()
 
+        vector_field_length = _get_feature_view_vector_field_metadata(table).vector_len or 512
+
         # Convert the embedding to a binary format instead of using SerializeToString()
-        query_embedding_bin = serialize_f32(embedding, config.online_store.vector_len)
+        query_embedding_bin = serialize_f32(embedding, vector_field_length)
         table_name = _table_id(project, table)
         vector_field = _get_vector_field(table)
 
         cur.execute(
             f"""
             CREATE VIRTUAL TABLE vec_table using vec0(
-                vector_value float[{config.online_store.vector_len}]
+                vector_value float[{vector_field_length}]
         );
         """
         )
@@ -378,7 +382,7 @@ def retrieve_online_documents(
         cur.execute(
             f"""
             CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0(
-                vector_value float[{config.online_store.vector_len}]
+                vector_value float[{vector_field_length}]
             );
             """
         )

@@ -45,6 +45,7 @@
 from feast.types import ComplexFeastType, PrimitiveFeastType, from_feast_to_pyarrow_type
 from feast.value_type import ValueType
 from feast.version import get_version
+from feast.field import Field
 
 if typing.TYPE_CHECKING:
     from feast.base_feature_view import BaseFeatureView
@@ -1405,3 +1406,16 @@ def _build_retrieve_online_document_record(
         vector_value_proto,
         distance_value_proto,
     )
+
+
+def _get_feature_view_vector_field_metadata(
+    feature_view: FeatureView,
+) -> Optional[Field]:
+    vector_fields = [field for field in feature_view.schema if field.vector_index]
+    if len(vector_fields) > 1:
+        raise ValueError(
+            f"Feature view {feature_view.name} has multiple vector fields. Only one vector field per feature view is supported."
+        )
+    if not vector_fields:
+        return None
+    return vector_fields[0]