From 428c1a68510ed172b14f7b5ef23ad2d4e89604fb Mon Sep 17 00:00:00 2001 From: Anush008 Date: Fri, 25 Oct 2024 17:33:47 +0530 Subject: [PATCH 1/2] feat: Qdrant vectorstore support Signed-off-by: Anush008 --- Makefile | 8 + docs/reference/alpha-vector-database.md | 22 +- docs/reference/online-stores/qdrant.md | 81 +++++ .../feast.infra.online_stores.contrib.rst | 16 + .../trino_offline_store/connectors/upload.py | 1 + .../infra/online_stores/contrib/qdrant.py | 311 ++++++++++++++++++ .../contrib/qdrant_repo_configuration.py | 12 + .../feast/infra/online_stores/vector_store.py | 5 +- sdk/python/feast/repo_config.py | 1 + sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 +- .../requirements/py3.10-ci-requirements.txt | 29 +- .../requirements/py3.11-ci-requirements.txt | 29 +- .../requirements/py3.9-ci-requirements.txt | 29 +- sdk/python/tests/conftest.py | 4 +- .../universal/online_store/qdrant.py | 28 ++ .../online_store/test_universal_online.py | 2 +- setup.py | 4 + 18 files changed, 563 insertions(+), 29 deletions(-) create mode 100644 docs/reference/online-stores/qdrant.md create mode 100644 sdk/python/feast/infra/online_stores/contrib/qdrant.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py diff --git a/Makefile b/Makefile index 22d4d25f4e3..30ac86e8919 100644 --- a/Makefile +++ b/Makefile @@ -351,6 +351,14 @@ test-python-universal-singlestore-online: not test_snowflake" \ sdk/python/tests + test-python-universal-qdrant-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.qdrant_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.qdrant \ + python -m pytest -n 8 --integration \ + -k "test_retrieve_online_documents" \ + sdk/python/tests/integration/online_store/test_universal_online.py + test-python-universal: python -m pytest -n 8 --integration sdk/python/tests diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 06909bd5654..fca31ee4780 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -14,8 +14,9 @@ Below are supported vector databases and implemented features: | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | | SQLite | [x] | [ ] | +| Qdrant | [x] | [x] | -Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. ## Example @@ -113,9 +114,11 @@ print_online_features(features) ``` ### Configuration -We offer two Online Store options for Vector Databases. PGVector and SQLite. + +We offer [PGVector](https://github.com/pgvector/pgvector), [SQLite](https://github.com/asg017/sqlite-vec), [Elasticsearch](https://www.elastic.co) and [Qdrant](https://qdrant.tech/) as Online Store options for Vector Databases. #### Installation with SQLite + If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: ```bash PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ @@ -124,6 +127,19 @@ PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ pyenv install 3.10.14 ``` And you can the Feast install package via: + ```bash pip install feast[sqlite_vec] -``` \ No newline at end of file +``` + +#### Installation with Elasticsearch + +```bash +pip install feast[elasticsearch] +``` + +#### Installation with Qdrant + +```bash +pip install feast[qdrant] +``` diff --git a/docs/reference/online-stores/qdrant.md b/docs/reference/online-stores/qdrant.md new file mode 100644 index 00000000000..66f4a59d24e --- /dev/null +++ b/docs/reference/online-stores/qdrant.md @@ -0,0 +1,81 @@ +# Qdrant online store (contrib) + +## Description + +[Qdrant](http://qdrant.tech) is a vector similarity search engine. It provides a production-ready service with a convenient API to store, search, and manage vectors with additional payload and extended filtering support. It makes it useful for all sorts of neural network or semantic-based matching, faceted search, and other applications. + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[qdrant]'`. + +## Example + +{% code title="feature_store.yaml" %} + +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: qdrant + host: localhost + port: 6333 + vector_len: 384 + write_batch_size: 100 +``` + +{% endcode %} + +The full set of configuration options is available in [QdrantOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.qdrant.QdrantOnlineStoreConfig). + +## Functionality Matrix + +| | Qdrant | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Retrieving online document vectors + +The Qdrant online store supports retrieving document vectors for a given list of entity keys. The document vectors are returned as a dictionary where the key is the entity key and the value is the document vector. The document vector is a dense vector of floats. + +{% code title="python" %} + +```python +from feast import FeatureStore + +feature_store = FeatureStore(repo_path="feature_store.yaml") + +query_vector = [1.0, 2.0, 3.0, 4.0, 5.0] +top_k = 5 + +# Retrieve the top k closest features to the query vector +# Since Qdrant supports multiple vectors per entry, +# the vector to use can be specified in the repo config. +# Reference: https://qdrant.tech/documentation/concepts/vectors/#named-vectors +feature_values = feature_store.retrieve_online_documents( + feature="my_feature", + query=query_vector, + top_k=top_k +) +``` + +{% endcode %} + +These APIs are subject to change in future versions of Feast to improve performance and usability. diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 8c9dd7e5491..2403b5b8d48 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -40,6 +40,22 @@ feast.infra.online\_stores.contrib.elasticsearch\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.qdrant module +------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.qdrant\_repo\_configuration module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.hazelcast\_repo\_configuration module ------------------------------------------------------------------------ diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py index 1b551991932..1cdbf7f01e6 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py @@ -45,6 +45,7 @@ "thrift", "tpcds", "tpch", + "qdrant", } CONNECTORS_WITHOUT_WITH_STATEMENTS: Set[str] = { "bigquery", diff --git a/sdk/python/feast/infra/online_stores/contrib/qdrant.py b/sdk/python/feast/infra/online_stores/contrib/qdrant.py new file mode 100644 index 00000000000..074c52ba5e8 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/qdrant.py @@ -0,0 +1,311 @@ +from __future__ import absolute_import + +import base64 +import json +import logging +import uuid +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from qdrant_client import QdrantClient, models + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import ( + get_list_val_str, + serialize_entity_key, +) +from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel +from feast.utils import _build_retrieve_online_document_record, to_naive_utc + +SCROLL_SIZE = 1000 + +DISTANCE_MAPPING = { + "cosine": models.Distance.COSINE, + "l2": models.Distance.EUCLID, + "dot": models.Distance.DOT, + "l1": models.Distance.MANHATTAN, +} + + +class QdrantOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): + """ + Configuration for the Qdrant online store. + """ + + type: str = "qdrant" + + location: Optional[str] = None + url: Optional[str] = None + port: Optional[int] = 6333 + grpc_port: int = 6334 + prefer_grpc: bool = False + https: Optional[bool] = None + api_key: Optional[str] = None + prefix: Optional[str] = None + timeout: Optional[int] = None + host: Optional[str] = None + path: Optional[str] = None + + # The name of the vector to use. + # Defaults to the single, unnamed vector + # Reference: https://qdrant.tech/documentation/concepts/vectors/#named-vectors + vector_name: str = "" + # The number of point to write in a single request + write_batch_size: Optional[int] = 64 + # Await for the upload results to be applied on the server side. + # If `true`, each request will explicitly wait for the confirmation of completion. Might be slower. + # If `false`, each reequest will return immediately after receiving an acknowledgement. + upload_wait: bool = True + + +class QdrantOnlineStore(OnlineStore): + _client: Optional[QdrantClient] = None + + def _get_client(self, config: RepoConfig) -> QdrantClient: + if self._client: + return self._client + online_store_config = config.online_store + assert isinstance( + online_store_config, QdrantOnlineStoreConfig + ), "Invalid type for online store config" + + assert online_store_config.similarity and ( + online_store_config.similarity.lower() in DISTANCE_MAPPING + ), f"Unsupported distance metric {online_store_config.similarity}" + + self._client = QdrantClient( + location=online_store_config.location, + url=online_store_config.url, + port=online_store_config.port, + grpc_port=online_store_config.grpc_port, + prefer_grpc=online_store_config.prefer_grpc, + https=online_store_config.https, + api_key=online_store_config.api_key, + prefix=online_store_config.prefix, + timeout=online_store_config.timeout, + host=online_store_config.host, + path=online_store_config.path, + ) + return self._client + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + points = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + + timestamp = to_naive_utc(timestamp) + if created_ts is not None: + created_ts = to_naive_utc(created_ts) + for feature_name, value in values.items(): + encoded_value = base64.b64encode(value.SerializeToString()).decode( + "utf-8" + ) + vector_val = json.loads(get_list_val_str(value)) + points.append( + models.PointStruct( + id=uuid.uuid4().hex, + payload={ + "entity_key": entity_key_bin, + "feature_name": feature_name, + "feature_value": encoded_value, + "timestamp": timestamp, + "created_ts": created_ts, + }, + vector={config.online_store.vector_name: vector_val}, + ) + ) + + self._get_client(config).upload_points( + collection_name=table.name, + batch_size=config.online_store.write_batch_size, + points=points, + wait=True, + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conditions: List[models.Condition] = [] + if entity_keys: + conditions.append( + models.FieldCondition( + key="entity_key", + match=models.MatchAny(any=entity_keys), # type: ignore + ) + ) + + if requested_features: + conditions.append( + models.FieldCondition( + key="feature_name", match=models.MatchAny(any=requested_features) + ) + ) + points = [] + next_offset = None + stop_scrolling = False + while not stop_scrolling: + records, next_offset = self._get_client(config).scroll( + collection_name=config.online_store.collection_name, + limit=SCROLL_SIZE, + offset=next_offset, + with_payload=True, + scroll_filter=models.Filter(must=conditions), + ) + stop_scrolling = next_offset is None + + points.extend(records) + + results = [] + for point in points: + assert isinstance(point.payload, Dict), "Invalid value of payload" + results.append( + ( + point.payload["timestamp"], + {point.payload["feature_name"]: point.payload["feature_value"]}, + ) + ) + + return results # type: ignore + + def create_collection(self, config: RepoConfig, table: FeatureView): + """ + Create a collection in Qdrant for the given table. + Args: + config: Feast repo configuration object. + table: FeatureView table for which the index needs to be created. + """ + + client: QdrantClient = self._get_client(config) + + client.create_collection( + collection_name=table.name, + vectors_config={ + config.online_store.vector_name: models.VectorParams( + size=config.online_store.vector_len, + distance=DISTANCE_MAPPING[config.online_store.similarity.lower()], + ) + }, + ) + client.create_payload_index( + collection_name=table.name, + field_name="entity_key", + field_schema=models.PayloadSchemaType.KEYWORD, + ) + client.create_payload_index( + collection_name=table.name, + field_name="feature_name", + field_schema=models.PayloadSchemaType.KEYWORD, + ) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + for table in tables_to_delete: + self._get_client(config).delete_collection(collection_name=table.name) + for table in tables_to_keep: + self.create_collection(config, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + project = config.project + try: + for table in tables: + self._get_client(config).delete_collection(collection_name=table.name) + except Exception as e: + logging.exception(f"Error deleting collection in project {project}: {e}") + raise + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = "cosine", + ) -> List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + result: List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + + if distance_metric and distance_metric.lower() not in DISTANCE_MAPPING: + raise ValueError(f"Unsupported distance metric: {distance_metric}") + points = ( + self._get_client(config) + .query_points( + collection_name=table.name, + query=embedding, + limit=top_k, + with_payload=True, + with_vectors=True, + using=config.online_store.vector_name or None, + ) + .points + ) + for point in points: + payload = point.payload or {} + entity_key = str(payload.get("entity_key")) + feature_value = str(payload.get("feature_value")) + timestamp_str = str(payload.get("timestamp")) + timestamp = datetime.strptime(timestamp_str, "%Y-%m-%dT%H:%M:%S.%f") + distance = point.score + vector_value = str( + point.vector[config.online_store.vector_name] + if isinstance(point.vector, Dict) + else point.vector + ) + + result.append( + _build_retrieve_online_document_record( + entity_key, + base64.b64decode(feature_value), + vector_value, + distance, + timestamp, + config.entity_key_serialization_version, + ) + ) + return result diff --git a/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py new file mode 100644 index 00000000000..eee77bb8775 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.qdrant import ( + QdrantOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="qdrant", online_store_creator=QdrantOnlineStoreCreator + ), +] diff --git a/sdk/python/feast/infra/online_stores/vector_store.py b/sdk/python/feast/infra/online_stores/vector_store.py index 051f9bcaedd..f071cd4347d 100644 --- a/sdk/python/feast/infra/online_stores/vector_store.py +++ b/sdk/python/feast/infra/online_stores/vector_store.py @@ -11,6 +11,9 @@ class VectorStoreConfig: # The vector similarity metric to use in KNN search # It is helpful for vector database that does not support config at retrieval runtime - # E.g. Elasticsearch dense_vector field at + # E.g. + # Elasticsearch: # https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html + # Qdrant: + # https://qdrant.tech/documentation/concepts/search/#metrics similarity: Optional[str] = "cosine" diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 0a5b484e8c7..b2b9374aa97 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -67,6 +67,7 @@ "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", "singlestore": "feast.infra.online_stores.contrib.singlestore_online_store.singlestore.SingleStoreOnlineStore", + "qdrant": "feast.infra.online_stores.contrib.qdrant.QdrantOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 36777ca0be3..b2da53a117c 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.40.0", + "@feast-dev/feast-ui": "0.40.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 7e83084445f..f1ef5bc7980 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.40.0": - version "0.40.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.40.0.tgz#0dc60cbbd4f63d161927321c0bbf57bbfe6b7d09" - integrity sha512-jiCtMYCBvNSfHCjemFRa0NFIIAR5y6spWBnUZyc4GXY2YxGcznw+PZSzOoi7JrOwpNzNPB0PTBUqJgBAxus20w== +"@feast-dev/feast-ui@0.40.1": + version "0.40.1" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.40.1.tgz#02080265706c7af6160aef6163a1a2862bdcb333" + integrity sha512-a50VJVN9haL6z2oSbBwq+DwtqmSXvQptjw7oBoucZKZ8D1lybN4MKqqvE1HlAG7mE+kArVZR5SPRQAq++D4xqA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index bc29f696718..8c940ba84e2 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -273,6 +273,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -282,7 +283,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -291,21 +294,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.5.0 # via # feast (setup.py) @@ -499,6 +509,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -564,7 +575,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -646,6 +659,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -747,6 +761,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -840,7 +856,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -864,7 +880,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -1010,6 +1026,7 @@ urllib3==2.2.3 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # testcontainers @@ -1041,7 +1058,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index a75b57f48eb..4d5d8a71885 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -266,6 +266,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -275,7 +276,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -284,21 +287,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.5.0 # via # feast (setup.py) @@ -490,6 +500,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -555,7 +566,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -637,6 +650,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -738,6 +752,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -831,7 +847,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -855,7 +871,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -986,6 +1002,7 @@ urllib3==2.2.3 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # testcontainers @@ -1017,7 +1034,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 5b8086099c2..2ba384e205e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -275,6 +275,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -284,7 +285,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -293,21 +296,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.0.0 # via # feast (setup.py) @@ -508,6 +518,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -572,7 +583,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -654,6 +667,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -755,6 +769,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -848,7 +864,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -872,7 +888,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -1020,6 +1036,7 @@ urllib3==1.26.20 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # snowflake-connector-python @@ -1052,7 +1069,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 08b8757b955..3b706d3c274 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -183,7 +183,9 @@ def start_test_local_server(repo_path: str, port: int): @pytest.fixture def environment(request, worker_id): e = construct_test_environment( - request.param, worker_id=worker_id, fixture_request=request + request.param, + worker_id=worker_id, + fixture_request=request, ) e.setup() diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py b/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py new file mode 100644 index 00000000000..f65725f41d5 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py @@ -0,0 +1,28 @@ +from typing import Any, Dict + +from testcontainers.qdrant import QdrantContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class QdrantOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = QdrantContainer( + "qdrant/qdrant", + ) + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + return { + "host": self.container.get_container_host_ip(), + "type": "qdrant", + "port": self.container.exposed_rest_port, + "vector_len": 2, + "similarity": "cosine", + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 1a0803acff5..41de53da145 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -845,7 +845,7 @@ def assert_feature_service_entity_mapping_correctness( @pytest.mark.integration -@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch", "qdrant"]) def test_retrieve_online_documents(vectordb_environment, fake_document_data): fs = vectordb_environment.feature_store df, data_source = fake_document_data diff --git a/setup.py b/setup.py index 96a6f311e57..b335d39c2b3 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,8 @@ FAISS_REQUIRED = ["faiss-cpu>=1.7.0,<2"] +QDRANT_REQUIRED = ["qdrant-client>=1.12.0"] + CI_REQUIRED = ( [ "build", @@ -214,6 +216,7 @@ + SINGLESTORE_REQUIRED + OPENTELEMETRY + FAISS_REQUIRED + + QDRANT_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -284,6 +287,7 @@ "singlestore": SINGLESTORE_REQUIRED, "opentelemetry": OPENTELEMETRY, "faiss": FAISS_REQUIRED, + "qdrant": QDRANT_REQUIRED }, include_package_data=True, license="Apache", From f33817c655088cb8cf9becd3b1e617807e2d9ef2 Mon Sep 17 00:00:00 2001 From: Anush008 Date: Sat, 26 Oct 2024 11:33:52 +0530 Subject: [PATCH 2/2] chore: make build-ui again Signed-off-by: Anush008 --- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index b2da53a117c..2a6329a166b 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.40.1", + "@feast-dev/feast-ui": "0.41.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index fb12a2ceafb..24de47b1232 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1570,10 +1570,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.40.1": - version "0.40.1" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.40.1.tgz#02080265706c7af6160aef6163a1a2862bdcb333" - integrity sha512-a50VJVN9haL6z2oSbBwq+DwtqmSXvQptjw7oBoucZKZ8D1lybN4MKqqvE1HlAG7mE+kArVZR5SPRQAq++D4xqA== +"@feast-dev/feast-ui@0.41.0": + version "0.41.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.41.0.tgz#67eca6328131ee524ee6a6f286cfc4386f698053" + integrity sha512-BkVb4zfR+j95IX9FBzeXFyCimG5Za1a3jyLqjmETRO3hpp5OJanpc2N35AaOn8ZPqka00Be/b8NZ8TjbsRWyVg== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^95.12.0"